1/*- 2 * Linux port done by David McCullough <david_mccullough@mcafee.com> 3 * Copyright (C) 2006-2010 David McCullough 4 * Copyright (C) 2004-2005 Intel Corporation. 5 * The license and original author are listed below. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * Copyright (c) 2002-2006 Sam Leffler. All rights reserved. 9 * 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30#if 0 31#include <sys/cdefs.h> 32__FBSDID("$FreeBSD: src/sys/opencrypto/crypto.c,v 1.27 2007/03/21 03:42:51 sam Exp $"); 33#endif 34 35/* 36 * Cryptographic Subsystem. 37 * 38 * This code is derived from the Openbsd Cryptographic Framework (OCF) 39 * that has the copyright shown below. Very little of the original 40 * code remains. 41 */ 42/*- 43 * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 44 * 45 * This code was written by Angelos D. Keromytis in Athens, Greece, in 46 * February 2000. Network Security Technologies Inc. (NSTI) kindly 47 * supported the development of this code. 48 * 49 * Copyright (c) 2000, 2001 Angelos D. Keromytis 50 * 51 * Permission to use, copy, and modify this software with or without fee 52 * is hereby granted, provided that this entire notice is included in 53 * all source code copies of any software which is or includes a copy or 54 * modification of this software. 55 * 56 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 57 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 58 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 59 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 60 * PURPOSE. 61 * 62__FBSDID("$FreeBSD: src/sys/opencrypto/crypto.c,v 1.16 2005/01/07 02:29:16 imp Exp $"); 63 */ 64 65 66#include <linux/version.h> 67#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,38) && !defined(AUTOCONF_INCLUDED) 68#include <linux/config.h> 69#endif 70#include <linux/module.h> 71#include <linux/init.h> 72#include <linux/list.h> 73#include <linux/slab.h> 74#include <linux/wait.h> 75#include <linux/sched.h> 76#include <linux/spinlock.h> 77#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,4) 78#include <linux/kthread.h> 79#endif 80#include <cryptodev.h> 81 82/* 83 * keep track of whether or not we have been initialised, a big 84 * issue if we are linked into the kernel and a driver gets started before 85 * us 86 */ 87static int crypto_initted = 0; 88 89/* 90 * Crypto drivers register themselves by allocating a slot in the 91 * crypto_drivers table with crypto_get_driverid() and then registering 92 * each algorithm they support with crypto_register() and crypto_kregister(). 93 */ 94 95/* 96 * lock on driver table 97 * we track its state as spin_is_locked does not do anything on non-SMP boxes 98 */ 99static spinlock_t crypto_drivers_lock; 100static int crypto_drivers_locked; /* for non-SMP boxes */ 101 102#define CRYPTO_DRIVER_LOCK() \ 103 ({ \ 104 spin_lock_irqsave(&crypto_drivers_lock, d_flags); \ 105 crypto_drivers_locked = 1; \ 106 dprintk("%s,%d: DRIVER_LOCK()\n", __FILE__, __LINE__); \ 107 }) 108#define CRYPTO_DRIVER_UNLOCK() \ 109 ({ \ 110 dprintk("%s,%d: DRIVER_UNLOCK()\n", __FILE__, __LINE__); \ 111 crypto_drivers_locked = 0; \ 112 spin_unlock_irqrestore(&crypto_drivers_lock, d_flags); \ 113 }) 114#define CRYPTO_DRIVER_ASSERT() \ 115 ({ \ 116 if (!crypto_drivers_locked) { \ 117 dprintk("%s,%d: DRIVER_ASSERT!\n", __FILE__, __LINE__); \ 118 } \ 119 }) 120 121/* 122 * Crypto device/driver capabilities structure. 123 * 124 * Synchronization: 125 * (d) - protected by CRYPTO_DRIVER_LOCK() 126 * (q) - protected by CRYPTO_Q_LOCK() 127 * Not tagged fields are read-only. 128 */ 129struct cryptocap { 130 device_t cc_dev; /* (d) device/driver */ 131 u_int32_t cc_sessions; /* (d) # of sessions */ 132 u_int32_t cc_koperations; /* (d) # os asym operations */ 133 /* 134 * Largest possible operator length (in bits) for each type of 135 * encryption algorithm. XXX not used 136 */ 137 u_int16_t cc_max_op_len[CRYPTO_ALGORITHM_MAX + 1]; 138 u_int8_t cc_alg[CRYPTO_ALGORITHM_MAX + 1]; 139 u_int8_t cc_kalg[CRK_ALGORITHM_MAX + 1]; 140 141 int cc_flags; /* (d) flags */ 142#define CRYPTOCAP_F_CLEANUP 0x80000000 /* needs resource cleanup */ 143 int cc_qblocked; /* (q) symmetric q blocked */ 144 int cc_kqblocked; /* (q) asymmetric q blocked */ 145 146 int cc_unqblocked; /* (q) symmetric q blocked */ 147 int cc_unkqblocked; /* (q) asymmetric q blocked */ 148}; 149static struct cryptocap *crypto_drivers = NULL; 150static int crypto_drivers_num = 0; 151 152/* 153 * There are two queues for crypto requests; one for symmetric (e.g. 154 * cipher) operations and one for asymmetric (e.g. MOD)operations. 155 * A single mutex is used to lock access to both queues. We could 156 * have one per-queue but having one simplifies handling of block/unblock 157 * operations. 158 */ 159static LIST_HEAD(crp_q); /* crypto request queue */ 160static LIST_HEAD(crp_kq); /* asym request queue */ 161 162static spinlock_t crypto_q_lock; 163 164int crypto_all_qblocked = 0; /* protect with Q_LOCK */ 165module_param(crypto_all_qblocked, int, 0444); 166MODULE_PARM_DESC(crypto_all_qblocked, "Are all crypto queues blocked"); 167 168int crypto_all_kqblocked = 0; /* protect with Q_LOCK */ 169module_param(crypto_all_kqblocked, int, 0444); 170MODULE_PARM_DESC(crypto_all_kqblocked, "Are all asym crypto queues blocked"); 171 172#define CRYPTO_Q_LOCK() \ 173 ({ \ 174 spin_lock_irqsave(&crypto_q_lock, q_flags); \ 175 dprintk("%s,%d: Q_LOCK()\n", __FILE__, __LINE__); \ 176 }) 177#define CRYPTO_Q_UNLOCK() \ 178 ({ \ 179 dprintk("%s,%d: Q_UNLOCK()\n", __FILE__, __LINE__); \ 180 spin_unlock_irqrestore(&crypto_q_lock, q_flags); \ 181 }) 182 183/* 184 * There are two queues for processing completed crypto requests; one 185 * for the symmetric and one for the asymmetric ops. We only need one 186 * but have two to avoid type futzing (cryptop vs. cryptkop). A single 187 * mutex is used to lock access to both queues. Note that this lock 188 * must be separate from the lock on request queues to insure driver 189 * callbacks don't generate lock order reversals. 190 */ 191static LIST_HEAD(crp_ret_q); /* callback queues */ 192static LIST_HEAD(crp_ret_kq); 193 194static spinlock_t crypto_ret_q_lock; 195#define CRYPTO_RETQ_LOCK() \ 196 ({ \ 197 spin_lock_irqsave(&crypto_ret_q_lock, r_flags); \ 198 dprintk("%s,%d: RETQ_LOCK\n", __FILE__, __LINE__); \ 199 }) 200#define CRYPTO_RETQ_UNLOCK() \ 201 ({ \ 202 dprintk("%s,%d: RETQ_UNLOCK\n", __FILE__, __LINE__); \ 203 spin_unlock_irqrestore(&crypto_ret_q_lock, r_flags); \ 204 }) 205#define CRYPTO_RETQ_EMPTY() (list_empty(&crp_ret_q) && list_empty(&crp_ret_kq)) 206 207#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) 208static kmem_cache_t *cryptop_zone; 209static kmem_cache_t *cryptodesc_zone; 210#else 211static struct kmem_cache *cryptop_zone; 212static struct kmem_cache *cryptodesc_zone; 213#endif 214 215#define debug crypto_debug 216int crypto_debug = 0; 217module_param(crypto_debug, int, 0644); 218MODULE_PARM_DESC(crypto_debug, "Enable debug"); 219EXPORT_SYMBOL(crypto_debug); 220 221/* 222 * Maximum number of outstanding crypto requests before we start 223 * failing requests. We need this to prevent DOS when too many 224 * requests are arriving for us to keep up. Otherwise we will 225 * run the system out of memory. Since crypto is slow, we are 226 * usually the bottleneck that needs to say, enough is enough. 227 * 228 * We cannot print errors when this condition occurs, we are already too 229 * slow, printing anything will just kill us 230 */ 231 232static int crypto_q_cnt = 0; 233module_param(crypto_q_cnt, int, 0444); 234MODULE_PARM_DESC(crypto_q_cnt, 235 "Current number of outstanding crypto requests"); 236 237static int crypto_q_max = 1000; 238module_param(crypto_q_max, int, 0644); 239MODULE_PARM_DESC(crypto_q_max, 240 "Maximum number of outstanding crypto requests"); 241 242#define bootverbose crypto_verbose 243static int crypto_verbose = 0; 244module_param(crypto_verbose, int, 0644); 245MODULE_PARM_DESC(crypto_verbose, 246 "Enable verbose crypto startup"); 247 248int crypto_usercrypto = 1; /* userland may do crypto reqs */ 249module_param(crypto_usercrypto, int, 0644); 250MODULE_PARM_DESC(crypto_usercrypto, 251 "Enable/disable user-mode access to crypto support"); 252 253int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ 254module_param(crypto_userasymcrypto, int, 0644); 255MODULE_PARM_DESC(crypto_userasymcrypto, 256 "Enable/disable user-mode access to asymmetric crypto support"); 257 258int crypto_devallowsoft = 0; /* only use hardware crypto */ 259module_param(crypto_devallowsoft, int, 0644); 260MODULE_PARM_DESC(crypto_devallowsoft, 261 "Enable/disable use of software crypto support"); 262 263/* 264 * This parameter controls the maximum number of crypto operations to 265 * do consecutively in the crypto kernel thread before scheduling to allow 266 * other processes to run. Without it, it is possible to get into a 267 * situation where the crypto thread never allows any other processes to run. 268 * Default to 1000 which should be less than one second. 269 */ 270static int crypto_max_loopcount = 1000; 271module_param(crypto_max_loopcount, int, 0644); 272MODULE_PARM_DESC(crypto_max_loopcount, 273 "Maximum number of crypto ops to do before yielding to other processes"); 274 275#ifndef CONFIG_NR_CPUS 276#define CONFIG_NR_CPUS 1 277#endif 278 279static struct task_struct *cryptoproc[CONFIG_NR_CPUS]; 280static struct task_struct *cryptoretproc[CONFIG_NR_CPUS]; 281static DECLARE_WAIT_QUEUE_HEAD(cryptoproc_wait); 282static DECLARE_WAIT_QUEUE_HEAD(cryptoretproc_wait); 283 284static int crypto_proc(void *arg); 285static int crypto_ret_proc(void *arg); 286static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint); 287static int crypto_kinvoke(struct cryptkop *krp, int flags); 288static void crypto_exit(void); 289static int crypto_init(void); 290 291static struct cryptostats cryptostats; 292 293static struct cryptocap * 294crypto_checkdriver(u_int32_t hid) 295{ 296 if (crypto_drivers == NULL) 297 return NULL; 298 return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]); 299} 300 301/* 302 * Compare a driver's list of supported algorithms against another 303 * list; return non-zero if all algorithms are supported. 304 */ 305static int 306driver_suitable(const struct cryptocap *cap, const struct cryptoini *cri) 307{ 308 const struct cryptoini *cr; 309 310 /* See if all the algorithms are supported. */ 311 for (cr = cri; cr; cr = cr->cri_next) 312 if (cap->cc_alg[cr->cri_alg] == 0) 313 return 0; 314 return 1; 315} 316 317 318/* 319 * Select a driver for a new session that supports the specified 320 * algorithms and, optionally, is constrained according to the flags. 321 * The algorithm we use here is pretty stupid; just use the 322 * first driver that supports all the algorithms we need. If there 323 * are multiple drivers we choose the driver with the fewest active 324 * sessions. We prefer hardware-backed drivers to software ones. 325 * 326 * XXX We need more smarts here (in real life too, but that's 327 * XXX another story altogether). 328 */ 329static struct cryptocap * 330crypto_select_driver(const struct cryptoini *cri, int flags) 331{ 332 struct cryptocap *cap, *best; 333 int match, hid; 334 335 CRYPTO_DRIVER_ASSERT(); 336 337 /* 338 * Look first for hardware crypto devices if permitted. 339 */ 340 if (flags & CRYPTOCAP_F_HARDWARE) 341 match = CRYPTOCAP_F_HARDWARE; 342 else 343 match = CRYPTOCAP_F_SOFTWARE; 344 best = NULL; 345again: 346 for (hid = 0; hid < crypto_drivers_num; hid++) { 347 cap = &crypto_drivers[hid]; 348 /* 349 * If it's not initialized, is in the process of 350 * going away, or is not appropriate (hardware 351 * or software based on match), then skip. 352 */ 353 if (cap->cc_dev == NULL || 354 (cap->cc_flags & CRYPTOCAP_F_CLEANUP) || 355 (cap->cc_flags & match) == 0) 356 continue; 357 358 /* verify all the algorithms are supported. */ 359 if (driver_suitable(cap, cri)) { 360 if (best == NULL || 361 cap->cc_sessions < best->cc_sessions) 362 best = cap; 363 } 364 } 365 if (best != NULL) 366 return best; 367 if (match == CRYPTOCAP_F_HARDWARE && (flags & CRYPTOCAP_F_SOFTWARE)) { 368 /* sort of an Algol 68-style for loop */ 369 match = CRYPTOCAP_F_SOFTWARE; 370 goto again; 371 } 372 return best; 373} 374 375/* 376 * Create a new session. The crid argument specifies a crypto 377 * driver to use or constraints on a driver to select (hardware 378 * only, software only, either). Whatever driver is selected 379 * must be capable of the requested crypto algorithms. 380 */ 381int 382crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int crid) 383{ 384 struct cryptocap *cap; 385 u_int32_t hid, lid; 386 int err; 387 unsigned long d_flags; 388 389 CRYPTO_DRIVER_LOCK(); 390 if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { 391 /* 392 * Use specified driver; verify it is capable. 393 */ 394 cap = crypto_checkdriver(crid); 395 if (cap != NULL && !driver_suitable(cap, cri)) 396 cap = NULL; 397 } else { 398 /* 399 * No requested driver; select based on crid flags. 400 */ 401 cap = crypto_select_driver(cri, crid); 402 /* 403 * if NULL then can't do everything in one session. 404 * XXX Fix this. We need to inject a "virtual" session 405 * XXX layer right about here. 406 */ 407 } 408 if (cap != NULL) { 409 /* Call the driver initialization routine. */ 410 hid = cap - crypto_drivers; 411 lid = hid; /* Pass the driver ID. */ 412 cap->cc_sessions++; 413 CRYPTO_DRIVER_UNLOCK(); 414 err = CRYPTODEV_NEWSESSION(cap->cc_dev, &lid, cri); 415 CRYPTO_DRIVER_LOCK(); 416 if (err == 0) { 417 (*sid) = (cap->cc_flags & 0xff000000) 418 | (hid & 0x00ffffff); 419 (*sid) <<= 32; 420 (*sid) |= (lid & 0xffffffff); 421 } else 422 cap->cc_sessions--; 423 } else 424 err = EINVAL; 425 CRYPTO_DRIVER_UNLOCK(); 426 return err; 427} 428 429static void 430crypto_remove(struct cryptocap *cap) 431{ 432 CRYPTO_DRIVER_ASSERT(); 433 if (cap->cc_sessions == 0 && cap->cc_koperations == 0) 434 bzero(cap, sizeof(*cap)); 435} 436 437/* 438 * Delete an existing session (or a reserved session on an unregistered 439 * driver). 440 */ 441int 442crypto_freesession(u_int64_t sid) 443{ 444 struct cryptocap *cap; 445 u_int32_t hid; 446 int err = 0; 447 unsigned long d_flags; 448 449 dprintk("%s()\n", __FUNCTION__); 450 CRYPTO_DRIVER_LOCK(); 451 452 if (crypto_drivers == NULL) { 453 err = EINVAL; 454 goto done; 455 } 456 457 /* Determine two IDs. */ 458 hid = CRYPTO_SESID2HID(sid); 459 460 if (hid >= crypto_drivers_num) { 461 dprintk("%s - INVALID DRIVER NUM %d\n", __FUNCTION__, hid); 462 err = ENOENT; 463 goto done; 464 } 465 cap = &crypto_drivers[hid]; 466 467 if (cap->cc_dev) { 468 CRYPTO_DRIVER_UNLOCK(); 469 /* Call the driver cleanup routine, if available, unlocked. */ 470 err = CRYPTODEV_FREESESSION(cap->cc_dev, sid); 471 CRYPTO_DRIVER_LOCK(); 472 } 473 474 if (cap->cc_sessions) 475 cap->cc_sessions--; 476 477 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) 478 crypto_remove(cap); 479 480done: 481 CRYPTO_DRIVER_UNLOCK(); 482 return err; 483} 484 485/* 486 * Return an unused driver id. Used by drivers prior to registering 487 * support for the algorithms they handle. 488 */ 489int32_t 490crypto_get_driverid(device_t dev, int flags) 491{ 492 struct cryptocap *newdrv; 493 int i; 494 unsigned long d_flags; 495 496 if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { 497 printf("%s: no flags specified when registering driver\n", 498 device_get_nameunit(dev)); 499 return -1; 500 } 501 502 CRYPTO_DRIVER_LOCK(); 503 504 for (i = 0; i < crypto_drivers_num; i++) { 505 if (crypto_drivers[i].cc_dev == NULL && 506 (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0) { 507 break; 508 } 509 } 510 511 /* Out of entries, allocate some more. */ 512 if (i == crypto_drivers_num) { 513 /* Be careful about wrap-around. */ 514 if (2 * crypto_drivers_num <= crypto_drivers_num) { 515 CRYPTO_DRIVER_UNLOCK(); 516 printk("crypto: driver count wraparound!\n"); 517 return -1; 518 } 519 520 newdrv = kmalloc(2 * crypto_drivers_num * sizeof(struct cryptocap), 521 GFP_KERNEL); 522 if (newdrv == NULL) { 523 CRYPTO_DRIVER_UNLOCK(); 524 printk("crypto: no space to expand driver table!\n"); 525 return -1; 526 } 527 528 memcpy(newdrv, crypto_drivers, 529 crypto_drivers_num * sizeof(struct cryptocap)); 530 memset(&newdrv[crypto_drivers_num], 0, 531 crypto_drivers_num * sizeof(struct cryptocap)); 532 533 crypto_drivers_num *= 2; 534 535 kfree(crypto_drivers); 536 crypto_drivers = newdrv; 537 } 538 539 /* NB: state is zero'd on free */ 540 crypto_drivers[i].cc_sessions = 1; /* Mark */ 541 crypto_drivers[i].cc_dev = dev; 542 crypto_drivers[i].cc_flags = flags; 543 if (bootverbose) 544 printf("crypto: assign %s driver id %u, flags %u\n", 545 device_get_nameunit(dev), i, flags); 546 547 CRYPTO_DRIVER_UNLOCK(); 548 549 return i; 550} 551 552/* 553 * Lookup a driver by name. We match against the full device 554 * name and unit, and against just the name. The latter gives 555 * us a simple widlcarding by device name. On success return the 556 * driver/hardware identifier; otherwise return -1. 557 */ 558int 559crypto_find_driver(const char *match) 560{ 561 int i, len = strlen(match); 562 unsigned long d_flags; 563 564 CRYPTO_DRIVER_LOCK(); 565 for (i = 0; i < crypto_drivers_num; i++) { 566 device_t dev = crypto_drivers[i].cc_dev; 567 if (dev == NULL || 568 (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP)) 569 continue; 570 if (strncmp(match, device_get_nameunit(dev), len) == 0 || 571 strncmp(match, device_get_name(dev), len) == 0) 572 break; 573 } 574 CRYPTO_DRIVER_UNLOCK(); 575 return i < crypto_drivers_num ? i : -1; 576} 577 578/* 579 * Return the device_t for the specified driver or NULL 580 * if the driver identifier is invalid. 581 */ 582device_t 583crypto_find_device_byhid(int hid) 584{ 585 struct cryptocap *cap = crypto_checkdriver(hid); 586 return cap != NULL ? cap->cc_dev : NULL; 587} 588 589/* 590 * Return the device/driver capabilities. 591 */ 592int 593crypto_getcaps(int hid) 594{ 595 struct cryptocap *cap = crypto_checkdriver(hid); 596 return cap != NULL ? cap->cc_flags : 0; 597} 598 599/* 600 * Register support for a key-related algorithm. This routine 601 * is called once for each algorithm supported a driver. 602 */ 603int 604crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags) 605{ 606 struct cryptocap *cap; 607 int err; 608 unsigned long d_flags; 609 610 dprintk("%s()\n", __FUNCTION__); 611 CRYPTO_DRIVER_LOCK(); 612 613 cap = crypto_checkdriver(driverid); 614 if (cap != NULL && 615 (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) { 616 /* 617 * XXX Do some performance testing to determine placing. 618 * XXX We probably need an auxiliary data structure that 619 * XXX describes relative performances. 620 */ 621 622 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 623 if (bootverbose) 624 printf("crypto: %s registers key alg %u flags %u\n" 625 , device_get_nameunit(cap->cc_dev) 626 , kalg 627 , flags 628 ); 629 err = 0; 630 } else 631 err = EINVAL; 632 633 CRYPTO_DRIVER_UNLOCK(); 634 return err; 635} 636 637/* 638 * Register support for a non-key-related algorithm. This routine 639 * is called once for each such algorithm supported by a driver. 640 */ 641int 642crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen, 643 u_int32_t flags) 644{ 645 struct cryptocap *cap; 646 int err; 647 unsigned long d_flags; 648 649 dprintk("%s(id=0x%x, alg=%d, maxoplen=%d, flags=0x%x)\n", __FUNCTION__, 650 driverid, alg, maxoplen, flags); 651 652 CRYPTO_DRIVER_LOCK(); 653 654 cap = crypto_checkdriver(driverid); 655 /* NB: algorithms are in the range [1..max] */ 656 if (cap != NULL && 657 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) { 658 /* 659 * XXX Do some performance testing to determine placing. 660 * XXX We probably need an auxiliary data structure that 661 * XXX describes relative performances. 662 */ 663 664 cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 665 cap->cc_max_op_len[alg] = maxoplen; 666 if (bootverbose) 667 printf("crypto: %s registers alg %u flags %u maxoplen %u\n" 668 , device_get_nameunit(cap->cc_dev) 669 , alg 670 , flags 671 , maxoplen 672 ); 673 cap->cc_sessions = 0; /* Unmark */ 674 err = 0; 675 } else 676 err = EINVAL; 677 678 CRYPTO_DRIVER_UNLOCK(); 679 return err; 680} 681 682static void 683driver_finis(struct cryptocap *cap) 684{ 685 u_int32_t ses, kops; 686 687 CRYPTO_DRIVER_ASSERT(); 688 689 ses = cap->cc_sessions; 690 kops = cap->cc_koperations; 691 bzero(cap, sizeof(*cap)); 692 if (ses != 0 || kops != 0) { 693 /* 694 * If there are pending sessions, 695 * just mark as invalid. 696 */ 697 cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 698 cap->cc_sessions = ses; 699 cap->cc_koperations = kops; 700 } 701} 702 703/* 704 * Unregister a crypto driver. If there are pending sessions using it, 705 * leave enough information around so that subsequent calls using those 706 * sessions will correctly detect the driver has been unregistered and 707 * reroute requests. 708 */ 709int 710crypto_unregister(u_int32_t driverid, int alg) 711{ 712 struct cryptocap *cap; 713 int i, err; 714 unsigned long d_flags; 715 716 dprintk("%s()\n", __FUNCTION__); 717 CRYPTO_DRIVER_LOCK(); 718 719 cap = crypto_checkdriver(driverid); 720 if (cap != NULL && 721 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) && 722 cap->cc_alg[alg] != 0) { 723 cap->cc_alg[alg] = 0; 724 cap->cc_max_op_len[alg] = 0; 725 726 /* Was this the last algorithm ? */ 727 for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++) 728 if (cap->cc_alg[i] != 0) 729 break; 730 731 if (i == CRYPTO_ALGORITHM_MAX + 1) 732 driver_finis(cap); 733 err = 0; 734 } else 735 err = EINVAL; 736 CRYPTO_DRIVER_UNLOCK(); 737 return err; 738} 739 740/* 741 * Unregister all algorithms associated with a crypto driver. 742 * If there are pending sessions using it, leave enough information 743 * around so that subsequent calls using those sessions will 744 * correctly detect the driver has been unregistered and reroute 745 * requests. 746 */ 747int 748crypto_unregister_all(u_int32_t driverid) 749{ 750 struct cryptocap *cap; 751 int err; 752 unsigned long d_flags; 753 754 dprintk("%s()\n", __FUNCTION__); 755 CRYPTO_DRIVER_LOCK(); 756 cap = crypto_checkdriver(driverid); 757 if (cap != NULL) { 758 driver_finis(cap); 759 err = 0; 760 } else 761 err = EINVAL; 762 CRYPTO_DRIVER_UNLOCK(); 763 764 return err; 765} 766 767/* 768 * Clear blockage on a driver. The what parameter indicates whether 769 * the driver is now ready for cryptop's and/or cryptokop's. 770 */ 771int 772crypto_unblock(u_int32_t driverid, int what) 773{ 774 struct cryptocap *cap; 775 int err; 776 unsigned long q_flags; 777 778 CRYPTO_Q_LOCK(); 779 cap = crypto_checkdriver(driverid); 780 if (cap != NULL) { 781 if (what & CRYPTO_SYMQ) { 782 cap->cc_qblocked = 0; 783 cap->cc_unqblocked = 0; 784 crypto_all_qblocked = 0; 785 } 786 if (what & CRYPTO_ASYMQ) { 787 cap->cc_kqblocked = 0; 788 cap->cc_unkqblocked = 0; 789 crypto_all_kqblocked = 0; 790 } 791 wake_up_interruptible(&cryptoproc_wait); 792 err = 0; 793 } else 794 err = EINVAL; 795 CRYPTO_Q_UNLOCK(); //DAVIDM should this be a driver lock 796 797 return err; 798} 799 800/* 801 * Add a crypto request to a queue, to be processed by the kernel thread. 802 */ 803int 804crypto_dispatch(struct cryptop *crp) 805{ 806 struct cryptocap *cap; 807 int result = -1; 808 unsigned long q_flags; 809 810 dprintk("%s()\n", __FUNCTION__); 811 812 cryptostats.cs_ops++; 813 814 CRYPTO_Q_LOCK(); 815 if (crypto_q_cnt >= crypto_q_max) { 816 cryptostats.cs_drops++; 817 CRYPTO_Q_UNLOCK(); 818 return ENOMEM; 819 } 820 crypto_q_cnt++; 821 822 /* make sure we are starting a fresh run on this crp. */ 823 crp->crp_flags &= ~CRYPTO_F_DONE; 824 crp->crp_etype = 0; 825 826 /* 827 * Caller marked the request to be processed immediately; dispatch 828 * it directly to the driver unless the driver is currently blocked. 829 */ 830 if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) { 831 int hid = CRYPTO_SESID2HID(crp->crp_sid); 832 cap = crypto_checkdriver(hid); 833 /* Driver cannot disappear when there is an active session. */ 834 KASSERT(cap != NULL, ("%s: Driver disappeared.", __func__)); 835 if (!cap->cc_qblocked) { 836 crypto_all_qblocked = 0; 837 crypto_drivers[hid].cc_unqblocked = 1; 838 CRYPTO_Q_UNLOCK(); 839 result = crypto_invoke(cap, crp, 0); 840 CRYPTO_Q_LOCK(); 841 if (result == ERESTART) 842 if (crypto_drivers[hid].cc_unqblocked) 843 crypto_drivers[hid].cc_qblocked = 1; 844 crypto_drivers[hid].cc_unqblocked = 0; 845 } 846 } 847 if (result == ERESTART) { 848 /* 849 * The driver ran out of resources, mark the 850 * driver ``blocked'' for cryptop's and put 851 * the request back in the queue. It would 852 * best to put the request back where we got 853 * it but that's hard so for now we put it 854 * at the front. This should be ok; putting 855 * it at the end does not work. 856 */ 857 list_add(&crp->crp_next, &crp_q); 858 cryptostats.cs_blocks++; 859 result = 0; 860 } else if (result == -1) { 861 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 862 result = 0; 863 } 864 wake_up_interruptible(&cryptoproc_wait); 865 CRYPTO_Q_UNLOCK(); 866 return result; 867} 868 869/* 870 * Add an asymetric crypto request to a queue, 871 * to be processed by the kernel thread. 872 */ 873int 874crypto_kdispatch(struct cryptkop *krp) 875{ 876 int error; 877 unsigned long q_flags; 878 879 cryptostats.cs_kops++; 880 881 error = crypto_kinvoke(krp, krp->krp_crid); 882 if (error == ERESTART) { 883 CRYPTO_Q_LOCK(); 884 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); 885 wake_up_interruptible(&cryptoproc_wait); 886 CRYPTO_Q_UNLOCK(); 887 error = 0; 888 } 889 return error; 890} 891 892/* 893 * Verify a driver is suitable for the specified operation. 894 */ 895static __inline int 896kdriver_suitable(const struct cryptocap *cap, const struct cryptkop *krp) 897{ 898 return (cap->cc_kalg[krp->krp_op] & CRYPTO_ALG_FLAG_SUPPORTED) != 0; 899} 900 901/* 902 * Select a driver for an asym operation. The driver must 903 * support the necessary algorithm. The caller can constrain 904 * which device is selected with the flags parameter. The 905 * algorithm we use here is pretty stupid; just use the first 906 * driver that supports the algorithms we need. If there are 907 * multiple suitable drivers we choose the driver with the 908 * fewest active operations. We prefer hardware-backed 909 * drivers to software ones when either may be used. 910 */ 911static struct cryptocap * 912crypto_select_kdriver(const struct cryptkop *krp, int flags) 913{ 914 struct cryptocap *cap, *best, *blocked; 915 int match, hid; 916 917 CRYPTO_DRIVER_ASSERT(); 918 919 /* 920 * Look first for hardware crypto devices if permitted. 921 */ 922 if (flags & CRYPTOCAP_F_HARDWARE) 923 match = CRYPTOCAP_F_HARDWARE; 924 else 925 match = CRYPTOCAP_F_SOFTWARE; 926 best = NULL; 927 blocked = NULL; 928again: 929 for (hid = 0; hid < crypto_drivers_num; hid++) { 930 cap = &crypto_drivers[hid]; 931 /* 932 * If it's not initialized, is in the process of 933 * going away, or is not appropriate (hardware 934 * or software based on match), then skip. 935 */ 936 if (cap->cc_dev == NULL || 937 (cap->cc_flags & CRYPTOCAP_F_CLEANUP) || 938 (cap->cc_flags & match) == 0) 939 continue; 940 941 /* verify all the algorithms are supported. */ 942 if (kdriver_suitable(cap, krp)) { 943 if (best == NULL || 944 cap->cc_koperations < best->cc_koperations) 945 best = cap; 946 } 947 } 948 if (best != NULL) 949 return best; 950 if (match == CRYPTOCAP_F_HARDWARE && (flags & CRYPTOCAP_F_SOFTWARE)) { 951 /* sort of an Algol 68-style for loop */ 952 match = CRYPTOCAP_F_SOFTWARE; 953 goto again; 954 } 955 return best; 956} 957 958/* 959 * Dispatch an assymetric crypto request. 960 */ 961static int 962crypto_kinvoke(struct cryptkop *krp, int crid) 963{ 964 struct cryptocap *cap = NULL; 965 int error; 966 unsigned long d_flags; 967 968 KASSERT(krp != NULL, ("%s: krp == NULL", __func__)); 969 KASSERT(krp->krp_callback != NULL, 970 ("%s: krp->crp_callback == NULL", __func__)); 971 972 CRYPTO_DRIVER_LOCK(); 973 if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { 974 cap = crypto_checkdriver(crid); 975 if (cap != NULL) { 976 /* 977 * Driver present, it must support the necessary 978 * algorithm and, if s/w drivers are excluded, 979 * it must be registered as hardware-backed. 980 */ 981 if (!kdriver_suitable(cap, krp) || 982 (!crypto_devallowsoft && 983 (cap->cc_flags & CRYPTOCAP_F_HARDWARE) == 0)) 984 cap = NULL; 985 } 986 } else { 987 /* 988 * No requested driver; select based on crid flags. 989 */ 990 if (!crypto_devallowsoft) /* NB: disallow s/w drivers */ 991 crid &= ~CRYPTOCAP_F_SOFTWARE; 992 cap = crypto_select_kdriver(krp, crid); 993 } 994 if (cap != NULL && !cap->cc_kqblocked) { 995 krp->krp_hid = cap - crypto_drivers; 996 cap->cc_koperations++; 997 CRYPTO_DRIVER_UNLOCK(); 998 error = CRYPTODEV_KPROCESS(cap->cc_dev, krp, 0); 999 CRYPTO_DRIVER_LOCK(); 1000 if (error == ERESTART) { 1001 cap->cc_koperations--; 1002 CRYPTO_DRIVER_UNLOCK(); 1003 return (error); 1004 } 1005 /* return the actual device used */ 1006 krp->krp_crid = krp->krp_hid; 1007 } else { 1008 /* 1009 * NB: cap is !NULL if device is blocked; in 1010 * that case return ERESTART so the operation 1011 * is resubmitted if possible. 1012 */ 1013 error = (cap == NULL) ? ENODEV : ERESTART; 1014 } 1015 CRYPTO_DRIVER_UNLOCK(); 1016 1017 if (error) { 1018 krp->krp_status = error; 1019 crypto_kdone(krp); 1020 } 1021 return 0; 1022} 1023 1024 1025/* 1026 * Dispatch a crypto request to the appropriate crypto devices. 1027 */ 1028static int 1029crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint) 1030{ 1031 KASSERT(crp != NULL, ("%s: crp == NULL", __func__)); 1032 KASSERT(crp->crp_callback != NULL, 1033 ("%s: crp->crp_callback == NULL", __func__)); 1034 KASSERT(crp->crp_desc != NULL, ("%s: crp->crp_desc == NULL", __func__)); 1035 1036 dprintk("%s()\n", __FUNCTION__); 1037 1038#ifdef CRYPTO_TIMING 1039 if (crypto_timing) 1040 crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp); 1041#endif 1042 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) { 1043 struct cryptodesc *crd; 1044 u_int64_t nid; 1045 1046 /* 1047 * Driver has unregistered; migrate the session and return 1048 * an error to the caller so they'll resubmit the op. 1049 * 1050 * XXX: What if there are more already queued requests for this 1051 * session? 1052 */ 1053 crypto_freesession(crp->crp_sid); 1054 1055 for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next) 1056 crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI); 1057 1058 /* XXX propagate flags from initial session? */ 1059 if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 1060 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0) 1061 crp->crp_sid = nid; 1062 1063 crp->crp_etype = EAGAIN; 1064 crypto_done(crp); 1065 return 0; 1066 } else { 1067 /* 1068 * Invoke the driver to process the request. 1069 */ 1070 return CRYPTODEV_PROCESS(cap->cc_dev, crp, hint); 1071 } 1072} 1073 1074/* 1075 * Release a set of crypto descriptors. 1076 */ 1077void 1078crypto_freereq(struct cryptop *crp) 1079{ 1080 struct cryptodesc *crd; 1081 1082 if (crp == NULL) 1083 return; 1084 1085#ifdef DIAGNOSTIC 1086 { 1087 struct cryptop *crp2; 1088 unsigned long q_flags; 1089 1090 CRYPTO_Q_LOCK(); 1091 TAILQ_FOREACH(crp2, &crp_q, crp_next) { 1092 KASSERT(crp2 != crp, 1093 ("Freeing cryptop from the crypto queue (%p).", 1094 crp)); 1095 } 1096 CRYPTO_Q_UNLOCK(); 1097 CRYPTO_RETQ_LOCK(); 1098 TAILQ_FOREACH(crp2, &crp_ret_q, crp_next) { 1099 KASSERT(crp2 != crp, 1100 ("Freeing cryptop from the return queue (%p).", 1101 crp)); 1102 } 1103 CRYPTO_RETQ_UNLOCK(); 1104 } 1105#endif 1106 1107 while ((crd = crp->crp_desc) != NULL) { 1108 crp->crp_desc = crd->crd_next; 1109 kmem_cache_free(cryptodesc_zone, crd); 1110 } 1111 kmem_cache_free(cryptop_zone, crp); 1112} 1113 1114/* 1115 * Acquire a set of crypto descriptors. 1116 */ 1117struct cryptop * 1118crypto_getreq(int num) 1119{ 1120 struct cryptodesc *crd; 1121 struct cryptop *crp; 1122 1123 crp = kmem_cache_alloc(cryptop_zone, SLAB_ATOMIC); 1124 if (crp != NULL) { 1125 memset(crp, 0, sizeof(*crp)); 1126 INIT_LIST_HEAD(&crp->crp_next); 1127 init_waitqueue_head(&crp->crp_waitq); 1128 while (num--) { 1129 crd = kmem_cache_alloc(cryptodesc_zone, SLAB_ATOMIC); 1130 if (crd == NULL) { 1131 crypto_freereq(crp); 1132 return NULL; 1133 } 1134 memset(crd, 0, sizeof(*crd)); 1135 crd->crd_next = crp->crp_desc; 1136 crp->crp_desc = crd; 1137 } 1138 } 1139 return crp; 1140} 1141 1142/* 1143 * Invoke the callback on behalf of the driver. 1144 */ 1145void 1146crypto_done(struct cryptop *crp) 1147{ 1148 unsigned long q_flags; 1149 1150 dprintk("%s()\n", __FUNCTION__); 1151 if ((crp->crp_flags & CRYPTO_F_DONE) == 0) { 1152 crp->crp_flags |= CRYPTO_F_DONE; 1153 CRYPTO_Q_LOCK(); 1154 crypto_q_cnt--; 1155 CRYPTO_Q_UNLOCK(); 1156 } else 1157 printk("crypto: crypto_done op already done, flags 0x%x", 1158 crp->crp_flags); 1159 if (crp->crp_etype != 0) 1160 cryptostats.cs_errs++; 1161 /* 1162 * CBIMM means unconditionally do the callback immediately; 1163 * CBIFSYNC means do the callback immediately only if the 1164 * operation was done synchronously. Both are used to avoid 1165 * doing extraneous context switches; the latter is mostly 1166 * used with the software crypto driver. 1167 */ 1168 if ((crp->crp_flags & CRYPTO_F_CBIMM) || 1169 ((crp->crp_flags & CRYPTO_F_CBIFSYNC) && 1170 (CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SYNC))) { 1171 /* 1172 * Do the callback directly. This is ok when the 1173 * callback routine does very little (e.g. the 1174 * /dev/crypto callback method just does a wakeup). 1175 */ 1176 crp->crp_callback(crp); 1177 } else { 1178 unsigned long r_flags; 1179 /* 1180 * Normal case; queue the callback for the thread. 1181 */ 1182 CRYPTO_RETQ_LOCK(); 1183 wake_up_interruptible(&cryptoretproc_wait);/* shared wait channel */ 1184 TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); 1185 CRYPTO_RETQ_UNLOCK(); 1186 } 1187} 1188 1189/* 1190 * Invoke the callback on behalf of the driver. 1191 */ 1192void 1193crypto_kdone(struct cryptkop *krp) 1194{ 1195 struct cryptocap *cap; 1196 unsigned long d_flags; 1197 1198 if ((krp->krp_flags & CRYPTO_KF_DONE) != 0) 1199 printk("crypto: crypto_kdone op already done, flags 0x%x", 1200 krp->krp_flags); 1201 krp->krp_flags |= CRYPTO_KF_DONE; 1202 if (krp->krp_status != 0) 1203 cryptostats.cs_kerrs++; 1204 1205 CRYPTO_DRIVER_LOCK(); 1206 /* XXX: What if driver is loaded in the meantime? */ 1207 if (krp->krp_hid < crypto_drivers_num) { 1208 cap = &crypto_drivers[krp->krp_hid]; 1209 cap->cc_koperations--; 1210 KASSERT(cap->cc_koperations >= 0, ("cc_koperations < 0")); 1211 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) 1212 crypto_remove(cap); 1213 } 1214 CRYPTO_DRIVER_UNLOCK(); 1215 1216 /* 1217 * CBIMM means unconditionally do the callback immediately; 1218 * This is used to avoid doing extraneous context switches 1219 */ 1220 if ((krp->krp_flags & CRYPTO_KF_CBIMM)) { 1221 /* 1222 * Do the callback directly. This is ok when the 1223 * callback routine does very little (e.g. the 1224 * /dev/crypto callback method just does a wakeup). 1225 */ 1226 krp->krp_callback(krp); 1227 } else { 1228 unsigned long r_flags; 1229 /* 1230 * Normal case; queue the callback for the thread. 1231 */ 1232 CRYPTO_RETQ_LOCK(); 1233 wake_up_interruptible(&cryptoretproc_wait);/* shared wait channel */ 1234 TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next); 1235 CRYPTO_RETQ_UNLOCK(); 1236 } 1237} 1238 1239int 1240crypto_getfeat(int *featp) 1241{ 1242 int hid, kalg, feat = 0; 1243 unsigned long d_flags; 1244 1245 CRYPTO_DRIVER_LOCK(); 1246 for (hid = 0; hid < crypto_drivers_num; hid++) { 1247 const struct cryptocap *cap = &crypto_drivers[hid]; 1248 1249 if ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) && 1250 !crypto_devallowsoft) { 1251 continue; 1252 } 1253 for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++) 1254 if (cap->cc_kalg[kalg] & CRYPTO_ALG_FLAG_SUPPORTED) 1255 feat |= 1 << kalg; 1256 } 1257 CRYPTO_DRIVER_UNLOCK(); 1258 *featp = feat; 1259 return (0); 1260} 1261 1262/* 1263 * Crypto thread, dispatches crypto requests. 1264 */ 1265static int 1266crypto_proc(void *arg) 1267{ 1268 struct cryptop *crp, *submit; 1269 struct cryptkop *krp, *krpp; 1270 struct cryptocap *cap; 1271 u_int32_t hid; 1272 int result, hint; 1273 unsigned long q_flags; 1274 int loopcount = 0; 1275 1276 set_current_state(TASK_INTERRUPTIBLE); 1277 1278 CRYPTO_Q_LOCK(); 1279 for (;;) { 1280 /* 1281 * we need to make sure we don't get into a busy loop with nothing 1282 * to do, the two crypto_all_*blocked vars help us find out when 1283 * we are all full and can do nothing on any driver or Q. If so we 1284 * wait for an unblock. 1285 */ 1286 crypto_all_qblocked = !list_empty(&crp_q); 1287 1288 /* 1289 * Find the first element in the queue that can be 1290 * processed and look-ahead to see if multiple ops 1291 * are ready for the same driver. 1292 */ 1293 submit = NULL; 1294 hint = 0; 1295 list_for_each_entry(crp, &crp_q, crp_next) { 1296 hid = CRYPTO_SESID2HID(crp->crp_sid); 1297 cap = crypto_checkdriver(hid); 1298 /* 1299 * Driver cannot disappear when there is an active 1300 * session. 1301 */ 1302 KASSERT(cap != NULL, ("%s:%u Driver disappeared.", 1303 __func__, __LINE__)); 1304 if (cap == NULL || cap->cc_dev == NULL) { 1305 /* Op needs to be migrated, process it. */ 1306 if (submit == NULL) 1307 submit = crp; 1308 break; 1309 } 1310 if (!cap->cc_qblocked) { 1311 if (submit != NULL) { 1312 /* 1313 * We stop on finding another op, 1314 * regardless whether its for the same 1315 * driver or not. We could keep 1316 * searching the queue but it might be 1317 * better to just use a per-driver 1318 * queue instead. 1319 */ 1320 if (CRYPTO_SESID2HID(submit->crp_sid) == hid) 1321 hint = CRYPTO_HINT_MORE; 1322 break; 1323 } else { 1324 submit = crp; 1325 if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) 1326 break; 1327 /* keep scanning for more are q'd */ 1328 } 1329 } 1330 } 1331 if (submit != NULL) { 1332 hid = CRYPTO_SESID2HID(submit->crp_sid); 1333 crypto_all_qblocked = 0; 1334 list_del(&submit->crp_next); 1335 crypto_drivers[hid].cc_unqblocked = 1; 1336 cap = crypto_checkdriver(hid); 1337 CRYPTO_Q_UNLOCK(); 1338 KASSERT(cap != NULL, ("%s:%u Driver disappeared.", 1339 __func__, __LINE__)); 1340 result = crypto_invoke(cap, submit, hint); 1341 CRYPTO_Q_LOCK(); 1342 if (result == ERESTART) { 1343 /* 1344 * The driver ran out of resources, mark the 1345 * driver ``blocked'' for cryptop's and put 1346 * the request back in the queue. It would 1347 * best to put the request back where we got 1348 * it but that's hard so for now we put it 1349 * at the front. This should be ok; putting 1350 * it at the end does not work. 1351 */ 1352 /* XXX validate sid again? */ 1353 list_add(&submit->crp_next, &crp_q); 1354 cryptostats.cs_blocks++; 1355 if (crypto_drivers[hid].cc_unqblocked) 1356 crypto_drivers[hid].cc_qblocked=0; 1357 crypto_drivers[hid].cc_unqblocked=0; 1358 } 1359 crypto_drivers[hid].cc_unqblocked = 0; 1360 } 1361 1362 crypto_all_kqblocked = !list_empty(&crp_kq); 1363 1364 /* As above, but for key ops */ 1365 krp = NULL; 1366 list_for_each_entry(krpp, &crp_kq, krp_next) { 1367 cap = crypto_checkdriver(krpp->krp_hid); 1368 if (cap == NULL || cap->cc_dev == NULL) { 1369 /* 1370 * Operation needs to be migrated, invalidate 1371 * the assigned device so it will reselect a 1372 * new one below. Propagate the original 1373 * crid selection flags if supplied. 1374 */ 1375 krp->krp_hid = krp->krp_crid & 1376 (CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE); 1377 if (krp->krp_hid == 0) 1378 krp->krp_hid = 1379 CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE; 1380 break; 1381 } 1382 if (!cap->cc_kqblocked) { 1383 krp = krpp; 1384 break; 1385 } 1386 } 1387 if (krp != NULL) { 1388 crypto_all_kqblocked = 0; 1389 list_del(&krp->krp_next); 1390 crypto_drivers[krp->krp_hid].cc_kqblocked = 1; 1391 CRYPTO_Q_UNLOCK(); 1392 result = crypto_kinvoke(krp, krp->krp_hid); 1393 CRYPTO_Q_LOCK(); 1394 if (result == ERESTART) { 1395 /* 1396 * The driver ran out of resources, mark the 1397 * driver ``blocked'' for cryptkop's and put 1398 * the request back in the queue. It would 1399 * best to put the request back where we got 1400 * it but that's hard so for now we put it 1401 * at the front. This should be ok; putting 1402 * it at the end does not work. 1403 */ 1404 /* XXX validate sid again? */ 1405 list_add(&krp->krp_next, &crp_kq); 1406 cryptostats.cs_kblocks++; 1407 } else 1408 crypto_drivers[krp->krp_hid].cc_kqblocked = 0; 1409 } 1410 1411 if (submit == NULL && krp == NULL) { 1412 /* 1413 * Nothing more to be processed. Sleep until we're 1414 * woken because there are more ops to process. 1415 * This happens either by submission or by a driver 1416 * becoming unblocked and notifying us through 1417 * crypto_unblock. Note that when we wakeup we 1418 * start processing each queue again from the 1419 * front. It's not clear that it's important to 1420 * preserve this ordering since ops may finish 1421 * out of order if dispatched to different devices 1422 * and some become blocked while others do not. 1423 */ 1424 dprintk("%s - sleeping (qe=%d qb=%d kqe=%d kqb=%d)\n", 1425 __FUNCTION__, 1426 list_empty(&crp_q), crypto_all_qblocked, 1427 list_empty(&crp_kq), crypto_all_kqblocked); 1428 loopcount = 0; 1429 CRYPTO_Q_UNLOCK(); 1430 wait_event_interruptible(cryptoproc_wait, 1431 !(list_empty(&crp_q) || crypto_all_qblocked) || 1432 !(list_empty(&crp_kq) || crypto_all_kqblocked) || 1433 kthread_should_stop()); 1434 if (signal_pending (current)) { 1435#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) 1436 spin_lock_irq(¤t->sigmask_lock); 1437#endif 1438 flush_signals(current); 1439#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) 1440 spin_unlock_irq(¤t->sigmask_lock); 1441#endif 1442 } 1443 CRYPTO_Q_LOCK(); 1444 dprintk("%s - awake\n", __FUNCTION__); 1445 if (kthread_should_stop()) 1446 break; 1447 cryptostats.cs_intrs++; 1448 } else if (loopcount > crypto_max_loopcount) { 1449 /* 1450 * Give other processes a chance to run if we've 1451 * been using the CPU exclusively for a while. 1452 */ 1453 loopcount = 0; 1454 CRYPTO_Q_UNLOCK(); 1455 schedule(); 1456 CRYPTO_Q_LOCK(); 1457 } 1458 loopcount++; 1459 } 1460 CRYPTO_Q_UNLOCK(); 1461 return 0; 1462} 1463 1464/* 1465 * Crypto returns thread, does callbacks for processed crypto requests. 1466 * Callbacks are done here, rather than in the crypto drivers, because 1467 * callbacks typically are expensive and would slow interrupt handling. 1468 */ 1469static int 1470crypto_ret_proc(void *arg) 1471{ 1472 struct cryptop *crpt; 1473 struct cryptkop *krpt; 1474 unsigned long r_flags; 1475 1476 set_current_state(TASK_INTERRUPTIBLE); 1477 1478 CRYPTO_RETQ_LOCK(); 1479 for (;;) { 1480 /* Harvest return q's for completed ops */ 1481 crpt = NULL; 1482 if (!list_empty(&crp_ret_q)) 1483 crpt = list_entry(crp_ret_q.next, typeof(*crpt), crp_next); 1484 if (crpt != NULL) 1485 list_del(&crpt->crp_next); 1486 1487 krpt = NULL; 1488 if (!list_empty(&crp_ret_kq)) 1489 krpt = list_entry(crp_ret_kq.next, typeof(*krpt), krp_next); 1490 if (krpt != NULL) 1491 list_del(&krpt->krp_next); 1492 1493 if (crpt != NULL || krpt != NULL) { 1494 CRYPTO_RETQ_UNLOCK(); 1495 /* 1496 * Run callbacks unlocked. 1497 */ 1498 if (crpt != NULL) 1499 crpt->crp_callback(crpt); 1500 if (krpt != NULL) 1501 krpt->krp_callback(krpt); 1502 CRYPTO_RETQ_LOCK(); 1503 } else { 1504 /* 1505 * Nothing more to be processed. Sleep until we're 1506 * woken because there are more returns to process. 1507 */ 1508 dprintk("%s - sleeping\n", __FUNCTION__); 1509 CRYPTO_RETQ_UNLOCK(); 1510 wait_event_interruptible(cryptoretproc_wait, 1511 !list_empty(&crp_ret_q) || 1512 !list_empty(&crp_ret_kq) || 1513 kthread_should_stop()); 1514 if (signal_pending (current)) { 1515#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) 1516 spin_lock_irq(¤t->sigmask_lock); 1517#endif 1518 flush_signals(current); 1519#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) 1520 spin_unlock_irq(¤t->sigmask_lock); 1521#endif 1522 } 1523 CRYPTO_RETQ_LOCK(); 1524 dprintk("%s - awake\n", __FUNCTION__); 1525 if (kthread_should_stop()) { 1526 dprintk("%s - EXITING!\n", __FUNCTION__); 1527 break; 1528 } 1529 cryptostats.cs_rets++; 1530 } 1531 } 1532 CRYPTO_RETQ_UNLOCK(); 1533 return 0; 1534} 1535 1536 1537#if 0 /* should put this into /proc or something */ 1538static void 1539db_show_drivers(void) 1540{ 1541 int hid; 1542 1543 db_printf("%12s %4s %4s %8s %2s %2s\n" 1544 , "Device" 1545 , "Ses" 1546 , "Kops" 1547 , "Flags" 1548 , "QB" 1549 , "KB" 1550 ); 1551 for (hid = 0; hid < crypto_drivers_num; hid++) { 1552 const struct cryptocap *cap = &crypto_drivers[hid]; 1553 if (cap->cc_dev == NULL) 1554 continue; 1555 db_printf("%-12s %4u %4u %08x %2u %2u\n" 1556 , device_get_nameunit(cap->cc_dev) 1557 , cap->cc_sessions 1558 , cap->cc_koperations 1559 , cap->cc_flags 1560 , cap->cc_qblocked 1561 , cap->cc_kqblocked 1562 ); 1563 } 1564} 1565 1566DB_SHOW_COMMAND(crypto, db_show_crypto) 1567{ 1568 struct cryptop *crp; 1569 1570 db_show_drivers(); 1571 db_printf("\n"); 1572 1573 db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n", 1574 "HID", "Caps", "Ilen", "Olen", "Etype", "Flags", 1575 "Desc", "Callback"); 1576 TAILQ_FOREACH(crp, &crp_q, crp_next) { 1577 db_printf("%4u %08x %4u %4u %4u %04x %8p %8p\n" 1578 , (int) CRYPTO_SESID2HID(crp->crp_sid) 1579 , (int) CRYPTO_SESID2CAPS(crp->crp_sid) 1580 , crp->crp_ilen, crp->crp_olen 1581 , crp->crp_etype 1582 , crp->crp_flags 1583 , crp->crp_desc 1584 , crp->crp_callback 1585 ); 1586 } 1587 if (!TAILQ_EMPTY(&crp_ret_q)) { 1588 db_printf("\n%4s %4s %4s %8s\n", 1589 "HID", "Etype", "Flags", "Callback"); 1590 TAILQ_FOREACH(crp, &crp_ret_q, crp_next) { 1591 db_printf("%4u %4u %04x %8p\n" 1592 , (int) CRYPTO_SESID2HID(crp->crp_sid) 1593 , crp->crp_etype 1594 , crp->crp_flags 1595 , crp->crp_callback 1596 ); 1597 } 1598 } 1599} 1600 1601DB_SHOW_COMMAND(kcrypto, db_show_kcrypto) 1602{ 1603 struct cryptkop *krp; 1604 1605 db_show_drivers(); 1606 db_printf("\n"); 1607 1608 db_printf("%4s %5s %4s %4s %8s %4s %8s\n", 1609 "Op", "Status", "#IP", "#OP", "CRID", "HID", "Callback"); 1610 TAILQ_FOREACH(krp, &crp_kq, krp_next) { 1611 db_printf("%4u %5u %4u %4u %08x %4u %8p\n" 1612 , krp->krp_op 1613 , krp->krp_status 1614 , krp->krp_iparams, krp->krp_oparams 1615 , krp->krp_crid, krp->krp_hid 1616 , krp->krp_callback 1617 ); 1618 } 1619 if (!TAILQ_EMPTY(&crp_ret_q)) { 1620 db_printf("%4s %5s %8s %4s %8s\n", 1621 "Op", "Status", "CRID", "HID", "Callback"); 1622 TAILQ_FOREACH(krp, &crp_ret_kq, krp_next) { 1623 db_printf("%4u %5u %08x %4u %8p\n" 1624 , krp->krp_op 1625 , krp->krp_status 1626 , krp->krp_crid, krp->krp_hid 1627 , krp->krp_callback 1628 ); 1629 } 1630 } 1631} 1632#endif 1633 1634 1635static int 1636crypto_init(void) 1637{ 1638 int error; 1639 unsigned long cpu; 1640 1641 dprintk("%s(%p)\n", __FUNCTION__, (void *) crypto_init); 1642 1643 if (crypto_initted) 1644 return 0; 1645 crypto_initted = 1; 1646 1647 spin_lock_init(&crypto_drivers_lock); 1648 spin_lock_init(&crypto_q_lock); 1649 spin_lock_init(&crypto_ret_q_lock); 1650 1651 cryptop_zone = kmem_cache_create("cryptop", sizeof(struct cryptop), 1652 0, SLAB_HWCACHE_ALIGN, NULL 1653#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,23) 1654 , NULL 1655#endif 1656 ); 1657 1658 cryptodesc_zone = kmem_cache_create("cryptodesc", sizeof(struct cryptodesc), 1659 0, SLAB_HWCACHE_ALIGN, NULL 1660#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,23) 1661 , NULL 1662#endif 1663 ); 1664 1665 if (cryptodesc_zone == NULL || cryptop_zone == NULL) { 1666 printk("crypto: crypto_init cannot setup crypto zones\n"); 1667 error = ENOMEM; 1668 goto bad; 1669 } 1670 1671 crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; 1672 crypto_drivers = kmalloc(crypto_drivers_num * sizeof(struct cryptocap), 1673 GFP_KERNEL); 1674 if (crypto_drivers == NULL) { 1675 printk("crypto: crypto_init cannot setup crypto drivers\n"); 1676 error = ENOMEM; 1677 goto bad; 1678 } 1679 1680 memset(crypto_drivers, 0, crypto_drivers_num * sizeof(struct cryptocap)); 1681 1682 ocf_for_each_cpu(cpu) { 1683 cryptoproc[cpu] = kthread_create(crypto_proc, (void *) cpu, 1684 "ocf_%d", (int) cpu); 1685 if (IS_ERR(cryptoproc[cpu])) { 1686 error = PTR_ERR(cryptoproc[cpu]); 1687 printk("crypto: crypto_init cannot start crypto thread; error %d", 1688 error); 1689 goto bad; 1690 } 1691 kthread_bind(cryptoproc[cpu], cpu); 1692 wake_up_process(cryptoproc[cpu]); 1693 1694 cryptoretproc[cpu] = kthread_create(crypto_ret_proc, (void *) cpu, 1695 "ocf_ret_%d", (int) cpu); 1696 if (IS_ERR(cryptoretproc[cpu])) { 1697 error = PTR_ERR(cryptoretproc[cpu]); 1698 printk("crypto: crypto_init cannot start cryptoret thread; error %d", 1699 error); 1700 goto bad; 1701 } 1702 kthread_bind(cryptoretproc[cpu], cpu); 1703 wake_up_process(cryptoretproc[cpu]); 1704 } 1705 1706 return 0; 1707bad: 1708 crypto_exit(); 1709 return error; 1710} 1711 1712 1713static void 1714crypto_exit(void) 1715{ 1716 int cpu; 1717 1718 dprintk("%s()\n", __FUNCTION__); 1719 1720 /* 1721 * Terminate any crypto threads. 1722 */ 1723 ocf_for_each_cpu(cpu) { 1724 kthread_stop(cryptoproc[cpu]); 1725 kthread_stop(cryptoretproc[cpu]); 1726 } 1727 1728 /* 1729 * Reclaim dynamically allocated resources. 1730 */ 1731 if (crypto_drivers != NULL) 1732 kfree(crypto_drivers); 1733 1734 if (cryptodesc_zone != NULL) 1735 kmem_cache_destroy(cryptodesc_zone); 1736 if (cryptop_zone != NULL) 1737 kmem_cache_destroy(cryptop_zone); 1738} 1739 1740 1741EXPORT_SYMBOL(crypto_newsession); 1742EXPORT_SYMBOL(crypto_freesession); 1743EXPORT_SYMBOL(crypto_get_driverid); 1744EXPORT_SYMBOL(crypto_kregister); 1745EXPORT_SYMBOL(crypto_register); 1746EXPORT_SYMBOL(crypto_unregister); 1747EXPORT_SYMBOL(crypto_unregister_all); 1748EXPORT_SYMBOL(crypto_unblock); 1749EXPORT_SYMBOL(crypto_dispatch); 1750EXPORT_SYMBOL(crypto_kdispatch); 1751EXPORT_SYMBOL(crypto_freereq); 1752EXPORT_SYMBOL(crypto_getreq); 1753EXPORT_SYMBOL(crypto_done); 1754EXPORT_SYMBOL(crypto_kdone); 1755EXPORT_SYMBOL(crypto_getfeat); 1756EXPORT_SYMBOL(crypto_userasymcrypto); 1757EXPORT_SYMBOL(crypto_getcaps); 1758EXPORT_SYMBOL(crypto_find_driver); 1759EXPORT_SYMBOL(crypto_find_device_byhid); 1760 1761module_init(crypto_init); 1762module_exit(crypto_exit); 1763 1764MODULE_LICENSE("BSD"); 1765MODULE_AUTHOR("David McCullough <david_mccullough@mcafee.com>"); 1766MODULE_DESCRIPTION("OCF (OpenBSD Cryptographic Framework)"); 1767