crypto.c revision 111297
167754Smsmith/* $FreeBSD: head/sys/opencrypto/crypto.c 111297 2003-02-23 07:25:48Z sam $ */ 267754Smsmith/* $OpenBSD: crypto.c,v 1.38 2002/06/11 11:14:29 beck Exp $ */ 367754Smsmith/* 4167802Sjkim * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 567754Smsmith * 667754Smsmith * This code was written by Angelos D. Keromytis in Athens, Greece, in 767754Smsmith * February 2000. Network Security Technologies Inc. (NSTI) kindly 867754Smsmith * supported the development of this code. 967754Smsmith * 1067754Smsmith * Copyright (c) 2000, 2001 Angelos D. Keromytis 1167754Smsmith * 12167802Sjkim * Permission to use, copy, and modify this software with or without fee 1370243Smsmith * is hereby granted, provided that this entire notice is included in 1467754Smsmith * all source code copies of any software which is or includes a copy or 1567754Smsmith * modification of this software. 1667754Smsmith * 1767754Smsmith * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 1867754Smsmith * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 1967754Smsmith * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 2067754Smsmith * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 2167754Smsmith * PURPOSE. 2267754Smsmith */ 2367754Smsmith#define CRYPTO_TIMING /* enable timing support */ 2467754Smsmith 2567754Smsmith#include <sys/param.h> 2667754Smsmith#include <sys/systm.h> 2767754Smsmith#include <sys/eventhandler.h> 2867754Smsmith#include <sys/kernel.h> 2967754Smsmith#include <sys/kthread.h> 3067754Smsmith#include <sys/lock.h> 3167754Smsmith#include <sys/mutex.h> 3267754Smsmith#include <sys/malloc.h> 3367754Smsmith#include <sys/proc.h> 3467754Smsmith#include <sys/sysctl.h> 3567754Smsmith 3667754Smsmith#include <vm/uma.h> 3767754Smsmith#include <opencrypto/cryptodev.h> 3867754Smsmith#include <opencrypto/xform.h> /* XXX for M_XDATA */ 3967754Smsmith 4067754Smsmith#define SESID2HID(sid) (((sid) >> 32) & 0xffffffff) 4167754Smsmith 4267754Smsmith/* 4367754Smsmith * Crypto drivers register themselves by allocating a slot in the 4467754Smsmith * crypto_drivers table with crypto_get_driverid() and then registering 4567754Smsmith * each algorithm they support with crypto_register() and crypto_kregister(). 4667754Smsmith */ 4767754Smsmithstatic struct mtx crypto_drivers_mtx; /* lock on driver table */ 4867754Smsmith#define CRYPTO_DRIVER_LOCK() mtx_lock(&crypto_drivers_mtx) 4967754Smsmith#define CRYPTO_DRIVER_UNLOCK() mtx_unlock(&crypto_drivers_mtx) 5067754Smsmithstatic struct cryptocap *crypto_drivers = NULL; 5167754Smsmithstatic int crypto_drivers_num = 0; 5267754Smsmith 5367754Smsmith/* 5467754Smsmith * There are two queues for crypto requests; one for symmetric (e.g. 5567754Smsmith * cipher) operations and one for asymmetric (e.g. MOD)operations. 5667754Smsmith * A single mutex is used to lock access to both queues. We could 5767754Smsmith * have one per-queue but having one simplifies handling of block/unblock 5867754Smsmith * operations. 5967754Smsmith */ 6067754Smsmithstatic TAILQ_HEAD(,cryptop) crp_q; /* request queues */ 6167754Smsmithstatic TAILQ_HEAD(,cryptkop) crp_kq; 6267754Smsmithstatic struct mtx crypto_q_mtx; 6367754Smsmith#define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx) 6467754Smsmith#define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx) 6567754Smsmith 6667754Smsmith/* 6767754Smsmith * There are two queues for processing completed crypto requests; one 6867754Smsmith * for the symmetric and one for the asymmetric ops. We only need one 6967754Smsmith * but have two to avoid type futzing (cryptop vs. cryptkop). A single 7067754Smsmith * mutex is used to lock access to both queues. Note that this lock 7167754Smsmith * must be separate from the lock on request queues to insure driver 7267754Smsmith * callbacks don't generate lock order reversals. 7367754Smsmith */ 7467754Smsmithstatic TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */ 7567754Smsmithstatic TAILQ_HEAD(,cryptkop) crp_ret_kq; 7667754Smsmithstatic struct mtx crypto_ret_q_mtx; 7767754Smsmith#define CRYPTO_RETQ_LOCK() mtx_lock(&crypto_ret_q_mtx) 7867754Smsmith#define CRYPTO_RETQ_UNLOCK() mtx_unlock(&crypto_ret_q_mtx) 7967754Smsmith 8067754Smsmithstatic uma_zone_t cryptop_zone; 8167754Smsmithstatic uma_zone_t cryptodesc_zone; 8267754Smsmith 8367754Smsmithint crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ 8467754SmsmithSYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW, 8567754Smsmith &crypto_userasymcrypto, 0, 8667754Smsmith "Enable/disable user-mode access to asymmetric crypto support"); 8767754Smsmithint crypto_devallowsoft = 0; /* only use hardware crypto for asym */ 8867754SmsmithSYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW, 8967754Smsmith &crypto_devallowsoft, 0, 9067754Smsmith "Enable/disable use of software asym crypto support"); 9167754Smsmith 9267754SmsmithMALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); 9367754Smsmith 9467754Smsmithstatic void crypto_proc(void); 9567754Smsmithstatic struct proc *cryptoproc; 9667754Smsmithstatic void crypto_ret_proc(void); 9767754Smsmithstatic struct proc *cryptoretproc; 9867754Smsmithstatic void crypto_destroy(void); 9967754Smsmithstatic int crypto_invoke(struct cryptop *crp, int hint); 10067754Smsmithstatic int crypto_kinvoke(struct cryptkop *krp, int hint); 10167754Smsmith 10267754Smsmithstatic struct cryptostats cryptostats; 10367754SmsmithSYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats, 10467754Smsmith cryptostats, "Crypto system statistics"); 10567754Smsmith 10667754Smsmith#ifdef CRYPTO_TIMING 10767754Smsmithstatic int crypto_timing = 0; 10867754SmsmithSYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW, 10967754Smsmith &crypto_timing, 0, "Enable/disable crypto timing support"); 11067754Smsmith#endif 11167754Smsmith 11267754Smsmithstatic int 11367754Smsmithcrypto_init(void) 11467754Smsmith{ 11567754Smsmith int error; 11667754Smsmith 11767754Smsmith mtx_init(&crypto_drivers_mtx, "crypto driver table", 11867754Smsmith NULL, MTX_DEF|MTX_QUIET); 11967754Smsmith 120167802Sjkim TAILQ_INIT(&crp_q); 12167754Smsmith TAILQ_INIT(&crp_kq); 122167802Sjkim mtx_init(&crypto_q_mtx, "crypto op queues", NULL, MTX_DEF); 123167802Sjkim 124167802Sjkim TAILQ_INIT(&crp_ret_q); 125167802Sjkim TAILQ_INIT(&crp_ret_kq); 126167802Sjkim mtx_init(&crypto_ret_q_mtx, "crypto return queues", NULL, MTX_DEF); 127167802Sjkim 128167802Sjkim cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop), 129167802Sjkim 0, 0, 0, 0, 13099679Siwasaki UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 131167802Sjkim cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc), 13299679Siwasaki 0, 0, 0, 0, 13367754Smsmith UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 13499679Siwasaki if (cryptodesc_zone == NULL || cryptop_zone == NULL) { 135117521Snjl printf("crypto_init: cannot setup crypto zones\n"); 136117521Snjl error = ENOMEM; 137117521Snjl goto bad; 13899679Siwasaki } 139117521Snjl 140117521Snjl crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; 141117521Snjl crypto_drivers = malloc(crypto_drivers_num * 142117521Snjl sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO); 14399679Siwasaki if (crypto_drivers == NULL) { 14499679Siwasaki printf("crypto_init: cannot setup crypto drivers\n"); 14599679Siwasaki error = ENOMEM; 146114237Snjl goto bad; 147167802Sjkim } 148167802Sjkim 149167802Sjkim error = kthread_create((void (*)(void *)) crypto_proc, NULL, 150167802Sjkim &cryptoproc, 0, 0, "crypto"); 151167802Sjkim if (error) { 152167802Sjkim printf("crypto_init: cannot start crypto thread; error %d", 153167802Sjkim error); 154167802Sjkim goto bad; 155167802Sjkim } 156167802Sjkim 157167802Sjkim error = kthread_create((void (*)(void *)) crypto_ret_proc, NULL, 158167802Sjkim &cryptoretproc, 0, 0, "crypto returns"); 159167802Sjkim if (error) { 160167802Sjkim printf("crypto_init: cannot start cryptoret thread; error %d", 161167802Sjkim error); 162167802Sjkim goto bad; 163167802Sjkim } 164167802Sjkim return 0; 165167802Sjkimbad: 166167802Sjkim crypto_destroy(); 167167802Sjkim return error; 168167802Sjkim} 169167802Sjkim 170167802Sjkim/* 171167802Sjkim * Signal a crypto thread to terminate. We use the driver 172167802Sjkim * table lock to synchronize the sleep/wakeups so that we 173167802Sjkim * are sure the threads have terminated before we release 174167802Sjkim * the data structures they use. See crypto_finis below 175167802Sjkim * for the other half of this song-and-dance. 176167802Sjkim */ 177167802Sjkimstatic void 178167802Sjkimcrypto_terminate(struct proc **pp, void *q) 179167802Sjkim{ 180167802Sjkim struct proc *p; 181167802Sjkim 182167802Sjkim mtx_assert(&crypto_drivers_mtx, MA_OWNED); 183167802Sjkim p = *pp; 184167802Sjkim *pp = NULL; 185167802Sjkim if (p) { 186167802Sjkim wakeup_one(q); 187167802Sjkim PROC_LOCK(p); /* NB: insure we don't miss wakeup */ 188167802Sjkim CRYPTO_DRIVER_UNLOCK(); /* let crypto_finis progress */ 189167802Sjkim msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0); 190167802Sjkim PROC_UNLOCK(p); 191167802Sjkim CRYPTO_DRIVER_LOCK(); 192167802Sjkim } 193114237Snjl} 19499679Siwasaki 195167802Sjkimstatic void 196167802Sjkimcrypto_destroy(void) 197167802Sjkim{ 198167802Sjkim /* 199167802Sjkim * Terminate any crypto threads. 200114237Snjl */ 201167802Sjkim CRYPTO_DRIVER_LOCK(); 202167802Sjkim crypto_terminate(&cryptoproc, &crp_q); 203167802Sjkim crypto_terminate(&cryptoretproc, &crp_ret_q); 204167802Sjkim CRYPTO_DRIVER_UNLOCK(); 205167802Sjkim 206114237Snjl /* XXX flush queues??? */ 207167802Sjkim 208167802Sjkim /* 209167802Sjkim * Reclaim dynamically allocated resources. 210167802Sjkim */ 21167754Smsmith if (crypto_drivers != NULL) 212167802Sjkim free(crypto_drivers, M_CRYPTO_DATA); 213167802Sjkim 214167802Sjkim if (cryptodesc_zone != NULL) 21567754Smsmith uma_zdestroy(cryptodesc_zone); 21699679Siwasaki if (cryptop_zone != NULL) 217114237Snjl uma_zdestroy(cryptop_zone); 218167802Sjkim mtx_destroy(&crypto_q_mtx); 219114237Snjl mtx_destroy(&crypto_ret_q_mtx); 220167802Sjkim mtx_destroy(&crypto_drivers_mtx); 22167754Smsmith} 22267754Smsmith 223114237Snjl/* 22467754Smsmith * Initialization code, both for static and dynamic loading. 225167802Sjkim */ 226167802Sjkimstatic int 227114237Snjlcrypto_modevent(module_t mod, int type, void *unused) 22899679Siwasaki{ 22967754Smsmith int error = EINVAL; 23069450Smsmith 23167754Smsmith switch (type) { 23299679Siwasaki case MOD_LOAD: 23399679Siwasaki error = crypto_init(); 23467754Smsmith if (error == 0 && bootverbose) 235167802Sjkim printf("crypto: <crypto core>\n"); 236167802Sjkim break; 237151937Sjkim case MOD_UNLOAD: 238151937Sjkim /*XXX disallow if active sessions */ 239151937Sjkim error = 0; 240151937Sjkim crypto_destroy(); 241151937Sjkim return 0; 242151937Sjkim } 243167802Sjkim return error; 244151937Sjkim} 245151937Sjkim 246151937Sjkimstatic moduledata_t crypto_mod = { 247151937Sjkim "crypto", 248151937Sjkim crypto_modevent, 24967754Smsmith 0 250107325Siwasaki}; 251167802SjkimMODULE_VERSION(crypto, 1); 252167802SjkimDECLARE_MODULE(crypto, crypto_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST); 253167802Sjkim 254167802Sjkim/* 255167802Sjkim * Create a new session. 256114237Snjl */ 25799679Siwasakiint 258107325Siwasakicrypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard) 259167802Sjkim{ 260114237Snjl struct cryptoini *cr; 261167802Sjkim u_int32_t hid, lid; 26267754Smsmith int err = EINVAL; 26367754Smsmith 264114237Snjl CRYPTO_DRIVER_LOCK(); 26567754Smsmith 266167802Sjkim if (crypto_drivers == NULL) 267167802Sjkim goto done; 268114237Snjl 26999679Siwasaki /* 27067754Smsmith * The algorithm we use here is pretty stupid; just use the 271167802Sjkim * first driver that supports all the algorithms we need. 27267754Smsmith * 27399679Siwasaki * XXX We need more smarts here (in real life too, but that's 27499679Siwasaki * XXX another story altogether). 27599679Siwasaki */ 27699679Siwasaki 277167802Sjkim for (hid = 0; hid < crypto_drivers_num; hid++) { 278167802Sjkim /* 279167802Sjkim * If it's not initialized or has remaining sessions 28099679Siwasaki * referencing it, skip. 28167754Smsmith */ 28267754Smsmith if (crypto_drivers[hid].cc_newsession == NULL || 28367754Smsmith (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP)) 284167802Sjkim continue; 28567754Smsmith 286167802Sjkim /* Hardware required -- ignore software drivers. */ 28799146Siwasaki if (hard > 0 && 28899146Siwasaki (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE)) 289167802Sjkim continue; 290167802Sjkim /* Software required -- ignore hardware drivers. */ 291167802Sjkim if (hard < 0 && 292167802Sjkim (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) == 0) 293167802Sjkim continue; 294167802Sjkim 295167802Sjkim /* See if all the algorithms are supported. */ 296167802Sjkim for (cr = cri; cr; cr = cr->cri_next) 297167802Sjkim if (crypto_drivers[hid].cc_alg[cr->cri_alg] == 0) 29899146Siwasaki break; 29999146Siwasaki 300167802Sjkim if (cr == NULL) { 30199146Siwasaki /* Ok, all algorithms are supported. */ 302167802Sjkim 303167802Sjkim /* 304167802Sjkim * Can't do everything in one session. 30599146Siwasaki * 306167802Sjkim * XXX Fix this. We need to inject a "virtual" session layer right 30799146Siwasaki * XXX about here. 308167802Sjkim */ 309167802Sjkim 310167802Sjkim /* Call the driver initialization routine. */ 311100966Siwasaki lid = hid; /* Pass the driver ID. */ 312167802Sjkim err = crypto_drivers[hid].cc_newsession( 313100966Siwasaki crypto_drivers[hid].cc_arg, &lid, cri); 314167802Sjkim if (err == 0) { 315167802Sjkim (*sid) = hid; 316167802Sjkim (*sid) <<= 32; 317167802Sjkim (*sid) |= (lid & 0xffffffff); 318167802Sjkim crypto_drivers[hid].cc_sessions++; 319167802Sjkim } 320151937Sjkim break; 321151937Sjkim } 322151937Sjkim } 323100966Siwasakidone: 324167802Sjkim CRYPTO_DRIVER_UNLOCK(); 325167802Sjkim return err; 326167802Sjkim} 327167802Sjkim 328167802Sjkim/* 329167802Sjkim * Delete an existing session (or a reserved session on an unregistered 33099146Siwasaki * driver). 331167802Sjkim */ 332167802Sjkimint 33367754Smsmithcrypto_freesession(u_int64_t sid) 334167802Sjkim{ 335167802Sjkim u_int32_t hid; 336167802Sjkim int err; 337167802Sjkim 338167802Sjkim CRYPTO_DRIVER_LOCK(); 339167802Sjkim 340167802Sjkim if (crypto_drivers == NULL) { 341167802Sjkim err = EINVAL; 342167802Sjkim goto done; 343167802Sjkim } 344167802Sjkim 345167802Sjkim /* Determine two IDs. */ 346167802Sjkim hid = SESID2HID(sid); 347167802Sjkim 348167802Sjkim if (hid >= crypto_drivers_num) { 349167802Sjkim err = ENOENT; 350167802Sjkim goto done; 351167802Sjkim } 352167802Sjkim 353167802Sjkim if (crypto_drivers[hid].cc_sessions) 354167802Sjkim crypto_drivers[hid].cc_sessions--; 355167802Sjkim 356167802Sjkim /* Call the driver cleanup routine, if available. */ 357167802Sjkim if (crypto_drivers[hid].cc_freesession) 358167802Sjkim err = crypto_drivers[hid].cc_freesession( 359167802Sjkim crypto_drivers[hid].cc_arg, sid); 360167802Sjkim else 361167802Sjkim err = 0; 362167802Sjkim 363167802Sjkim /* 364167802Sjkim * If this was the last session of a driver marked as invalid, 36567754Smsmith * make the entry available for reuse. 36667754Smsmith */ 36767754Smsmith if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) && 36867754Smsmith crypto_drivers[hid].cc_sessions == 0) 36967754Smsmith bzero(&crypto_drivers[hid], sizeof(struct cryptocap)); 37067754Smsmith 37167754Smsmithdone: 37267754Smsmith CRYPTO_DRIVER_UNLOCK(); 37367754Smsmith return err; 37467754Smsmith} 37567754Smsmith 37667754Smsmith/* 37767754Smsmith * Return an unused driver id. Used by drivers prior to registering 37867754Smsmith * support for the algorithms they handle. 37967754Smsmith */ 38067754Smsmithint32_t 381167802Sjkimcrypto_get_driverid(u_int32_t flags) 38267754Smsmith{ 38367754Smsmith struct cryptocap *newdrv; 38480062Smsmith int i; 385114237Snjl 386167802Sjkim CRYPTO_DRIVER_LOCK(); 38767754Smsmith 388114237Snjl for (i = 0; i < crypto_drivers_num; i++) 38982367Smsmith if (crypto_drivers[i].cc_process == NULL && 390167802Sjkim (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0 && 391167802Sjkim crypto_drivers[i].cc_sessions == 0) 39267754Smsmith break; 39382367Smsmith 39482367Smsmith /* Out of entries, allocate some more. */ 395114237Snjl if (i == crypto_drivers_num) { 39684491Smsmith /* Be careful about wrap-around. */ 397167802Sjkim if (2 * crypto_drivers_num <= crypto_drivers_num) { 398167802Sjkim CRYPTO_DRIVER_UNLOCK(); 39982367Smsmith printf("crypto: driver count wraparound!\n"); 40084491Smsmith return -1; 40184491Smsmith } 402114237Snjl 40399679Siwasaki newdrv = malloc(2 * crypto_drivers_num * 404167802Sjkim sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO); 405167802Sjkim if (newdrv == NULL) { 40684491Smsmith CRYPTO_DRIVER_UNLOCK(); 40799679Siwasaki printf("crypto: no space to expand driver table!\n"); 40899679Siwasaki return -1; 40999679Siwasaki } 410167802Sjkim 411167802Sjkim bcopy(crypto_drivers, newdrv, 412167802Sjkim crypto_drivers_num * sizeof(struct cryptocap)); 413167802Sjkim 414167802Sjkim crypto_drivers_num *= 2; 415167802Sjkim 41667754Smsmith free(crypto_drivers, M_CRYPTO_DATA); 417167802Sjkim crypto_drivers = newdrv; 418167802Sjkim } 419167802Sjkim 42067754Smsmith /* NB: state is zero'd on free */ 42167754Smsmith crypto_drivers[i].cc_sessions = 1; /* Mark */ 42267754Smsmith crypto_drivers[i].cc_flags = flags; 42370243Smsmith if (bootverbose) 42467754Smsmith printf("crypto: assign driver %u, flags %u\n", i, flags); 425138287Smarks 42669450Smsmith CRYPTO_DRIVER_UNLOCK(); 427138287Smarks 42899679Siwasaki return i; 42984491Smsmith} 43067754Smsmith 43167754Smsmithstatic struct cryptocap * 432138287Smarkscrypto_checkdriver(u_int32_t hid) 433138287Smarks{ 434138287Smarks if (crypto_drivers == NULL) 435138287Smarks return NULL; 43667754Smsmith return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]); 43767754Smsmith} 43867754Smsmith 43967754Smsmith/* 440167802Sjkim * Register support for a key-related algorithm. This routine 44167754Smsmith * is called once for each algorithm supported a driver. 44277424Smsmith */ 44377424Smsmithint 44477424Smsmithcrypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags, 44577424Smsmith int (*kprocess)(void*, struct cryptkop *, int), 44667754Smsmith void *karg) 44767754Smsmith{ 44867754Smsmith struct cryptocap *cap; 44967754Smsmith int err; 45091116Smsmith 45191116Smsmith CRYPTO_DRIVER_LOCK(); 45291116Smsmith 45391116Smsmith cap = crypto_checkdriver(driverid); 45467754Smsmith if (cap != NULL && 45580062Smsmith (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) { 45680062Smsmith /* 45780062Smsmith * XXX Do some performance testing to determine placing. 458167802Sjkim * XXX We probably need an auxiliary data structure that 459167802Sjkim * XXX describes relative performances. 46067754Smsmith */ 46167754Smsmith 46278986Smsmith cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 46367754Smsmith if (bootverbose) 46478986Smsmith printf("crypto: driver %u registers key alg %u flags %u\n" 46578986Smsmith , driverid 46667754Smsmith , kalg 46767754Smsmith , flags 46867754Smsmith ); 46967754Smsmith 47067754Smsmith if (cap->cc_kprocess == NULL) { 47171867Smsmith cap->cc_karg = karg; 47267754Smsmith cap->cc_kprocess = kprocess; 47377424Smsmith } 47467754Smsmith err = 0; 47577424Smsmith } else 47677424Smsmith err = EINVAL; 47777424Smsmith 47877424Smsmith CRYPTO_DRIVER_UNLOCK(); 47977424Smsmith return err; 48077424Smsmith} 48167754Smsmith 48291116Smsmith/* 48391116Smsmith * Register support for a non-key-related algorithm. This routine 48491116Smsmith * is called once for each such algorithm supported by a driver. 48591116Smsmith */ 48691116Smsmithint 48791116Smsmithcrypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen, 48891116Smsmith u_int32_t flags, 48978986Smsmith int (*newses)(void*, u_int32_t*, struct cryptoini*), 49087031Smsmith int (*freeses)(void*, u_int64_t), 49187031Smsmith int (*process)(void*, struct cryptop *, int), 49287031Smsmith void *arg) 49387031Smsmith{ 49487031Smsmith struct cryptocap *cap; 49587031Smsmith int err; 49678986Smsmith 49778986Smsmith CRYPTO_DRIVER_LOCK(); 49878986Smsmith 49978986Smsmith cap = crypto_checkdriver(driverid); 50078986Smsmith /* NB: algorithms are in the range [1..max] */ 50178986Smsmith if (cap != NULL && 50278986Smsmith (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) { 50378986Smsmith /* 50478986Smsmith * XXX Do some performance testing to determine placing. 50584491Smsmith * XXX We probably need an auxiliary data structure that 50677424Smsmith * XXX describes relative performances. 50767754Smsmith */ 508107325Siwasaki 509107325Siwasaki cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 510107325Siwasaki cap->cc_max_op_len[alg] = maxoplen; 51191116Smsmith if (bootverbose) 51267754Smsmith printf("crypto: driver %u registers alg %u flags %u maxoplen %u\n" 513128212Snjl , driverid 514107325Siwasaki , alg 51567754Smsmith , flags 51667754Smsmith , maxoplen 51767754Smsmith ); 51887031Smsmith 51987031Smsmith if (cap->cc_process == NULL) { 52087031Smsmith cap->cc_arg = arg; 52187031Smsmith cap->cc_newsession = newses; 52287031Smsmith cap->cc_process = process; 52387031Smsmith cap->cc_freesession = freeses; 52487031Smsmith cap->cc_sessions = 0; /* Unmark */ 52587031Smsmith } 52687031Smsmith err = 0; 52787031Smsmith } else 52887031Smsmith err = EINVAL; 52987031Smsmith 53087031Smsmith CRYPTO_DRIVER_UNLOCK(); 53187031Smsmith return err; 53287031Smsmith} 53387031Smsmith 53487031Smsmith/* 53567754Smsmith * Unregister a crypto driver. If there are pending sessions using it, 536107325Siwasaki * leave enough information around so that subsequent calls using those 53767754Smsmith * sessions will correctly detect the driver has been unregistered and 53867754Smsmith * reroute requests. 539114237Snjl */ 540114237Snjlint 541114237Snjlcrypto_unregister(u_int32_t driverid, int alg) 542107325Siwasaki{ 543107325Siwasaki int i, err; 54467754Smsmith u_int32_t ses; 545107325Siwasaki struct cryptocap *cap; 546107325Siwasaki 547107325Siwasaki CRYPTO_DRIVER_LOCK(); 548107325Siwasaki 549107325Siwasaki cap = crypto_checkdriver(driverid); 550128212Snjl if (cap != NULL && 551128212Snjl (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) && 552128212Snjl cap->cc_alg[alg] != 0) { 553128212Snjl cap->cc_alg[alg] = 0; 554128212Snjl cap->cc_max_op_len[alg] = 0; 555128212Snjl 55667754Smsmith /* Was this the last algorithm ? */ 557128212Snjl for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++) 55867754Smsmith if (cap->cc_alg[i] != 0) 559107325Siwasaki break; 560107325Siwasaki 561107325Siwasaki if (i == CRYPTO_ALGORITHM_MAX + 1) { 562107325Siwasaki ses = cap->cc_sessions; 563128212Snjl bzero(cap, sizeof(struct cryptocap)); 564128212Snjl if (ses != 0) { 56567754Smsmith /* 566128212Snjl * If there are pending sessions, just mark as invalid. 56769450Smsmith */ 568107325Siwasaki cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 56967754Smsmith cap->cc_sessions = ses; 570128212Snjl } 57167754Smsmith } 57267754Smsmith err = 0; 57367754Smsmith } else 57487031Smsmith err = EINVAL; 57587031Smsmith 57687031Smsmith CRYPTO_DRIVER_UNLOCK(); 57787031Smsmith return err; 578107325Siwasaki} 57977424Smsmith 58077424Smsmith/* 581128212Snjl * Unregister all algorithms associated with a crypto driver. 58267754Smsmith * If there are pending sessions using it, leave enough information 58367754Smsmith * around so that subsequent calls using those sessions will 58467754Smsmith * correctly detect the driver has been unregistered and reroute 58567754Smsmith * requests. 58667754Smsmith */ 58767754Smsmithint 58891116Smsmithcrypto_unregister_all(u_int32_t driverid) 58991116Smsmith{ 59091116Smsmith int i, err; 59191116Smsmith u_int32_t ses; 59291116Smsmith struct cryptocap *cap; 59391116Smsmith 59491116Smsmith CRYPTO_DRIVER_LOCK(); 59567754Smsmith 59667754Smsmith cap = crypto_checkdriver(driverid); 597128212Snjl if (cap != NULL) { 59867754Smsmith for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) { 59967754Smsmith cap->cc_alg[i] = 0; 60067754Smsmith cap->cc_max_op_len[i] = 0; 60167754Smsmith } 60284491Smsmith ses = cap->cc_sessions; 60384491Smsmith bzero(cap, sizeof(struct cryptocap)); 60484491Smsmith if (ses != 0) { 60584491Smsmith /* 60684491Smsmith * If there are pending sessions, just mark as invalid. 60784491Smsmith */ 60884491Smsmith cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 60967754Smsmith cap->cc_sessions = ses; 61067754Smsmith } 61167754Smsmith err = 0; 61267754Smsmith } else 61369450Smsmith err = EINVAL; 61467754Smsmith 61584491Smsmith CRYPTO_DRIVER_UNLOCK(); 61684491Smsmith return err; 61767754Smsmith} 618128212Snjl 619128212Snjl/* 620128212Snjl * Clear blockage on a driver. The what parameter indicates whether 621128212Snjl * the driver is now ready for cryptop's and/or cryptokop's. 622128212Snjl */ 623128212Snjlint 62467754Smsmithcrypto_unblock(u_int32_t driverid, int what) 625129684Snjl{ 626129684Snjl struct cryptocap *cap; 627129684Snjl int needwakeup, err; 628129684Snjl 629128212Snjl CRYPTO_Q_LOCK(); 630128212Snjl cap = crypto_checkdriver(driverid); 631129684Snjl if (cap != NULL) { 632129684Snjl needwakeup = 0; 633129684Snjl if (what & CRYPTO_SYMQ) { 634129684Snjl needwakeup |= cap->cc_qblocked; 635129684Snjl cap->cc_qblocked = 0; 636129684Snjl } 637129684Snjl if (what & CRYPTO_ASYMQ) { 638129684Snjl needwakeup |= cap->cc_kqblocked; 639129684Snjl cap->cc_kqblocked = 0; 640167802Sjkim } 641128212Snjl if (needwakeup) 642129684Snjl wakeup_one(&crp_q); 643129684Snjl err = 0; 644129684Snjl } else 645128212Snjl err = EINVAL; 646129684Snjl CRYPTO_Q_UNLOCK(); 647129684Snjl 648129684Snjl return err; 649129684Snjl} 650128212Snjl 651129684Snjl/* 652129684Snjl * Add a crypto request to a queue, to be processed by the kernel thread. 653129684Snjl */ 654129684Snjlint 655128212Snjlcrypto_dispatch(struct cryptop *crp) 656129684Snjl{ 657129684Snjl u_int32_t hid = SESID2HID(crp->crp_sid); 658129684Snjl int result; 659129684Snjl 660129684Snjl cryptostats.cs_ops++; 661129684Snjl 662129684Snjl#ifdef CRYPTO_TIMING 663129684Snjl if (crypto_timing) 664129684Snjl binuptime(&crp->crp_tstamp); 665129684Snjl#endif 666128212Snjl 667128212Snjl if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) { 668128212Snjl struct cryptocap *cap; 669129684Snjl /* 670129684Snjl * Caller marked the request to be processed 671128212Snjl * immediately; dispatch it directly to the 672128212Snjl * driver unless the driver is currently blocked. 673128212Snjl */ 674128212Snjl cap = crypto_checkdriver(hid); 675128212Snjl if (cap && !cap->cc_qblocked) { 67667754Smsmith result = crypto_invoke(crp, 0); 67767754Smsmith if (result == ERESTART) { 678129684Snjl /* 679129684Snjl * The driver ran out of resources, mark the 680129684Snjl * driver ``blocked'' for cryptop's and put 681129684Snjl * the request on the queue. 68267754Smsmith */ 683107325Siwasaki CRYPTO_Q_LOCK(); 68467754Smsmith crypto_drivers[hid].cc_qblocked = 1; 68567754Smsmith TAILQ_INSERT_HEAD(&crp_q, crp, crp_next); 68667754Smsmith CRYPTO_Q_UNLOCK(); 68767754Smsmith cryptostats.cs_blocks++; 68877424Smsmith } 68967754Smsmith } else { 69077424Smsmith /* 69177424Smsmith * The driver is blocked, just queue the op until 69277424Smsmith * it unblocks and the kernel thread gets kicked. 69377424Smsmith */ 69477424Smsmith CRYPTO_Q_LOCK(); 69577424Smsmith TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 69677424Smsmith CRYPTO_Q_UNLOCK(); 69791116Smsmith result = 0; 698114237Snjl } 69967754Smsmith } else { 70067754Smsmith int wasempty; 70167754Smsmith /* 70299679Siwasaki * Caller marked the request as ``ok to delay''; 70399679Siwasaki * queue it for the dispatch thread. This is desirable 70499679Siwasaki * when the operation is low priority and/or suitable 70599679Siwasaki * for batching. 70699679Siwasaki */ 70799679Siwasaki CRYPTO_Q_LOCK(); 70899679Siwasaki wasempty = TAILQ_EMPTY(&crp_q); 70999679Siwasaki TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 71099679Siwasaki if (wasempty) 71199679Siwasaki wakeup_one(&crp_q); 712151937Sjkim CRYPTO_Q_UNLOCK(); 71399679Siwasaki result = 0; 714151937Sjkim } 715151937Sjkim 716151937Sjkim return result; 717151937Sjkim} 718151937Sjkim 719151937Sjkim/* 720151937Sjkim * Add an asymetric crypto request to a queue, 72199679Siwasaki * to be processed by the kernel thread. 722151937Sjkim */ 723151937Sjkimint 724151937Sjkimcrypto_kdispatch(struct cryptkop *krp) 725151937Sjkim{ 726151937Sjkim struct cryptocap *cap; 727151937Sjkim int result; 72899679Siwasaki 729151937Sjkim cryptostats.cs_kops++; 73099679Siwasaki 731151937Sjkim CRYPTO_Q_LOCK(); 73299679Siwasaki cap = crypto_checkdriver(krp->krp_hid); 73399679Siwasaki if (cap && !cap->cc_kqblocked) { 734107325Siwasaki result = crypto_kinvoke(krp, 0); 73599679Siwasaki if (result == ERESTART) { 73667754Smsmith /* 73767754Smsmith * The driver ran out of resources, mark the 738114237Snjl * driver ``blocked'' for cryptkop's and put 73967754Smsmith * the request back in the queue. It would 740167802Sjkim * best to put the request back where we got 74167754Smsmith * it but that's hard so for now we put it 74267754Smsmith * at the front. This should be ok; putting 743167802Sjkim * it at the end does not work. 744167802Sjkim */ 74571867Smsmith crypto_drivers[krp->krp_hid].cc_kqblocked = 1; 74667754Smsmith TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); 74767754Smsmith cryptostats.cs_kblocks++; 74867754Smsmith } 749167802Sjkim } else { 750167802Sjkim /* 751167802Sjkim * The driver is blocked, just queue the op until 75267754Smsmith * it unblocks and the kernel thread gets kicked. 75367754Smsmith */ 75467754Smsmith TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); 75567754Smsmith result = 0; 756167802Sjkim } 757167802Sjkim CRYPTO_Q_UNLOCK(); 758167802Sjkim 75967754Smsmith return result; 76067754Smsmith} 76167754Smsmith 76267754Smsmith/* 763167802Sjkim * Dispatch an assymetric crypto request to the appropriate crypto devices. 764167802Sjkim */ 765167802Sjkimstatic int 76667754Smsmithcrypto_kinvoke(struct cryptkop *krp, int hint) 76767754Smsmith{ 76867754Smsmith u_int32_t hid; 76967754Smsmith int error; 770167802Sjkim 771167802Sjkim mtx_assert(&crypto_q_mtx, MA_OWNED); 772167802Sjkim 77367754Smsmith /* Sanity checks. */ 77467754Smsmith if (krp == NULL) 77567754Smsmith return EINVAL; 77667754Smsmith if (krp->krp_callback == NULL) { 777167802Sjkim free(krp, M_XDATA); /* XXX allocated in cryptodev */ 778167802Sjkim return EINVAL; 779167802Sjkim } 780167802Sjkim 78167754Smsmith for (hid = 0; hid < crypto_drivers_num; hid++) { 78267754Smsmith if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) && 78367754Smsmith !crypto_devallowsoft) 78467754Smsmith continue; 785167802Sjkim if (crypto_drivers[hid].cc_kprocess == NULL) 786167802Sjkim continue; 787167802Sjkim if ((crypto_drivers[hid].cc_kalg[krp->krp_op] & 78867754Smsmith CRYPTO_ALG_FLAG_SUPPORTED) == 0) 78967754Smsmith continue; 79091116Smsmith break; 79167754Smsmith } 79267754Smsmith if (hid < crypto_drivers_num) { 79367754Smsmith krp->krp_hid = hid; 79467754Smsmith error = crypto_drivers[hid].cc_kprocess( 79567754Smsmith crypto_drivers[hid].cc_karg, krp, hint); 796114237Snjl } else 79767754Smsmith error = ENODEV; 798167802Sjkim 799167802Sjkim if (error) { 80067754Smsmith krp->krp_status = error; 80191116Smsmith crypto_kdone(krp); 80267754Smsmith } 80367754Smsmith return 0; 80467754Smsmith} 80567754Smsmith 80667754Smsmith#ifdef CRYPTO_TIMING 80791116Smsmithstatic void 80891116Smsmithcrypto_tstat(struct cryptotstat *ts, struct bintime *bt) 80991116Smsmith{ 81091116Smsmith struct bintime now, delta; 811114237Snjl struct timespec t; 81267754Smsmith uint64_t u; 813167802Sjkim 814167802Sjkim binuptime(&now); 81567754Smsmith u = now.frac; 81667754Smsmith delta.frac = now.frac - bt->frac; 81767754Smsmith delta.sec = now.sec - bt->sec; 81867754Smsmith if (u < delta.frac) 81967754Smsmith delta.sec--; 82067754Smsmith bintime2timespec(&delta, &t); 82167754Smsmith timespecadd(&ts->acc, &t); 82267754Smsmith if (timespeccmp(&t, &ts->min, <)) 82367754Smsmith ts->min = t; 82467754Smsmith if (timespeccmp(&t, &ts->max, >)) 82567754Smsmith ts->max = t; 82667754Smsmith ts->count++; 82767754Smsmith 82867754Smsmith *bt = now; 82967754Smsmith} 83091116Smsmith#endif 83191116Smsmith 83291116Smsmith/* 83391116Smsmith * Dispatch a crypto request to the appropriate crypto devices. 83467754Smsmith */ 83567754Smsmithstatic int 83667754Smsmithcrypto_invoke(struct cryptop *crp, int hint) 83767754Smsmith{ 83867754Smsmith u_int32_t hid; 839114237Snjl int (*process)(void*, struct cryptop *, int); 84067754Smsmith 841167802Sjkim#ifdef CRYPTO_TIMING 842167802Sjkim if (crypto_timing) 843167802Sjkim crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp); 844167802Sjkim#endif 845167802Sjkim /* Sanity checks. */ 846167802Sjkim if (crp == NULL) 847167802Sjkim return EINVAL; 84867754Smsmith if (crp->crp_callback == NULL) { 84967754Smsmith crypto_freereq(crp); 85067754Smsmith return EINVAL; 85167754Smsmith } 85267754Smsmith if (crp->crp_desc == NULL) { 853117521Snjl crp->crp_etype = EINVAL; 854117521Snjl crypto_done(crp); 855117521Snjl return 0; 856138287Smarks } 857167802Sjkim 858117521Snjl hid = SESID2HID(crp->crp_sid); 859117521Snjl if (hid < crypto_drivers_num) { 860138287Smarks if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) 861167802Sjkim crypto_freesession(crp->crp_sid); 862117521Snjl process = crypto_drivers[hid].cc_process; 863117521Snjl } else { 86467754Smsmith process = NULL; 86567754Smsmith } 86667754Smsmith 86777424Smsmith if (process == NULL) { 868167802Sjkim struct cryptodesc *crd; 86967754Smsmith u_int64_t nid; 87067754Smsmith 87177424Smsmith /* 872167802Sjkim * Driver has unregistered; migrate the session and return 873167802Sjkim * an error to the caller so they'll resubmit the op. 874167802Sjkim */ 87567754Smsmith for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next) 87687031Smsmith crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI); 87787031Smsmith 878167802Sjkim if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 0) == 0) 879167802Sjkim crp->crp_sid = nid; 880167802Sjkim 88167754Smsmith crp->crp_etype = EAGAIN; 88299679Siwasaki crypto_done(crp); 88399679Siwasaki return 0; 884167802Sjkim } else { 885167802Sjkim /* 88687031Smsmith * Invoke the driver to process the request. 88799679Siwasaki */ 88899679Siwasaki return (*process)(crypto_drivers[hid].cc_arg, crp, hint); 889138287Smarks } 890138287Smarks} 891167802Sjkim 892167802Sjkim/* 893167802Sjkim * Release a set of crypto descriptors. 894167802Sjkim */ 895167802Sjkimvoid 89699679Siwasakicrypto_freereq(struct cryptop *crp) 897138287Smarks{ 898127175Snjl struct cryptodesc *crd; 89977424Smsmith 90067754Smsmith if (crp == NULL) 90177424Smsmith return; 902167802Sjkim 903167802Sjkim while ((crd = crp->crp_desc) != NULL) { 904167802Sjkim crp->crp_desc = crd->crd_next; 905167802Sjkim uma_zfree(cryptodesc_zone, crd); 906167802Sjkim } 907167802Sjkim 90867754Smsmith uma_zfree(cryptop_zone, crp); 909167802Sjkim} 91067754Smsmith 91167754Smsmith/* 91267754Smsmith * Acquire a set of crypto descriptors. 91377424Smsmith */ 914167802Sjkimstruct cryptop * 915167802Sjkimcrypto_getreq(int num) 916167802Sjkim{ 917167802Sjkim struct cryptodesc *crd; 91867754Smsmith struct cryptop *crp; 91967754Smsmith 92067754Smsmith crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO); 92167754Smsmith if (crp != NULL) { 92267754Smsmith while (num--) { 92377424Smsmith crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO); 924167802Sjkim if (crd == NULL) { 925167802Sjkim crypto_freereq(crp); 926167802Sjkim return NULL; 927167802Sjkim } 92867754Smsmith 92967754Smsmith crd->crd_next = crp->crp_desc; 93067754Smsmith crp->crp_desc = crd; 93167754Smsmith } 93291116Smsmith } 93391116Smsmith return crp; 93467754Smsmith} 93567754Smsmith 936117521Snjl/* 93767754Smsmith * Invoke the callback on behalf of the driver. 938117521Snjl */ 939117521Snjlvoid 940167802Sjkimcrypto_done(struct cryptop *crp) 94167754Smsmith{ 942117521Snjl if (crp->crp_etype != 0) 94367754Smsmith cryptostats.cs_errs++; 944117521Snjl#ifdef CRYPTO_TIMING 945117521Snjl if (crypto_timing) 946117521Snjl crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp); 947117521Snjl#endif 948167802Sjkim if (crp->crp_flags & CRYPTO_F_CBIMM) { 949117521Snjl /* 950117521Snjl * Do the callback directly. This is ok when the 951117521Snjl * callback routine does very little (e.g. the 952117521Snjl * /dev/crypto callback method just does a wakeup). 953117521Snjl */ 954167802Sjkim#ifdef CRYPTO_TIMING 955167802Sjkim if (crypto_timing) { 956167802Sjkim /* 957117521Snjl * NB: We must copy the timestamp before 958117521Snjl * doing the callback as the cryptop is 959117521Snjl * likely to be reclaimed. 960117521Snjl */ 961117521Snjl struct bintime t = crp->crp_tstamp; 962117521Snjl crypto_tstat(&cryptostats.cs_cb, &t); 963117521Snjl crp->crp_callback(crp); 964117521Snjl crypto_tstat(&cryptostats.cs_finis, &t); 965117521Snjl } else 966117521Snjl#endif 967117521Snjl crp->crp_callback(crp); 968129684Snjl } else { 969117521Snjl int wasempty; 970167802Sjkim /* 971117521Snjl * Normal case; queue the callback for the thread. 972167802Sjkim */ 973167802Sjkim CRYPTO_RETQ_LOCK(); 974167802Sjkim wasempty = TAILQ_EMPTY(&crp_ret_q); 975167802Sjkim TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); 976167802Sjkim 977167802Sjkim if (wasempty) 978167802Sjkim wakeup_one(&crp_ret_q); /* shared wait channel */ 979167802Sjkim CRYPTO_RETQ_UNLOCK(); 98067754Smsmith } 981167802Sjkim} 982167802Sjkim 98367754Smsmith/* 98467754Smsmith * Invoke the callback on behalf of the driver. 985114237Snjl */ 98667754Smsmithvoid 98767754Smsmithcrypto_kdone(struct cryptkop *krp) 988114237Snjl{ 98967754Smsmith int wasempty; 99067754Smsmith 99167754Smsmith if (krp->krp_status != 0) 992117521Snjl cryptostats.cs_kerrs++; 993117521Snjl CRYPTO_RETQ_LOCK(); 994114237Snjl wasempty = TAILQ_EMPTY(&crp_ret_kq); 99567754Smsmith TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next); 99667754Smsmith 99767754Smsmith if (wasempty) 998167802Sjkim wakeup_one(&crp_ret_q); /* shared wait channel */ 999167802Sjkim CRYPTO_RETQ_UNLOCK(); 1000167802Sjkim} 1001167802Sjkim 1002167802Sjkimint 1003167802Sjkimcrypto_getfeat(int *featp) 1004167802Sjkim{ 1005114237Snjl int hid, kalg, feat = 0; 100667754Smsmith 100767754Smsmith if (!crypto_userasymcrypto) 100867754Smsmith goto out; 100967754Smsmith 101067754Smsmith CRYPTO_DRIVER_LOCK(); 1011114237Snjl for (hid = 0; hid < crypto_drivers_num; hid++) { 101267754Smsmith if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) && 1013167802Sjkim !crypto_devallowsoft) { 1014167802Sjkim continue; 1015167802Sjkim } 1016167802Sjkim if (crypto_drivers[hid].cc_kprocess == NULL) 1017114237Snjl continue; 101880062Smsmith for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++) 101967754Smsmith if ((crypto_drivers[hid].cc_kalg[kalg] & 102067754Smsmith CRYPTO_ALG_FLAG_SUPPORTED) != 0) 1021114237Snjl feat |= 1 << kalg; 102267754Smsmith } 1023167802Sjkim CRYPTO_DRIVER_UNLOCK(); 1024167802Sjkimout: 1025167802Sjkim *featp = feat; 1026167802Sjkim return (0); 1027167802Sjkim} 1028114237Snjl 102977424Smsmith/* 103067754Smsmith * Terminate a thread at module unload. The process that 103167754Smsmith * initiated this is waiting for us to signal that we're gone; 103267754Smsmith * wake it up and exit. We use the driver table lock to insure 103367754Smsmith * we don't do the wakeup before they're waiting. There is no 103467754Smsmith * race here because the waiter sleeps on the proc lock for the 1035151937Sjkim * thread so it gets notified at the right time because of an 1036151937Sjkim * extra wakeup that's done in exit1(). 103767754Smsmith */ 103867754Smsmithstatic void 103967754Smsmithcrypto_finis(void *chan) 104067754Smsmith{ 104191116Smsmith CRYPTO_DRIVER_LOCK(); 104291116Smsmith wakeup_one(chan); 104367754Smsmith CRYPTO_DRIVER_UNLOCK(); 104491116Smsmith mtx_lock(&Giant); 104591116Smsmith kthread_exit(0); 104691116Smsmith} 104791116Smsmith 104867754Smsmith/* 104967754Smsmith * Crypto thread, dispatches crypto requests. 105067754Smsmith */ 105185756Smsmithstatic void 105267754Smsmithcrypto_proc(void) 105367754Smsmith{ 105491116Smsmith struct cryptop *crp, *submit; 105591116Smsmith struct cryptkop *krp; 105691116Smsmith struct cryptocap *cap; 105767754Smsmith int result, hint; 1058167802Sjkim 1059114237Snjl CRYPTO_Q_LOCK(); 1060167802Sjkim for (;;) { 1061167802Sjkim /* 106267754Smsmith * Find the first element in the queue that can be 106367754Smsmith * processed and look-ahead to see if multiple ops 106467754Smsmith * are ready for the same driver. 106591116Smsmith */ 106691116Smsmith submit = NULL; 106767754Smsmith hint = 0; 106867754Smsmith TAILQ_FOREACH(crp, &crp_q, crp_next) { 106967754Smsmith u_int32_t hid = SESID2HID(crp->crp_sid); 107067754Smsmith cap = crypto_checkdriver(hid); 1071151937Sjkim if (cap == NULL || cap->cc_process == NULL) { 1072151937Sjkim /* Op needs to be migrated, process it. */ 107367754Smsmith if (submit == NULL) 107491116Smsmith submit = crp; 107591116Smsmith break; 107667754Smsmith } 107791116Smsmith if (!cap->cc_qblocked) { 107891116Smsmith if (submit != NULL) { 107967754Smsmith /* 108067754Smsmith * We stop on finding another op, 108167754Smsmith * regardless whether its for the same 108267754Smsmith * driver or not. We could keep 108391116Smsmith * searching the queue but it might be 108491116Smsmith * better to just use a per-driver 108591116Smsmith * queue instead. 108691116Smsmith */ 108767754Smsmith if (SESID2HID(submit->crp_sid) == hid) 108891116Smsmith hint = CRYPTO_HINT_MORE; 108991116Smsmith break; 109067754Smsmith } else { 109191116Smsmith submit = crp; 109291116Smsmith if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) 109391116Smsmith break; 109467754Smsmith /* keep scanning for more are q'd */ 109567754Smsmith } 109667754Smsmith } 109767754Smsmith } 109891116Smsmith if (submit != NULL) { 109991116Smsmith TAILQ_REMOVE(&crp_q, submit, crp_next); 110091116Smsmith result = crypto_invoke(submit, hint); 110167754Smsmith if (result == ERESTART) { 110267754Smsmith /* 110367754Smsmith * The driver ran out of resources, mark the 110467754Smsmith * driver ``blocked'' for cryptop's and put 110591116Smsmith * the request back in the queue. It would 110691116Smsmith * best to put the request back where we got 110791116Smsmith * it but that's hard so for now we put it 110867754Smsmith * at the front. This should be ok; putting 110991116Smsmith * it at the end does not work. 111091116Smsmith */ 111167754Smsmith /* XXX validate sid again? */ 111291116Smsmith crypto_drivers[SESID2HID(submit->crp_sid)].cc_qblocked = 1; 111391116Smsmith TAILQ_INSERT_HEAD(&crp_q, submit, crp_next); 111467754Smsmith cryptostats.cs_blocks++; 111591116Smsmith } 111691116Smsmith } 111767754Smsmith 111867754Smsmith /* As above, but for key ops */ 111967754Smsmith TAILQ_FOREACH(krp, &crp_kq, krp_next) { 1120151937Sjkim cap = crypto_checkdriver(krp->krp_hid); 1121151937Sjkim if (cap == NULL || cap->cc_kprocess == NULL) { 1122151937Sjkim /* Op needs to be migrated, process it. */ 1123151937Sjkim break; 1124151937Sjkim } 1125151937Sjkim if (!cap->cc_kqblocked) 1126167802Sjkim break; 1127167802Sjkim } 1128167802Sjkim if (krp != NULL) { 1129167802Sjkim TAILQ_REMOVE(&crp_kq, krp, krp_next); 1130167802Sjkim result = crypto_kinvoke(krp, 0); 1131167802Sjkim if (result == ERESTART) { 1132167802Sjkim /* 1133167802Sjkim * The driver ran out of resources, mark the 1134167802Sjkim * driver ``blocked'' for cryptkop's and put 1135167802Sjkim * the request back in the queue. It would 1136167802Sjkim * best to put the request back where we got 1137167802Sjkim * it but that's hard so for now we put it 1138167802Sjkim * at the front. This should be ok; putting 1139167802Sjkim * it at the end does not work. 1140151937Sjkim */ 114167754Smsmith /* XXX validate sid again? */ 114267754Smsmith crypto_drivers[krp->krp_hid].cc_kqblocked = 1; 1143114237Snjl TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); 114467754Smsmith cryptostats.cs_kblocks++; 1145167802Sjkim } 1146167802Sjkim } 1147167802Sjkim 1148167802Sjkim if (submit == NULL && krp == NULL) { 1149167802Sjkim /* 115067754Smsmith * Nothing more to be processed. Sleep until we're 115177424Smsmith * woken because there are more ops to process. 115267754Smsmith * This happens either by submission or by a driver 1153151937Sjkim * becoming unblocked and notifying us through 1154114237Snjl * crypto_unblock. Note that when we wakeup we 115567754Smsmith * start processing each queue again from the 1156167802Sjkim * front. It's not clear that it's important to 1157167802Sjkim * preserve this ordering since ops may finish 1158167802Sjkim * out of order if dispatched to different devices 1159167802Sjkim * and some become blocked while others do not. 1160167802Sjkim */ 116167754Smsmith msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0); 116277424Smsmith if (cryptoproc == NULL) 116367754Smsmith break; 1164151937Sjkim cryptostats.cs_intrs++; 1165151937Sjkim } 116667754Smsmith } 1167167802Sjkim CRYPTO_Q_UNLOCK(); 1168167802Sjkim 116967754Smsmith crypto_finis(&crp_q); 1170151937Sjkim} 117167754Smsmith 1172151937Sjkim/* 117367754Smsmith * Crypto returns thread, does callbacks for processed crypto requests. 117467754Smsmith * Callbacks are done here, rather than in the crypto drivers, because 1175107325Siwasaki * callbacks typically are expensive and would slow interrupt handling. 117667754Smsmith */ 117767754Smsmithstatic void 1178151937Sjkimcrypto_ret_proc(void) 1179114237Snjl{ 118067754Smsmith struct cryptop *crpt; 1181167802Sjkim struct cryptkop *krpt; 1182167802Sjkim 1183167802Sjkim CRYPTO_RETQ_LOCK(); 1184167802Sjkim for (;;) { 1185167802Sjkim /* Harvest return q's for completed ops */ 118667754Smsmith crpt = TAILQ_FIRST(&crp_ret_q); 118777424Smsmith if (crpt != NULL) 118867754Smsmith TAILQ_REMOVE(&crp_ret_q, crpt, crp_next); 1189114237Snjl 119067754Smsmith krpt = TAILQ_FIRST(&crp_ret_kq); 1191167802Sjkim if (krpt != NULL) 1192167802Sjkim TAILQ_REMOVE(&crp_ret_kq, krpt, krp_next); 119367754Smsmith 119477424Smsmith if (crpt != NULL || krpt != NULL) { 119567754Smsmith CRYPTO_RETQ_UNLOCK(); 1196114237Snjl /* 119767754Smsmith * Run callbacks unlocked. 1198167802Sjkim */ 1199167802Sjkim if (crpt != NULL) { 120067754Smsmith#ifdef CRYPTO_TIMING 120177424Smsmith if (crypto_timing) { 120267754Smsmith /* 1203167802Sjkim * NB: We must copy the timestamp before 1204167802Sjkim * doing the callback as the cryptop is 1205167802Sjkim * likely to be reclaimed. 1206167802Sjkim */ 1207167802Sjkim struct bintime t = crpt->crp_tstamp; 1208167802Sjkim crypto_tstat(&cryptostats.cs_cb, &t); 1209167802Sjkim crpt->crp_callback(crpt); 1210167802Sjkim crypto_tstat(&cryptostats.cs_finis, &t); 1211167802Sjkim } else 1212167802Sjkim#endif 1213167802Sjkim crpt->crp_callback(crpt); 1214114237Snjl } 121567754Smsmith if (krpt != NULL) 1216167802Sjkim krpt->krp_callback(krpt); 121791116Smsmith CRYPTO_RETQ_LOCK(); 121891116Smsmith } else { 121991116Smsmith /* 1220151937Sjkim * Nothing more to be processed. Sleep until we're 122191116Smsmith * woken because there are more returns to process. 1222167802Sjkim */ 1223167802Sjkim msleep(&crp_ret_q, &crypto_ret_q_mtx, PWAIT, 1224167802Sjkim "crypto_ret_wait", 0); 1225167802Sjkim if (cryptoretproc == NULL) 1226167802Sjkim break; 122767754Smsmith cryptostats.cs_rets++; 1228151937Sjkim } 122967754Smsmith } 1230151937Sjkim CRYPTO_RETQ_UNLOCK(); 123167754Smsmith 1232167802Sjkim crypto_finis(&crp_ret_q); 1233167802Sjkim} 1234167802Sjkim