1/*- 2 * Copyright (c) 2003-2009 RMI Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. Neither the name of RMI Corporation, nor the names of its contributors, 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * RMI_BSD 30 */ 31 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD$"); 34 35#include <sys/param.h> 36#include <sys/systm.h> 37#include <sys/proc.h> 38#include <sys/errno.h> 39#include <sys/malloc.h> 40#include <sys/kernel.h> 41#include <sys/module.h> 42#include <sys/mbuf.h> 43#include <sys/lock.h> 44#include <sys/mutex.h> 45#include <sys/sysctl.h> 46#include <sys/bus.h> 47#include <sys/random.h> 48#include <sys/rman.h> 49#include <sys/uio.h> 50#include <sys/kobj.h> 51#include <opencrypto/cryptodev.h> 52 53#include "cryptodev_if.h" 54 55#include <vm/vm.h> 56#include <vm/pmap.h> 57 58#include <mips/rmi/dev/sec/rmilib.h> 59 60/* #define RMI_SEC_DEBUG */ 61 62void xlr_sec_print_data(struct cryptop *crp); 63 64static int xlr_sec_newsession(device_t dev, uint32_t * sidp, struct cryptoini *cri); 65static int xlr_sec_freesession(device_t dev, uint64_t tid); 66static int xlr_sec_process(device_t dev, struct cryptop *crp, int hint); 67 68static int xlr_sec_probe(device_t); 69static int xlr_sec_attach(device_t); 70static int xlr_sec_detach(device_t); 71 72 73static device_method_t xlr_sec_methods[] = { 74 /* device interface */ 75 DEVMETHOD(device_probe, xlr_sec_probe), 76 DEVMETHOD(device_attach, xlr_sec_attach), 77 DEVMETHOD(device_detach, xlr_sec_detach), 78 79 /* crypto device methods */ 80 DEVMETHOD(cryptodev_newsession, xlr_sec_newsession), 81 DEVMETHOD(cryptodev_freesession,xlr_sec_freesession), 82 DEVMETHOD(cryptodev_process, xlr_sec_process), 83 84 DEVMETHOD_END 85}; 86 87static driver_t xlr_sec_driver = { 88 "rmisec", 89 xlr_sec_methods, 90 sizeof(struct xlr_sec_softc) 91}; 92static devclass_t xlr_sec_devclass; 93 94DRIVER_MODULE(rmisec, iodi, xlr_sec_driver, xlr_sec_devclass, 0, 0); 95MODULE_DEPEND(rmisec, crypto, 1, 1, 1); 96 97static int 98xlr_sec_probe(device_t dev) 99{ 100 101 device_set_desc(dev, "XLR Security Accelerator"); 102 return (BUS_PROBE_DEFAULT); 103} 104 105/* 106 * Attach an interface that successfully probed. 107 */ 108static int 109xlr_sec_attach(device_t dev) 110{ 111 struct xlr_sec_softc *sc = device_get_softc(dev); 112 113 sc->sc_dev = dev; 114 mtx_init(&sc->sc_mtx, device_get_nameunit(dev), "rmi crypto driver", 115 MTX_DEF); 116 sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); 117 if (sc->sc_cid < 0) { 118 printf("xlr_sec - error : could not get the driver id\n"); 119 goto error_exit; 120 } 121 if (crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0) != 0) 122 printf("register failed for CRYPTO_DES_CBC\n"); 123 124 if (crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0) != 0) 125 printf("register failed for CRYPTO_3DES_CBC\n"); 126 127 if (crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0) != 0) 128 printf("register failed for CRYPTO_AES_CBC\n"); 129 130 if (crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0) != 0) 131 printf("register failed for CRYPTO_ARC4\n"); 132 133 if (crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0) != 0) 134 printf("register failed for CRYPTO_MD5\n"); 135 136 if (crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0) != 0) 137 printf("register failed for CRYPTO_SHA1\n"); 138 139 if (crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0) != 0) 140 printf("register failed for CRYPTO_MD5_HMAC\n"); 141 142 if (crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0) != 0) 143 printf("register failed for CRYPTO_SHA1_HMAC\n"); 144 145 xlr_sec_init(sc); 146 device_printf(dev, "Initialization complete!\n"); 147 return (0); 148 149error_exit: 150 return (ENXIO); 151 152} 153 154/* 155 * Detach an interface that successfully probed. 156 */ 157static int 158xlr_sec_detach(device_t dev) 159{ 160 int sesn; 161 struct xlr_sec_softc *sc = device_get_softc(dev); 162 struct xlr_sec_session *ses = NULL; 163 symkey_desc_pt desc; 164 165 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 166 ses = &sc->sc_sessions[sesn]; 167 desc = (symkey_desc_pt) ses->desc_ptr; 168 free(desc->user.kern_src, M_DEVBUF); 169 free(desc->user.kern_dest, M_DEVBUF); 170 free(desc->next_src_buf, M_DEVBUF); 171 free(desc->next_dest_buf, M_DEVBUF); 172 free(ses->desc_ptr, M_DEVBUF); 173 } 174 175 return (0); 176} 177 178/* 179 * Allocate a new 'session' and return an encoded session id. 'sidp' 180 * contains our registration id, and should contain an encoded session 181 * id on successful allocation. 182 */ 183static int 184xlr_sec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri) 185{ 186 struct cryptoini *c; 187 struct xlr_sec_softc *sc = device_get_softc(dev); 188 int mac = 0, cry = 0, sesn; 189 struct xlr_sec_session *ses = NULL; 190 191 if (sidp == NULL || cri == NULL || sc == NULL) 192 return (EINVAL); 193 194 if (sc->sc_sessions == NULL) { 195 ses = sc->sc_sessions = (struct xlr_sec_session *)malloc( 196 sizeof(struct xlr_sec_session), M_DEVBUF, M_NOWAIT); 197 if (ses == NULL) 198 return (ENOMEM); 199 sesn = 0; 200 sc->sc_nsessions = 1; 201 } else { 202 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 203 if (!sc->sc_sessions[sesn].hs_used) { 204 ses = &sc->sc_sessions[sesn]; 205 break; 206 } 207 } 208 209 if (ses == NULL) { 210 sesn = sc->sc_nsessions; 211 ses = (struct xlr_sec_session *)malloc((sesn + 1) * 212 sizeof(struct xlr_sec_session), M_DEVBUF, M_NOWAIT); 213 if (ses == NULL) 214 return (ENOMEM); 215 bcopy(sc->sc_sessions, ses, sesn * sizeof(*ses)); 216 bzero(sc->sc_sessions, sesn * sizeof(*ses)); 217 free(sc->sc_sessions, M_DEVBUF); 218 sc->sc_sessions = ses; 219 ses = &sc->sc_sessions[sesn]; 220 sc->sc_nsessions++; 221 } 222 } 223 bzero(ses, sizeof(*ses)); 224 ses->sessionid = sesn; 225 ses->desc_ptr = xlr_sec_allocate_desc(ses); 226 if (ses->desc_ptr == NULL) 227 return (ENOMEM); 228 ses->hs_used = 1; 229 230 for (c = cri; c != NULL; c = c->cri_next) { 231 switch (c->cri_alg) { 232 case CRYPTO_MD5: 233 case CRYPTO_SHA1: 234 case CRYPTO_MD5_HMAC: 235 case CRYPTO_SHA1_HMAC: 236 if (mac) 237 return (EINVAL); 238 mac = 1; 239 ses->hs_mlen = c->cri_mlen; 240 if (ses->hs_mlen == 0) { 241 switch (c->cri_alg) { 242 case CRYPTO_MD5: 243 case CRYPTO_MD5_HMAC: 244 ses->hs_mlen = 16; 245 break; 246 case CRYPTO_SHA1: 247 case CRYPTO_SHA1_HMAC: 248 ses->hs_mlen = 20; 249 break; 250 } 251 } 252 break; 253 case CRYPTO_DES_CBC: 254 case CRYPTO_3DES_CBC: 255 case CRYPTO_AES_CBC: 256 /* XXX this may read fewer, does it matter? */ 257 /* 258 * read_random(ses->hs_iv, c->cri_alg == 259 * CRYPTO_AES_CBC ? XLR_SEC_AES_IV_LENGTH : 260 * XLR_SEC_IV_LENGTH); 261 */ 262 /* FALLTHROUGH */ 263 case CRYPTO_ARC4: 264 if (cry) 265 return (EINVAL); 266 cry = 1; 267 break; 268 default: 269 return (EINVAL); 270 } 271 } 272 if (mac == 0 && cry == 0) 273 return (EINVAL); 274 275 *sidp = XLR_SEC_SID(device_get_unit(sc->sc_dev), sesn); 276 return (0); 277} 278 279/* 280 * Deallocate a session. 281 * XXX this routine should run a zero'd mac/encrypt key into context ram. 282 * XXX to blow away any keys already stored there. 283 */ 284static int 285xlr_sec_freesession(device_t dev, u_int64_t tid) 286{ 287 struct xlr_sec_softc *sc = device_get_softc(dev); 288 int session; 289 u_int32_t sid = CRYPTO_SESID2LID(tid); 290 291 if (sc == NULL) 292 return (EINVAL); 293 294 session = XLR_SEC_SESSION(sid); 295 if (session >= sc->sc_nsessions) 296 return (EINVAL); 297 298 sc->sc_sessions[session].hs_used = 0; 299 return (0); 300} 301 302#ifdef RMI_SEC_DEBUG 303 304void 305xlr_sec_print_data(struct cryptop *crp) 306{ 307 int i, key_len; 308 struct cryptodesc *crp_desc; 309 310 printf("session id = 0x%llx, crp_ilen = %d, crp_olen=%d \n", 311 crp->crp_sid, crp->crp_ilen, crp->crp_olen); 312 313 printf("crp_flags = 0x%x\n", crp->crp_flags); 314 315 316 printf("crp buf:\n"); 317 for (i = 0; i < crp->crp_ilen; i++) { 318 printf("%c ", crp->crp_buf[i]); 319 if (i % 10 == 0) 320 printf("\n"); 321 } 322 323 printf("\n"); 324 printf("****************** desc ****************\n"); 325 crp_desc = crp->crp_desc; 326 printf("crd_skip=%d, crd_len=%d, crd_flags=0x%x, crd_alg=%d\n", 327 crp_desc->crd_skip, crp_desc->crd_len, crp_desc->crd_flags, crp_desc->crd_alg); 328 329 key_len = crp_desc->crd_klen / 8; 330 printf("key(%d) :\n", key_len); 331 for (i = 0; i < key_len; i++) 332 printf("%d", crp_desc->crd_key[i]); 333 printf("\n"); 334 335 printf(" IV : \n"); 336 for (i = 0; i < EALG_MAX_BLOCK_LEN; i++) 337 printf("%d", crp_desc->crd_iv[i]); 338 printf("\n"); 339 340 printf("crd_next=%p\n", crp_desc->crd_next); 341 return; 342} 343 344#endif 345 346static int 347xlr_sec_process(device_t dev, struct cryptop *crp, int hint) 348{ 349 struct xlr_sec_softc *sc = device_get_softc(dev); 350 struct xlr_sec_command *cmd = NULL; 351 int session, err; 352 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 353 struct xlr_sec_session *ses; 354 355 if (crp == NULL || crp->crp_callback == NULL) { 356 return (EINVAL); 357 } 358 session = XLR_SEC_SESSION(crp->crp_sid); 359 if (sc == NULL || session >= sc->sc_nsessions) { 360 err = EINVAL; 361 goto errout; 362 } 363 ses = &sc->sc_sessions[session]; 364 365 cmd = &ses->cmd; 366 if (cmd == NULL) { 367 err = ENOMEM; 368 goto errout; 369 } 370 crd1 = crp->crp_desc; 371 if (crd1 == NULL) { 372 err = EINVAL; 373 goto errout; 374 } 375 crd2 = crd1->crd_next; 376 377 if (crd2 == NULL) { 378 if (crd1->crd_alg == CRYPTO_MD5_HMAC || 379 crd1->crd_alg == CRYPTO_SHA1_HMAC || 380 crd1->crd_alg == CRYPTO_SHA1 || 381 crd1->crd_alg == CRYPTO_MD5) { 382 maccrd = crd1; 383 enccrd = NULL; 384 } else if (crd1->crd_alg == CRYPTO_DES_CBC || 385 crd1->crd_alg == CRYPTO_3DES_CBC || 386 crd1->crd_alg == CRYPTO_AES_CBC || 387 crd1->crd_alg == CRYPTO_ARC4) { 388 maccrd = NULL; 389 enccrd = crd1; 390 } else { 391 err = EINVAL; 392 goto errout; 393 } 394 } else { 395 if ((crd1->crd_alg == CRYPTO_MD5_HMAC || 396 crd1->crd_alg == CRYPTO_SHA1_HMAC || 397 crd1->crd_alg == CRYPTO_MD5 || 398 crd1->crd_alg == CRYPTO_SHA1) && 399 (crd2->crd_alg == CRYPTO_DES_CBC || 400 crd2->crd_alg == CRYPTO_3DES_CBC || 401 crd2->crd_alg == CRYPTO_AES_CBC || 402 crd2->crd_alg == CRYPTO_ARC4)) { 403 maccrd = crd1; 404 enccrd = crd2; 405 } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 406 crd1->crd_alg == CRYPTO_ARC4 || 407 crd1->crd_alg == CRYPTO_3DES_CBC || 408 crd1->crd_alg == CRYPTO_AES_CBC) && 409 (crd2->crd_alg == CRYPTO_MD5_HMAC || 410 crd2->crd_alg == CRYPTO_SHA1_HMAC || 411 crd2->crd_alg == CRYPTO_MD5 || 412 crd2->crd_alg == CRYPTO_SHA1) && 413 (crd1->crd_flags & CRD_F_ENCRYPT)) { 414 enccrd = crd1; 415 maccrd = crd2; 416 } else { 417 err = EINVAL; 418 goto errout; 419 } 420 } 421 422 bzero(&cmd->op, sizeof(xlr_sec_io_t)); 423 424 cmd->op.source_buf = (uint64_t) (unsigned long)crp->crp_buf; 425 cmd->op.source_buf_size = crp->crp_ilen; 426 if (crp->crp_flags & CRYPTO_F_REL) { 427 cmd->op.dest_buf = (uint64_t) (unsigned long)crp->crp_buf; 428 cmd->op.dest_buf_size = crp->crp_ilen; 429 } else { 430 cmd->op.dest_buf = (uint64_t) (unsigned long)crp->crp_buf; 431 cmd->op.dest_buf_size = crp->crp_ilen; 432 } 433 cmd->op.num_packets = 1; 434 cmd->op.num_fragments = 1; 435 436 if (cmd->op.source_buf_size > SEC_MAX_FRAG_LEN) { 437 ses->multi_frag_flag = 1; 438 } else { 439 ses->multi_frag_flag = 0; 440 } 441 442 if (maccrd) { 443 cmd->maccrd = maccrd; 444 cmd->op.cipher_op = XLR_SEC_CIPHER_MODE_PASS; 445 cmd->op.cipher_mode = XLR_SEC_CIPHER_MODE_NONE; 446 cmd->op.cipher_type = XLR_SEC_CIPHER_TYPE_NONE; 447 cmd->op.cipher_init = 0; 448 cmd->op.cipher_offset = 0; 449 450 switch (maccrd->crd_alg) { 451 case CRYPTO_MD5: 452 cmd->op.digest_type = XLR_SEC_DIGEST_TYPE_MD5; 453 cmd->op.digest_init = XLR_SEC_DIGEST_INIT_NEWKEY; 454 cmd->op.digest_src = XLR_SEC_DIGEST_SRC_DMA; 455 cmd->op.digest_offset = 0; 456 457 cmd->op.cksum_type = XLR_SEC_CKSUM_TYPE_NOP; 458 cmd->op.cksum_src = XLR_SEC_CKSUM_SRC_CIPHER; 459 cmd->op.cksum_offset = 0; 460 461 cmd->op.pkt_hmac = XLR_SEC_LOADHMACKEY_MODE_OLD; 462 cmd->op.pkt_hash = XLR_SEC_PADHASH_PAD; 463 cmd->op.pkt_hashbytes = XLR_SEC_HASHBYTES_ALL8; 464 cmd->op.pkt_next = XLR_SEC_NEXT_FINISH; 465 cmd->op.pkt_iv = XLR_SEC_PKT_IV_OLD; 466 cmd->op.pkt_lastword = XLR_SEC_LASTWORD_128; 467 468 default: 469 printf("currently not handled\n"); 470 } 471 } 472 if (enccrd) { 473 cmd->enccrd = enccrd; 474 475#ifdef RMI_SEC_DEBUG 476 xlr_sec_print_data(crp); 477#endif 478 479 if (enccrd->crd_flags & CRD_F_ENCRYPT) { 480 cmd->op.cipher_op = XLR_SEC_CIPHER_OP_ENCRYPT; 481 } else 482 cmd->op.cipher_op = XLR_SEC_CIPHER_OP_DECRYPT; 483 484 switch (enccrd->crd_alg) { 485 case CRYPTO_DES_CBC: 486 case CRYPTO_3DES_CBC: 487 if (enccrd->crd_alg == CRYPTO_DES_CBC) { 488 cmd->op.cipher_type = XLR_SEC_CIPHER_TYPE_DES; 489 memcpy(&cmd->op.crypt_key[0], enccrd->crd_key, XLR_SEC_DES_KEY_LENGTH); 490 } else { 491 cmd->op.cipher_type = XLR_SEC_CIPHER_TYPE_3DES; 492 //if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) 493 { 494 memcpy(&cmd->op.crypt_key[0], enccrd->crd_key, 495 XLR_SEC_3DES_KEY_LENGTH); 496 } 497 } 498 499 cmd->op.cipher_mode = XLR_SEC_CIPHER_MODE_CBC; 500 cmd->op.cipher_init = XLR_SEC_CIPHER_INIT_NK; 501 cmd->op.cipher_offset = XLR_SEC_DES_IV_LENGTH; 502 503 cmd->op.digest_type = XLR_SEC_DIGEST_TYPE_NONE; 504 cmd->op.digest_init = XLR_SEC_DIGEST_INIT_OLDKEY; 505 cmd->op.digest_src = XLR_SEC_DIGEST_SRC_DMA; 506 cmd->op.digest_offset = 0; 507 508 cmd->op.cksum_type = XLR_SEC_CKSUM_TYPE_NOP; 509 cmd->op.cksum_src = XLR_SEC_CKSUM_SRC_CIPHER; 510 cmd->op.cksum_offset = 0; 511 512 cmd->op.pkt_hmac = XLR_SEC_LOADHMACKEY_MODE_OLD; 513 cmd->op.pkt_hash = XLR_SEC_PADHASH_PAD; 514 cmd->op.pkt_hashbytes = XLR_SEC_HASHBYTES_ALL8; 515 cmd->op.pkt_next = XLR_SEC_NEXT_FINISH; 516 cmd->op.pkt_iv = XLR_SEC_PKT_IV_NEW; 517 cmd->op.pkt_lastword = XLR_SEC_LASTWORD_128; 518 519 //if ((!(enccrd->crd_flags & CRD_F_IV_PRESENT)) && 520 if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT)) { 521 memcpy(&cmd->op.initial_vector[0], enccrd->crd_iv, 522 XLR_SEC_DES_IV_LENGTH); 523 } 524 break; 525 526 case CRYPTO_AES_CBC: 527 if (enccrd->crd_alg == CRYPTO_AES_CBC) { 528 cmd->op.cipher_type = XLR_SEC_CIPHER_TYPE_AES128; 529 //if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) 530 { 531 memcpy(&cmd->op.crypt_key[0], enccrd->crd_key, 532 XLR_SEC_AES128_KEY_LENGTH); 533 } 534 } 535 cmd->op.cipher_mode = XLR_SEC_CIPHER_MODE_CBC; 536 cmd->op.cipher_init = XLR_SEC_CIPHER_INIT_NK; 537 cmd->op.cipher_offset = XLR_SEC_AES_BLOCK_SIZE; 538 539 cmd->op.digest_type = XLR_SEC_DIGEST_TYPE_NONE; 540 cmd->op.digest_init = XLR_SEC_DIGEST_INIT_OLDKEY; 541 cmd->op.digest_src = XLR_SEC_DIGEST_SRC_DMA; 542 cmd->op.digest_offset = 0; 543 544 cmd->op.cksum_type = XLR_SEC_CKSUM_TYPE_NOP; 545 cmd->op.cksum_src = XLR_SEC_CKSUM_SRC_CIPHER; 546 cmd->op.cksum_offset = 0; 547 548 cmd->op.pkt_hmac = XLR_SEC_LOADHMACKEY_MODE_OLD; 549 cmd->op.pkt_hash = XLR_SEC_PADHASH_PAD; 550 cmd->op.pkt_hashbytes = XLR_SEC_HASHBYTES_ALL8; 551 cmd->op.pkt_next = XLR_SEC_NEXT_FINISH; 552 cmd->op.pkt_iv = XLR_SEC_PKT_IV_NEW; 553 cmd->op.pkt_lastword = XLR_SEC_LASTWORD_128; 554 555 //if (!(enccrd->crd_flags & CRD_F_IV_PRESENT)) { 556 if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT)) { 557 memcpy(&cmd->op.initial_vector[0], enccrd->crd_iv, 558 XLR_SEC_AES_BLOCK_SIZE); 559 } 560 //} 561 break; 562 } 563 } 564 cmd->crp = crp; 565 cmd->session_num = session; 566 xlr_sec_setup(ses, cmd, (symkey_desc_pt) ses->desc_ptr); 567 568 return (0); 569 570errout: 571 if (cmd != NULL) 572 free(cmd, M_DEVBUF); 573 crp->crp_etype = err; 574 crypto_done(crp); 575 return (err); 576} 577