1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2003-2012 Broadcom Corporation 5 * All Rights Reserved 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 25 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 26 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 27 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 28 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31#include <sys/cdefs.h> 32__FBSDID("$FreeBSD$"); 33 34#include <sys/cdefs.h> 35#include <sys/param.h> 36#include <sys/systm.h> 37#include <sys/proc.h> 38#include <sys/errno.h> 39#include <sys/malloc.h> 40#include <sys/kernel.h> 41#include <sys/module.h> 42#include <sys/mbuf.h> 43#include <sys/lock.h> 44#include <sys/mutex.h> 45#include <sys/sysctl.h> 46#include <sys/bus.h> 47#include <sys/random.h> 48#include <sys/rman.h> 49#include <sys/uio.h> 50#include <sys/kobj.h> 51 52#include <dev/pci/pcivar.h> 53 54#include <opencrypto/cryptodev.h> 55#include <opencrypto/xform_auth.h> 56 57#include "cryptodev_if.h" 58 59#include <vm/vm.h> 60#include <vm/pmap.h> 61 62#include <mips/nlm/hal/haldefs.h> 63#include <mips/nlm/hal/iomap.h> 64#include <mips/nlm/xlp.h> 65#include <mips/nlm/hal/sys.h> 66#include <mips/nlm/hal/fmn.h> 67#include <mips/nlm/hal/nlmsaelib.h> 68#include <mips/nlm/dev/sec/nlmseclib.h> 69#include <mips/nlm/hal/cop2.h> 70#include <mips/nlm/hal/mips-extns.h> 71#include <mips/nlm/msgring.h> 72 73unsigned int creditleft; 74 75static int xlp_sec_init(struct xlp_sec_softc *sc); 76static int xlp_sec_probesession(device_t, 77 const struct crypto_session_params *); 78static int xlp_sec_newsession(device_t , crypto_session_t, 79 const struct crypto_session_params *); 80static int xlp_sec_process(device_t , struct cryptop *, int); 81static void xlp_copyiv(struct xlp_sec_softc *, struct xlp_sec_command *, 82 const struct crypto_session_params *); 83static int xlp_get_nsegs(struct cryptop *, unsigned int *); 84static int xlp_alloc_cmd_params(struct xlp_sec_command *, unsigned int); 85static void xlp_free_cmd_params(struct xlp_sec_command *); 86 87static int xlp_sec_probe(device_t); 88static int xlp_sec_attach(device_t); 89static int xlp_sec_detach(device_t); 90 91static device_method_t xlp_sec_methods[] = { 92 /* device interface */ 93 DEVMETHOD(device_probe, xlp_sec_probe), 94 DEVMETHOD(device_attach, xlp_sec_attach), 95 DEVMETHOD(device_detach, xlp_sec_detach), 96 97 /* bus interface */ 98 DEVMETHOD(bus_print_child, bus_generic_print_child), 99 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 100 101 /* crypto device methods */ 102 DEVMETHOD(cryptodev_probesession, xlp_sec_probesession), 103 DEVMETHOD(cryptodev_newsession, xlp_sec_newsession), 104 DEVMETHOD(cryptodev_process, xlp_sec_process), 105 106 DEVMETHOD_END 107}; 108 109static driver_t xlp_sec_driver = { 110 "nlmsec", 111 xlp_sec_methods, 112 sizeof(struct xlp_sec_softc) 113}; 114static devclass_t xlp_sec_devclass; 115 116DRIVER_MODULE(nlmsec, pci, xlp_sec_driver, xlp_sec_devclass, 0, 0); 117MODULE_DEPEND(nlmsec, crypto, 1, 1, 1); 118 119void 120nlm_xlpsec_msgring_handler(int vc, int size, int code, int src_id, 121 struct nlm_fmn_msg *msg, void *data); 122 123#ifdef NLM_SEC_DEBUG 124 125#define extract_bits(x, bitshift, bitcnt) \ 126 (((unsigned long long)x >> bitshift) & ((1ULL << bitcnt) - 1)) 127 128void 129print_crypto_params(struct xlp_sec_command *cmd, struct nlm_fmn_msg m) 130{ 131 unsigned long long msg0,msg1,msg2,msg3,msg4,msg5,msg6,msg7,msg8; 132 133 msg0 = cmd->ctrlp->desc0; 134 msg1 = cmd->paramp->desc0; 135 msg2 = cmd->paramp->desc1; 136 msg3 = cmd->paramp->desc2; 137 msg4 = cmd->paramp->desc3; 138 msg5 = cmd->paramp->segment[0][0]; 139 msg6 = cmd->paramp->segment[0][1]; 140 msg7 = m.msg[0]; 141 msg8 = m.msg[1]; 142 143 printf("msg0 %llx msg1 %llx msg2 %llx msg3 %llx msg4 %llx msg5 %llx" 144 "msg6 %llx msg7 %llx msg8 %llx\n", msg0, msg1, msg2, msg3, msg4, 145 msg5, msg6, msg7, msg8); 146 147 printf("c0: hmac %d htype %d hmode %d ctype %d cmode %d arc4 %x\n", 148 (unsigned int)extract_bits(msg0, 61, 1), 149 (unsigned int)extract_bits(msg0, 52, 8), 150 (unsigned int)extract_bits(msg0, 43, 8), 151 (unsigned int)extract_bits(msg0, 34, 8), 152 (unsigned int)extract_bits(msg0, 25, 8), 153 (unsigned int)extract_bits(msg0, 0, 23)); 154 155 printf("p0: tls %d hsrc %d hl3 %d enc %d ivl %d hd %llx\n", 156 (unsigned int)extract_bits(msg1, 63, 1), 157 (unsigned int)extract_bits(msg1,62,1), 158 (unsigned int)extract_bits(msg1,60,1), 159 (unsigned int)extract_bits(msg1,59,1), 160 (unsigned int)extract_bits(msg1,41,16), extract_bits(msg1,0,40)); 161 162 printf("p1: clen %u hl %u\n", (unsigned int)extract_bits(msg2, 32, 32), 163 (unsigned int)extract_bits(msg2,0,32)); 164 165 printf("p2: ivoff %d cbit %d coff %d hbit %d hclb %d hoff %d\n", 166 (unsigned int)extract_bits(msg3, 45, 17), 167 (unsigned int)extract_bits(msg3, 42,3), 168 (unsigned int)extract_bits(msg3, 22,16), 169 (unsigned int)extract_bits(msg3, 19,3), 170 (unsigned int)extract_bits(msg3, 18,1), 171 (unsigned int)extract_bits(msg3, 0, 16)); 172 173 printf("p3: desfbid %d tlen %d arc4 %x hmacpad %d\n", 174 (unsigned int)extract_bits(msg4, 48,16), 175 (unsigned int)extract_bits(msg4,11,16), 176 (unsigned int)extract_bits(msg4,6,3), 177 (unsigned int)extract_bits(msg4,5,1)); 178 179 printf("p4: sflen %d sddr %llx \n", 180 (unsigned int)extract_bits(msg5, 48, 16),extract_bits(msg5, 0, 40)); 181 182 printf("p5: dflen %d cl3 %d cclob %d cdest %llx \n", 183 (unsigned int)extract_bits(msg6, 48, 16), 184 (unsigned int)extract_bits(msg6, 46, 1), 185 (unsigned int)extract_bits(msg6, 41, 1), extract_bits(msg6, 0, 40)); 186 187 printf("fmn0: fbid %d dfrlen %d dfrv %d cklen %d cdescaddr %llx\n", 188 (unsigned int)extract_bits(msg7, 48, 16), 189 (unsigned int)extract_bits(msg7,46,2), 190 (unsigned int)extract_bits(msg7,45,1), 191 (unsigned int)extract_bits(msg7,40,5), 192 (extract_bits(msg7,0,34)<< 6)); 193 194 printf("fmn1: arc4 %d hklen %d pdesclen %d pktdescad %llx\n", 195 (unsigned int)extract_bits(msg8, 63, 1), 196 (unsigned int)extract_bits(msg8,56,5), 197 (unsigned int)extract_bits(msg8,43,12), 198 (extract_bits(msg8,0,34) << 6)); 199 200 return; 201} 202 203void 204print_cmd(struct xlp_sec_command *cmd) 205{ 206 printf("session_num :%d\n",cmd->session_num); 207 printf("crp :0x%x\n",(uint32_t)cmd->crp); 208 printf("enccrd :0x%x\n",(uint32_t)cmd->enccrd); 209 printf("maccrd :0x%x\n",(uint32_t)cmd->maccrd); 210 printf("ses :%d\n",(uint32_t)cmd->ses); 211 printf("ctrlp :0x%x\n",(uint32_t)cmd->ctrlp); 212 printf("paramp :0x%x\n",(uint32_t)cmd->paramp); 213 printf("hashdest :0x%x\n",(uint32_t)cmd->hashdest); 214 printf("hashsrc :%d\n",cmd->hashsrc); 215 printf("hmacpad :%d\n",cmd->hmacpad); 216 printf("hashoff :%d\n",cmd->hashoff); 217 printf("hashlen :%d\n",cmd->hashlen); 218 printf("cipheroff :%d\n",cmd->cipheroff); 219 printf("cipherlen :%d\n",cmd->cipherlen); 220 printf("ivoff :%d\n",cmd->ivoff); 221 printf("ivlen :%d\n",cmd->ivlen); 222 printf("hashalg :%d\n",cmd->hashalg); 223 printf("hashmode :%d\n",cmd->hashmode); 224 printf("cipheralg :%d\n",cmd->cipheralg); 225 printf("ciphermode :%d\n",cmd->ciphermode); 226 printf("nsegs :%d\n",cmd->nsegs); 227 printf("hash_dst_len :%d\n",cmd->hash_dst_len); 228} 229#endif /* NLM_SEC_DEBUG */ 230 231static int 232xlp_sec_init(struct xlp_sec_softc *sc) 233{ 234 235 /* Register interrupt handler for the SEC CMS messages */ 236 if (register_msgring_handler(sc->sec_vc_start, 237 sc->sec_vc_end, nlm_xlpsec_msgring_handler, sc) != 0) { 238 printf("Couldn't register sec msgring handler\n"); 239 return (-1); 240 } 241 242 /* Do the CMS credit initialization */ 243 /* Currently it is configured by default to 50 when kernel comes up */ 244 245 return (0); 246} 247 248/* This function is called from an interrupt handler */ 249void 250nlm_xlpsec_msgring_handler(int vc, int size, int code, int src_id, 251 struct nlm_fmn_msg *msg, void *data) 252{ 253 struct xlp_sec_command *cmd = NULL; 254 struct xlp_sec_softc *sc = NULL; 255 uint8_t hash[HASH_MAX_LEN]; 256 257 KASSERT(code == FMN_SWCODE_CRYPTO, 258 ("%s: bad code = %d, expected code = %d\n", __FUNCTION__, 259 code, FMN_SWCODE_CRYPTO)); 260 261 sc = (struct xlp_sec_softc *)data; 262 KASSERT(src_id >= sc->sec_vc_start && src_id <= sc->sec_vc_end, 263 ("%s: bad src_id = %d, expect %d - %d\n", __FUNCTION__, 264 src_id, sc->sec_vc_start, sc->sec_vc_end)); 265 266 cmd = (struct xlp_sec_command *)(uintptr_t)msg->msg[0]; 267 KASSERT(cmd != NULL && cmd->crp != NULL, 268 ("%s :cmd not received properly\n",__FUNCTION__)); 269 270 KASSERT(CRYPTO_ERROR(msg->msg[1]) == 0, 271 ("%s: Message rcv msg0 %llx msg1 %llx err %x \n", __FUNCTION__, 272 (unsigned long long)msg->msg[0], (unsigned long long)msg->msg[1], 273 (int)CRYPTO_ERROR(msg->msg[1]))); 274 275 /* If there are not enough credits to send, then send request 276 * will fail with ERESTART and the driver will be blocked until it is 277 * unblocked here after knowing that there are sufficient credits to 278 * send the request again. 279 */ 280 if (sc->sc_needwakeup) { 281 atomic_add_int(&creditleft, sc->sec_msgsz); 282 if (creditleft >= (NLM_CRYPTO_LEFT_REQS)) { 283 crypto_unblock(sc->sc_cid, sc->sc_needwakeup); 284 sc->sc_needwakeup &= (~(CRYPTO_SYMQ | CRYPTO_ASYMQ)); 285 } 286 } 287 if (cmd->hash_dst_len != 0) { 288 if (cmd->crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 289 crypto_copydata(cmd->crp, cmd->crp->crp_digest_start, 290 cmd->hash_dst_len, hash); 291 if (timingsafe_bcmp(cmd->hashdest, hash, 292 cmd->hash_dst_len) != 0) 293 cmd->crp->crp_etype = EBADMSG; 294 } else 295 crypto_copyback(cmd->crp, cmd->crp->crp_digest_start, 296 cmd->hash_dst_len, cmd->hashdest); 297 } 298 299 /* This indicates completion of the crypto operation */ 300 crypto_done(cmd->crp); 301 302 xlp_free_cmd_params(cmd); 303 304 return; 305} 306 307static int 308xlp_sec_probe(device_t dev) 309{ 310 struct xlp_sec_softc *sc; 311 312 if (pci_get_vendor(dev) == PCI_VENDOR_NETLOGIC && 313 pci_get_device(dev) == PCI_DEVICE_ID_NLM_SAE) { 314 sc = device_get_softc(dev); 315 return (BUS_PROBE_DEFAULT); 316 } 317 return (ENXIO); 318} 319 320/* 321 * Attach an interface that successfully probed. 322 */ 323static int 324xlp_sec_attach(device_t dev) 325{ 326 struct xlp_sec_softc *sc = device_get_softc(dev); 327 uint64_t base; 328 int qstart, qnum; 329 int freq, node; 330 331 sc->sc_dev = dev; 332 333 node = nlm_get_device_node(pci_get_slot(dev)); 334 freq = nlm_set_device_frequency(node, DFS_DEVICE_SAE, 250); 335 if (bootverbose) 336 device_printf(dev, "SAE Freq: %dMHz\n", freq); 337 if(pci_get_device(dev) == PCI_DEVICE_ID_NLM_SAE) { 338 device_set_desc(dev, "XLP Security Accelerator"); 339 sc->sc_cid = crypto_get_driverid(dev, 340 sizeof(struct xlp_sec_session), CRYPTOCAP_F_HARDWARE); 341 if (sc->sc_cid < 0) { 342 printf("xlp_sec - error : could not get the driver" 343 " id\n"); 344 goto error_exit; 345 } 346 347 base = nlm_get_sec_pcibase(node); 348 qstart = nlm_qidstart(base); 349 qnum = nlm_qnum(base); 350 sc->sec_vc_start = qstart; 351 sc->sec_vc_end = qstart + qnum - 1; 352 } 353 354 if (xlp_sec_init(sc) != 0) 355 goto error_exit; 356 if (bootverbose) 357 device_printf(dev, "SEC Initialization complete!\n"); 358 return (0); 359 360error_exit: 361 return (ENXIO); 362 363} 364 365/* 366 * Detach an interface that successfully probed. 367 */ 368static int 369xlp_sec_detach(device_t dev) 370{ 371 return (0); 372} 373 374static bool 375xlp_sec_auth_supported(const struct crypto_session_params *csp) 376{ 377 378 switch (csp->csp_auth_alg) { 379 case CRYPTO_SHA1: 380 case CRYPTO_SHA1_HMAC: 381 break; 382 default: 383 return (false); 384 } 385 return (true); 386} 387 388static bool 389xlp_sec_cipher_supported(const struct crypto_session_params *csp) 390{ 391 392 switch (csp->csp_cipher_alg) { 393 case CRYPTO_AES_CBC: 394 if (csp->csp_ivlen != XLP_SEC_AES_IV_LENGTH) 395 return (false); 396 break; 397 default: 398 return (false); 399 } 400 401 return (true); 402} 403 404static int 405xlp_sec_probesession(device_t dev, const struct crypto_session_params *csp) 406{ 407 408 if (csp->csp_flags != 0) 409 return (EINVAL); 410 switch (csp->csp_mode) { 411 case CSP_MODE_DIGEST: 412 if (!xlp_sec_auth_supported(csp)) 413 return (EINVAL); 414 break; 415 case CSP_MODE_CIPHER: 416 if (!xlp_sec_cipher_supported(csp)) 417 return (EINVAL); 418 break; 419 case CSP_MODE_ETA: 420 if (!xlp_sec_auth_supported(csp) || 421 !xlp_sec_cipher_supported(csp)) 422 return (EINVAL); 423 break; 424 default: 425 return (EINVAL); 426 } 427 return (CRYPTODEV_PROBE_HARDWARE); 428} 429 430static int 431xlp_sec_newsession(device_t dev, crypto_session_t cses, 432 const struct crypto_session_params *csp) 433{ 434 struct xlp_sec_session *ses; 435 436 ses = crypto_get_driver_session(cses); 437 438 if (csp->csp_auth_alg != 0) { 439 if (csp->csp_auth_mlen == 0) 440 ses->hs_mlen = crypto_auth_hash(csp)->hashsize; 441 else 442 ses->hs_mlen = csp->csp_auth_mlen; 443 } 444 445 return (0); 446} 447 448/* 449 * XXX freesession routine should run a zero'd mac/encrypt key into context 450 * ram. to blow away any keys already stored there. 451 */ 452 453static void 454xlp_copyiv(struct xlp_sec_softc *sc, struct xlp_sec_command *cmd, 455 const struct crypto_session_params *csp) 456{ 457 struct cryptop *crp = NULL; 458 459 crp = cmd->crp; 460 461 if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) 462 memcpy(cmd->iv, crp->crp_iv, csp->csp_ivlen); 463} 464 465static int 466xlp_get_nsegs(struct cryptop *crp, unsigned int *nsegs) 467{ 468 469 switch (crp->crp_buf.cb_type) { 470 case CRYPTO_BUF_MBUF: 471 { 472 struct mbuf *m = NULL; 473 474 m = crp->crp_buf.cb_mbuf; 475 while (m != NULL) { 476 *nsegs += NLM_CRYPTO_NUM_SEGS_REQD(m->m_len); 477 m = m->m_next; 478 } 479 break; 480 } 481 case CRYPTO_BUF_UIO: 482 { 483 struct uio *uio = NULL; 484 struct iovec *iov = NULL; 485 int iol = 0; 486 487 uio = crp->crp_buf.cb_uio; 488 iov = uio->uio_iov; 489 iol = uio->uio_iovcnt; 490 while (iol > 0) { 491 *nsegs += NLM_CRYPTO_NUM_SEGS_REQD(iov->iov_len); 492 iol--; 493 iov++; 494 } 495 break; 496 } 497 case CRYPTO_BUF_CONTIG: 498 *nsegs = NLM_CRYPTO_NUM_SEGS_REQD(crp->crp_buf.cb_buf_len); 499 break; 500 default: 501 return (EINVAL); 502 } 503 return (0); 504} 505 506static int 507xlp_alloc_cmd_params(struct xlp_sec_command *cmd, unsigned int nsegs) 508{ 509 int err = 0; 510 511 if(cmd == NULL) { 512 err = EINVAL; 513 goto error; 514 } 515 if ((cmd->ctrlp = malloc(sizeof(struct nlm_crypto_pkt_ctrl), M_DEVBUF, 516 M_NOWAIT | M_ZERO)) == NULL) { 517 err = ENOMEM; 518 goto error; 519 } 520 if (((uintptr_t)cmd->ctrlp & (XLP_L2L3_CACHELINE_SIZE - 1))) { 521 err = EINVAL; 522 goto error; 523 } 524 /* (nsegs - 1) because one seg is part of the structure already */ 525 if ((cmd->paramp = malloc(sizeof(struct nlm_crypto_pkt_param) + 526 (16 * (nsegs - 1)), M_DEVBUF, M_NOWAIT | M_ZERO)) == NULL) { 527 err = ENOMEM; 528 goto error; 529 } 530 if (((uintptr_t)cmd->paramp & (XLP_L2L3_CACHELINE_SIZE - 1))) { 531 err = EINVAL; 532 goto error; 533 } 534 if ((cmd->iv = malloc(EALG_MAX_BLOCK_LEN, M_DEVBUF, 535 M_NOWAIT | M_ZERO)) == NULL) { 536 err = ENOMEM; 537 goto error; 538 } 539 if ((cmd->hashdest = malloc(HASH_MAX_LEN, M_DEVBUF, 540 M_NOWAIT | M_ZERO)) == NULL) { 541 err = ENOMEM; 542 goto error; 543 } 544error: 545 return (err); 546} 547 548static void 549xlp_free_cmd_params(struct xlp_sec_command *cmd) 550{ 551 if (cmd->ctrlp != NULL) 552 free(cmd->ctrlp, M_DEVBUF); 553 if (cmd->paramp != NULL) 554 free(cmd->paramp, M_DEVBUF); 555 if (cmd->iv != NULL) 556 free(cmd->iv, M_DEVBUF); 557 if (cmd->hashdest != NULL) 558 free(cmd->hashdest, M_DEVBUF); 559 if (cmd != NULL) 560 free(cmd, M_DEVBUF); 561 return; 562} 563 564static int 565xlp_sec_process(device_t dev, struct cryptop *crp, int hint) 566{ 567 struct xlp_sec_softc *sc = device_get_softc(dev); 568 const struct crypto_session_params *csp; 569 struct xlp_sec_command *cmd = NULL; 570 int err = -1, ret = 0; 571 struct xlp_sec_session *ses; 572 unsigned int nsegs = 0; 573 574 ses = crypto_get_driver_session(crp->crp_session); 575 csp = crypto_get_params(crp->crp_session); 576 577 /* 578 * This device only support AAD requests where the AAD is 579 * adjacent to the payload. 580 */ 581 if (crp->crp_aad_length != 0 && crp->crp_payload_start != 582 crp->crp_aad_start + crp->crp_aad_length) { 583 err = EFBIG; 584 goto errout; 585 } 586 587 if ((cmd = malloc(sizeof(struct xlp_sec_command), M_DEVBUF, 588 M_NOWAIT | M_ZERO)) == NULL) { 589 err = ENOMEM; 590 goto errout; 591 } 592 593 cmd->crp = crp; 594 cmd->ses = ses; 595 cmd->hash_dst_len = ses->hs_mlen; 596 597 if ((ret = xlp_get_nsegs(crp, &nsegs)) != 0) { 598 err = EINVAL; 599 goto errout; 600 } 601 602 if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) { 603 /* Since IV is given as separate segment to avoid copy */ 604 nsegs += 1; 605 } 606 cmd->nsegs = nsegs; 607 608 if ((err = xlp_alloc_cmd_params(cmd, nsegs)) != 0) 609 goto errout; 610 611 switch (csp->csp_mode) { 612 case CSP_MODE_CIPHER: 613 if ((ret = nlm_get_cipher_param(cmd, csp)) != 0) { 614 err = EINVAL; 615 goto errout; 616 } 617 cmd->cipheroff = crp->crp_payload_start; 618 cmd->cipherlen = crp->crp_payload_length; 619 if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) { 620 cmd->cipheroff += cmd->ivlen; 621 cmd->ivoff = 0; 622 } else 623 cmd->ivoff = crp->crp_iv_start; 624 xlp_copyiv(sc, cmd, csp); 625 if ((err = nlm_crypto_do_cipher(sc, cmd, csp)) != 0) 626 goto errout; 627 break; 628 case CSP_MODE_DIGEST: 629 if ((ret = nlm_get_digest_param(cmd, csp)) != 0) { 630 err = EINVAL; 631 goto errout; 632 } 633 cmd->hashoff = crp->crp_payload_start; 634 cmd->hashlen = crp->crp_payload_length; 635 cmd->hmacpad = 0; 636 cmd->hashsrc = 0; 637 if ((err = nlm_crypto_do_digest(sc, cmd, csp)) != 0) 638 goto errout; 639 break; 640 case CSP_MODE_ETA: 641 if ((ret = nlm_get_cipher_param(cmd, csp)) != 0) { 642 err = EINVAL; 643 goto errout; 644 } 645 if ((ret = nlm_get_digest_param(cmd, csp)) != 0) { 646 err = EINVAL; 647 goto errout; 648 } 649 if (crp->crp_aad_length != 0) { 650 cmd->hashoff = crp->crp_aad_start; 651 cmd->hashlen = crp->crp_aad_length + 652 crp->crp_payload_length; 653 } else { 654 cmd->hashoff = crp->crp_payload_start; 655 cmd->hashlen = crp->crp_payload_length; 656 } 657 cmd->hmacpad = 0; 658 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 659 cmd->hashsrc = 1; 660 else 661 cmd->hashsrc = 0; 662 cmd->cipheroff = crp->crp_payload_start; 663 cmd->cipherlen = crp->crp_payload_length; 664 if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) { 665 cmd->hashoff += cmd->ivlen; 666 cmd->cipheroff += cmd->ivlen; 667 cmd->ivoff = 0; 668 } else 669 cmd->ivoff = crp->crp_iv_start; 670 xlp_copyiv(sc, cmd, csp); 671 if ((err = nlm_crypto_do_cipher_digest(sc, cmd, csp)) != 0) 672 goto errout; 673 break; 674 default: 675 err = EINVAL; 676 goto errout; 677 } 678 return (0); 679errout: 680 xlp_free_cmd_params(cmd); 681 if (err == ERESTART) { 682 sc->sc_needwakeup |= CRYPTO_SYMQ; 683 creditleft = 0; 684 return (err); 685 } 686 crp->crp_etype = err; 687 crypto_done(crp); 688 return (err); 689} 690