1/*- 2 * Copyright (c) 2003-2012 Broadcom Corporation 3 * All Rights Reserved 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in 13 * the documentation and/or other materials provided with the 14 * distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29#include <sys/cdefs.h> 30__FBSDID("$FreeBSD$"); 31 32#include <sys/cdefs.h> 33#include <sys/param.h> 34#include <sys/systm.h> 35#include <sys/proc.h> 36#include <sys/errno.h> 37#include <sys/malloc.h> 38#include <sys/kernel.h> 39#include <sys/module.h> 40#include <sys/mbuf.h> 41#include <sys/lock.h> 42#include <sys/mutex.h> 43#include <sys/sysctl.h> 44#include <sys/bus.h> 45#include <sys/random.h> 46#include <sys/rman.h> 47#include <sys/uio.h> 48#include <sys/kobj.h> 49 50#include <dev/pci/pcivar.h> 51 52#include <opencrypto/cryptodev.h> 53 54#include "cryptodev_if.h" 55 56#include <vm/vm.h> 57#include <vm/pmap.h> 58 59#include <mips/nlm/hal/haldefs.h> 60#include <mips/nlm/hal/iomap.h> 61#include <mips/nlm/xlp.h> 62#include <mips/nlm/hal/sys.h> 63#include <mips/nlm/hal/fmn.h> 64#include <mips/nlm/hal/nlmsaelib.h> 65#include <mips/nlm/dev/sec/nlmseclib.h> 66#include <mips/nlm/hal/cop2.h> 67#include <mips/nlm/hal/mips-extns.h> 68#include <mips/nlm/msgring.h> 69 70unsigned int creditleft; 71 72void xlp_sec_print_data(struct cryptop *crp); 73 74static int xlp_sec_init(struct xlp_sec_softc *sc); 75static int xlp_sec_newsession(device_t , uint32_t *, struct cryptoini *); 76static int xlp_sec_freesession(device_t , uint64_t); 77static int xlp_sec_process(device_t , struct cryptop *, int); 78static int xlp_copyiv(struct xlp_sec_softc *, struct xlp_sec_command *, 79 struct cryptodesc *enccrd); 80static int xlp_get_nsegs(struct cryptop *, unsigned int *); 81static int xlp_alloc_cmd_params(struct xlp_sec_command *, unsigned int); 82static void xlp_free_cmd_params(struct xlp_sec_command *); 83 84static int xlp_sec_probe(device_t); 85static int xlp_sec_attach(device_t); 86static int xlp_sec_detach(device_t); 87 88static device_method_t xlp_sec_methods[] = { 89 /* device interface */ 90 DEVMETHOD(device_probe, xlp_sec_probe), 91 DEVMETHOD(device_attach, xlp_sec_attach), 92 DEVMETHOD(device_detach, xlp_sec_detach), 93 94 /* bus interface */ 95 DEVMETHOD(bus_print_child, bus_generic_print_child), 96 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 97 98 /* crypto device methods */ 99 DEVMETHOD(cryptodev_newsession, xlp_sec_newsession), 100 DEVMETHOD(cryptodev_freesession,xlp_sec_freesession), 101 DEVMETHOD(cryptodev_process, xlp_sec_process), 102 103 DEVMETHOD_END 104}; 105 106static driver_t xlp_sec_driver = { 107 "nlmsec", 108 xlp_sec_methods, 109 sizeof(struct xlp_sec_softc) 110}; 111static devclass_t xlp_sec_devclass; 112 113DRIVER_MODULE(nlmsec, pci, xlp_sec_driver, xlp_sec_devclass, 0, 0); 114MODULE_DEPEND(nlmsec, crypto, 1, 1, 1); 115 116void 117nlm_xlpsec_msgring_handler(int vc, int size, int code, int src_id, 118 struct nlm_fmn_msg *msg, void *data); 119 120#ifdef NLM_SEC_DEBUG 121 122#define extract_bits(x, bitshift, bitcnt) \ 123 (((unsigned long long)x >> bitshift) & ((1ULL << bitcnt) - 1)) 124 125void 126print_crypto_params(struct xlp_sec_command *cmd, struct nlm_fmn_msg m) 127{ 128 unsigned long long msg0,msg1,msg2,msg3,msg4,msg5,msg6,msg7,msg8; 129 130 msg0 = cmd->ctrlp->desc0; 131 msg1 = cmd->paramp->desc0; 132 msg2 = cmd->paramp->desc1; 133 msg3 = cmd->paramp->desc2; 134 msg4 = cmd->paramp->desc3; 135 msg5 = cmd->paramp->segment[0][0]; 136 msg6 = cmd->paramp->segment[0][1]; 137 msg7 = m.msg[0]; 138 msg8 = m.msg[1]; 139 140 printf("msg0 %llx msg1 %llx msg2 %llx msg3 %llx msg4 %llx msg5 %llx" 141 "msg6 %llx msg7 %llx msg8 %llx\n", msg0, msg1, msg2, msg3, msg4, 142 msg5, msg6, msg7, msg8); 143 144 printf("c0: hmac %d htype %d hmode %d ctype %d cmode %d arc4 %x\n", 145 (unsigned int)extract_bits(msg0, 61, 1), 146 (unsigned int)extract_bits(msg0, 52, 8), 147 (unsigned int)extract_bits(msg0, 43, 8), 148 (unsigned int)extract_bits(msg0, 34, 8), 149 (unsigned int)extract_bits(msg0, 25, 8), 150 (unsigned int)extract_bits(msg0, 0, 23)); 151 152 printf("p0: tls %d hsrc %d hl3 %d enc %d ivl %d hd %llx\n", 153 (unsigned int)extract_bits(msg1, 63, 1), 154 (unsigned int)extract_bits(msg1,62,1), 155 (unsigned int)extract_bits(msg1,60,1), 156 (unsigned int)extract_bits(msg1,59,1), 157 (unsigned int)extract_bits(msg1,41,16), extract_bits(msg1,0,40)); 158 159 printf("p1: clen %u hl %u\n", (unsigned int)extract_bits(msg2, 32, 32), 160 (unsigned int)extract_bits(msg2,0,32)); 161 162 printf("p2: ivoff %d cbit %d coff %d hbit %d hclb %d hoff %d\n", 163 (unsigned int)extract_bits(msg3, 45, 17), 164 (unsigned int)extract_bits(msg3, 42,3), 165 (unsigned int)extract_bits(msg3, 22,16), 166 (unsigned int)extract_bits(msg3, 19,3), 167 (unsigned int)extract_bits(msg3, 18,1), 168 (unsigned int)extract_bits(msg3, 0, 16)); 169 170 printf("p3: desfbid %d tlen %d arc4 %x hmacpad %d\n", 171 (unsigned int)extract_bits(msg4, 48,16), 172 (unsigned int)extract_bits(msg4,11,16), 173 (unsigned int)extract_bits(msg4,6,3), 174 (unsigned int)extract_bits(msg4,5,1)); 175 176 printf("p4: sflen %d sddr %llx \n", 177 (unsigned int)extract_bits(msg5, 48, 16),extract_bits(msg5, 0, 40)); 178 179 printf("p5: dflen %d cl3 %d cclob %d cdest %llx \n", 180 (unsigned int)extract_bits(msg6, 48, 16), 181 (unsigned int)extract_bits(msg6, 46, 1), 182 (unsigned int)extract_bits(msg6, 41, 1), extract_bits(msg6, 0, 40)); 183 184 printf("fmn0: fbid %d dfrlen %d dfrv %d cklen %d cdescaddr %llx\n", 185 (unsigned int)extract_bits(msg7, 48, 16), 186 (unsigned int)extract_bits(msg7,46,2), 187 (unsigned int)extract_bits(msg7,45,1), 188 (unsigned int)extract_bits(msg7,40,5), 189 (extract_bits(msg7,0,34)<< 6)); 190 191 printf("fmn1: arc4 %d hklen %d pdesclen %d pktdescad %llx\n", 192 (unsigned int)extract_bits(msg8, 63, 1), 193 (unsigned int)extract_bits(msg8,56,5), 194 (unsigned int)extract_bits(msg8,43,12), 195 (extract_bits(msg8,0,34) << 6)); 196 197 return; 198} 199 200void 201xlp_sec_print_data(struct cryptop *crp) 202{ 203 int i, key_len; 204 struct cryptodesc *crp_desc; 205 206 printf("session id = 0x%llx, crp_ilen = %d, crp_olen=%d \n", 207 crp->crp_sid, crp->crp_ilen, crp->crp_olen); 208 209 printf("crp_flags = 0x%x\n", crp->crp_flags); 210 211 printf("crp buf:\n"); 212 for (i = 0; i < crp->crp_ilen; i++) { 213 printf("%c ", crp->crp_buf[i]); 214 if (i % 10 == 0) 215 printf("\n"); 216 } 217 218 printf("\n"); 219 printf("****************** desc ****************\n"); 220 crp_desc = crp->crp_desc; 221 printf("crd_skip=%d, crd_len=%d, crd_flags=0x%x, crd_alg=%d\n", 222 crp_desc->crd_skip, crp_desc->crd_len, crp_desc->crd_flags, 223 crp_desc->crd_alg); 224 225 key_len = crp_desc->crd_klen / 8; 226 printf("key(%d) :\n", key_len); 227 for (i = 0; i < key_len; i++) 228 printf("%d", crp_desc->crd_key[i]); 229 printf("\n"); 230 231 printf(" IV : \n"); 232 for (i = 0; i < EALG_MAX_BLOCK_LEN; i++) 233 printf("%d", crp_desc->crd_iv[i]); 234 printf("\n"); 235 236 printf("crd_next=%p\n", crp_desc->crd_next); 237 return; 238} 239 240void 241print_cmd(struct xlp_sec_command *cmd) 242{ 243 printf("session_num :%d\n",cmd->session_num); 244 printf("crp :0x%x\n",(uint32_t)cmd->crp); 245 printf("enccrd :0x%x\n",(uint32_t)cmd->enccrd); 246 printf("maccrd :0x%x\n",(uint32_t)cmd->maccrd); 247 printf("ses :%d\n",(uint32_t)cmd->ses); 248 printf("ctrlp :0x%x\n",(uint32_t)cmd->ctrlp); 249 printf("paramp :0x%x\n",(uint32_t)cmd->paramp); 250 printf("hashdest :0x%x\n",(uint32_t)cmd->hashdest); 251 printf("hashsrc :%d\n",cmd->hashsrc); 252 printf("hmacpad :%d\n",cmd->hmacpad); 253 printf("hashoff :%d\n",cmd->hashoff); 254 printf("hashlen :%d\n",cmd->hashlen); 255 printf("cipheroff :%d\n",cmd->cipheroff); 256 printf("cipherlen :%d\n",cmd->cipherlen); 257 printf("ivoff :%d\n",cmd->ivoff); 258 printf("ivlen :%d\n",cmd->ivlen); 259 printf("hashalg :%d\n",cmd->hashalg); 260 printf("hashmode :%d\n",cmd->hashmode); 261 printf("cipheralg :%d\n",cmd->cipheralg); 262 printf("ciphermode :%d\n",cmd->ciphermode); 263 printf("nsegs :%d\n",cmd->nsegs); 264 printf("hash_dst_len :%d\n",cmd->hash_dst_len); 265} 266#endif /* NLM_SEC_DEBUG */ 267 268static int 269xlp_sec_init(struct xlp_sec_softc *sc) 270{ 271 272 /* Register interrupt handler for the SEC CMS messages */ 273 if (register_msgring_handler(sc->sec_vc_start, 274 sc->sec_vc_end, nlm_xlpsec_msgring_handler, sc) != 0) { 275 printf("Couldn't register sec msgring handler\n"); 276 return (-1); 277 } 278 279 /* Do the CMS credit initialization */ 280 /* Currently it is configured by default to 50 when kernel comes up */ 281 282 return (0); 283} 284 285/* This function is called from an interrupt handler */ 286void 287nlm_xlpsec_msgring_handler(int vc, int size, int code, int src_id, 288 struct nlm_fmn_msg *msg, void *data) 289{ 290 struct xlp_sec_command *cmd = NULL; 291 struct xlp_sec_softc *sc = NULL; 292 struct cryptodesc *crd = NULL; 293 unsigned int ivlen = 0; 294 295 KASSERT(code == FMN_SWCODE_CRYPTO, 296 ("%s: bad code = %d, expected code = %d\n", __FUNCTION__, 297 code, FMN_SWCODE_CRYPTO)); 298 299 sc = (struct xlp_sec_softc *)data; 300 KASSERT(src_id >= sc->sec_vc_start && src_id <= sc->sec_vc_end, 301 ("%s: bad src_id = %d, expect %d - %d\n", __FUNCTION__, 302 src_id, sc->sec_vc_start, sc->sec_vc_end)); 303 304 cmd = (struct xlp_sec_command *)(uintptr_t)msg->msg[0]; 305 KASSERT(cmd != NULL && cmd->crp != NULL, 306 ("%s :cmd not received properly\n",__FUNCTION__)); 307 308 KASSERT(CRYPTO_ERROR(msg->msg[1]) == 0, 309 ("%s: Message rcv msg0 %llx msg1 %llx err %x \n", __FUNCTION__, 310 (unsigned long long)msg->msg[0], (unsigned long long)msg->msg[1], 311 (int)CRYPTO_ERROR(msg->msg[1]))); 312 313 crd = cmd->enccrd; 314 /* Copy the last 8 or 16 bytes to the session iv, so that in few 315 * cases this will be used as IV for the next request 316 */ 317 if (crd != NULL) { 318 if ((crd->crd_alg == CRYPTO_DES_CBC || 319 crd->crd_alg == CRYPTO_3DES_CBC || 320 crd->crd_alg == CRYPTO_AES_CBC) && 321 (crd->crd_flags & CRD_F_ENCRYPT)) { 322 ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ? 323 XLP_SEC_AES_IV_LENGTH : XLP_SEC_DES_IV_LENGTH); 324 crypto_copydata(cmd->crp->crp_flags, cmd->crp->crp_buf, 325 crd->crd_skip + crd->crd_len - ivlen, ivlen, 326 sc->sc_sessions[cmd->session_num].ses_iv); 327 } 328 } 329 330 /* If there are not enough credits to send, then send request 331 * will fail with ERESTART and the driver will be blocked until it is 332 * unblocked here after knowing that there are sufficient credits to 333 * send the request again. 334 */ 335 if (sc->sc_needwakeup) { 336 atomic_add_int(&creditleft, sc->sec_msgsz); 337 if (creditleft >= (NLM_CRYPTO_LEFT_REQS)) { 338 crypto_unblock(sc->sc_cid, sc->sc_needwakeup); 339 sc->sc_needwakeup &= (~(CRYPTO_SYMQ | CRYPTO_ASYMQ)); 340 } 341 } 342 if(cmd->maccrd) { 343 crypto_copyback(cmd->crp->crp_flags, 344 cmd->crp->crp_buf, cmd->maccrd->crd_inject, 345 cmd->hash_dst_len, cmd->hashdest); 346 } 347 348 /* This indicates completion of the crypto operation */ 349 crypto_done(cmd->crp); 350 351 xlp_free_cmd_params(cmd); 352 353 return; 354} 355 356static int 357xlp_sec_probe(device_t dev) 358{ 359 struct xlp_sec_softc *sc; 360 361 if (pci_get_vendor(dev) == PCI_VENDOR_NETLOGIC && 362 pci_get_device(dev) == PCI_DEVICE_ID_NLM_SAE) { 363 sc = device_get_softc(dev); 364 return (BUS_PROBE_DEFAULT); 365 } 366 return (ENXIO); 367} 368 369/* 370 * Attach an interface that successfully probed. 371 */ 372static int 373xlp_sec_attach(device_t dev) 374{ 375 struct xlp_sec_softc *sc = device_get_softc(dev); 376 uint64_t base; 377 int qstart, qnum; 378 int freq, node; 379 380 sc->sc_dev = dev; 381 382 node = nlm_get_device_node(pci_get_slot(dev)); 383 freq = nlm_set_device_frequency(node, DFS_DEVICE_SAE, 250); 384 if (bootverbose) 385 device_printf(dev, "SAE Freq: %dMHz\n", freq); 386 if(pci_get_device(dev) == PCI_DEVICE_ID_NLM_SAE) { 387 device_set_desc(dev, "XLP Security Accelerator"); 388 sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); 389 if (sc->sc_cid < 0) { 390 printf("xlp_sec - error : could not get the driver" 391 " id\n"); 392 goto error_exit; 393 } 394 if (crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0) != 0) 395 printf("register failed for CRYPTO_DES_CBC\n"); 396 397 if (crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0) != 0) 398 printf("register failed for CRYPTO_3DES_CBC\n"); 399 400 if (crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0) != 0) 401 printf("register failed for CRYPTO_AES_CBC\n"); 402 403 if (crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0) != 0) 404 printf("register failed for CRYPTO_ARC4\n"); 405 406 if (crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0) != 0) 407 printf("register failed for CRYPTO_MD5\n"); 408 409 if (crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0) != 0) 410 printf("register failed for CRYPTO_SHA1\n"); 411 412 if (crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0) != 0) 413 printf("register failed for CRYPTO_MD5_HMAC\n"); 414 415 if (crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0) != 0) 416 printf("register failed for CRYPTO_SHA1_HMAC\n"); 417 418 base = nlm_get_sec_pcibase(node); 419 qstart = nlm_qidstart(base); 420 qnum = nlm_qnum(base); 421 sc->sec_vc_start = qstart; 422 sc->sec_vc_end = qstart + qnum - 1; 423 } 424 425 if (xlp_sec_init(sc) != 0) 426 goto error_exit; 427 if (bootverbose) 428 device_printf(dev, "SEC Initialization complete!\n"); 429 return (0); 430 431error_exit: 432 return (ENXIO); 433 434} 435 436/* 437 * Detach an interface that successfully probed. 438 */ 439static int 440xlp_sec_detach(device_t dev) 441{ 442 return (0); 443} 444 445/* 446 * Allocate a new 'session' and return an encoded session id. 'sidp' 447 * contains our registration id, and should contain an encoded session 448 * id on successful allocation. 449 */ 450static int 451xlp_sec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri) 452{ 453 struct cryptoini *c; 454 struct xlp_sec_softc *sc = device_get_softc(dev); 455 int mac = 0, cry = 0, sesn; 456 struct xlp_sec_session *ses = NULL; 457 struct xlp_sec_command *cmd = NULL; 458 459 if (sidp == NULL || cri == NULL || sc == NULL) 460 return (EINVAL); 461 462 if (sc->sc_sessions == NULL) { 463 ses = sc->sc_sessions = malloc(sizeof(struct xlp_sec_session), 464 M_DEVBUF, M_NOWAIT); 465 if (ses == NULL) 466 return (ENOMEM); 467 sesn = 0; 468 sc->sc_nsessions = 1; 469 } else { 470 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 471 if (!sc->sc_sessions[sesn].hs_used) { 472 ses = &sc->sc_sessions[sesn]; 473 break; 474 } 475 } 476 477 if (ses == NULL) { 478 sesn = sc->sc_nsessions; 479 ses = malloc((sesn + 1)*sizeof(struct xlp_sec_session), 480 M_DEVBUF, M_NOWAIT); 481 if (ses == NULL) 482 return (ENOMEM); 483 bcopy(sc->sc_sessions, ses, sesn * sizeof(*ses)); 484 bzero(sc->sc_sessions, sesn * sizeof(*ses)); 485 free(sc->sc_sessions, M_DEVBUF); 486 sc->sc_sessions = ses; 487 ses = &sc->sc_sessions[sesn]; 488 sc->sc_nsessions++; 489 } 490 } 491 bzero(ses, sizeof(*ses)); 492 ses->sessionid = sesn; 493 cmd = &ses->cmd; 494 ses->hs_used = 1; 495 496 for (c = cri; c != NULL; c = c->cri_next) { 497 switch (c->cri_alg) { 498 case CRYPTO_MD5: 499 case CRYPTO_SHA1: 500 case CRYPTO_MD5_HMAC: 501 case CRYPTO_SHA1_HMAC: 502 if (mac) 503 return (EINVAL); 504 mac = 1; 505 ses->hs_mlen = c->cri_mlen; 506 if (ses->hs_mlen == 0) { 507 switch (c->cri_alg) { 508 case CRYPTO_MD5: 509 case CRYPTO_MD5_HMAC: 510 ses->hs_mlen = 16; 511 break; 512 case CRYPTO_SHA1: 513 case CRYPTO_SHA1_HMAC: 514 ses->hs_mlen = 20; 515 break; 516 } 517 } 518 break; 519 case CRYPTO_DES_CBC: 520 case CRYPTO_3DES_CBC: 521 case CRYPTO_AES_CBC: 522 /* XXX this may read fewer, does it matter? */ 523 read_random(ses->ses_iv, c->cri_alg == 524 CRYPTO_AES_CBC ? XLP_SEC_AES_IV_LENGTH : 525 XLP_SEC_DES_IV_LENGTH); 526 /* FALLTHROUGH */ 527 case CRYPTO_ARC4: 528 if (cry) 529 return (EINVAL); 530 cry = 1; 531 break; 532 default: 533 return (EINVAL); 534 } 535 } 536 if (mac == 0 && cry == 0) 537 return (EINVAL); 538 539 cmd->hash_dst_len = ses->hs_mlen; 540 *sidp = XLP_SEC_SID(device_get_unit(sc->sc_dev), sesn); 541 return (0); 542} 543 544/* 545 * Deallocate a session. 546 * XXX this routine should run a zero'd mac/encrypt key into context ram. 547 * XXX to blow away any keys already stored there. 548 */ 549static int 550xlp_sec_freesession(device_t dev, u_int64_t tid) 551{ 552 struct xlp_sec_softc *sc = device_get_softc(dev); 553 int session; 554 u_int32_t sid = CRYPTO_SESID2LID(tid); 555 556 if (sc == NULL) 557 return (EINVAL); 558 559 session = XLP_SEC_SESSION(sid); 560 if (session >= sc->sc_nsessions) 561 return (EINVAL); 562 563 sc->sc_sessions[session].hs_used = 0; 564 return (0); 565} 566 567static int 568xlp_copyiv(struct xlp_sec_softc *sc, struct xlp_sec_command *cmd, 569 struct cryptodesc *enccrd) 570{ 571 unsigned int ivlen = 0; 572 int session; 573 struct cryptop *crp = NULL; 574 575 crp = cmd->crp; 576 session = cmd->session_num; 577 578 if (enccrd->crd_alg != CRYPTO_ARC4) { 579 ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ? 580 XLP_SEC_AES_IV_LENGTH : XLP_SEC_DES_IV_LENGTH); 581 if (enccrd->crd_flags & CRD_F_ENCRYPT) { 582 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) { 583 bcopy(enccrd->crd_iv, cmd->iv, ivlen); 584 } else { 585 bcopy(sc->sc_sessions[session].ses_iv, cmd->iv, 586 ivlen); 587 } 588 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { 589 crypto_copyback(crp->crp_flags, 590 crp->crp_buf, enccrd->crd_inject, 591 ivlen, cmd->iv); 592 } 593 } else { 594 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) { 595 bcopy(enccrd->crd_iv, cmd->iv, ivlen); 596 } else { 597 crypto_copydata(crp->crp_flags, crp->crp_buf, 598 enccrd->crd_inject, ivlen, cmd->iv); 599 } 600 } 601 } 602 return (0); 603} 604 605static int 606xlp_get_nsegs(struct cryptop *crp, unsigned int *nsegs) 607{ 608 609 if (crp->crp_flags & CRYPTO_F_IMBUF) { 610 struct mbuf *m = NULL; 611 612 m = (struct mbuf *)crp->crp_buf; 613 while (m != NULL) { 614 *nsegs += NLM_CRYPTO_NUM_SEGS_REQD(m->m_len); 615 m = m->m_next; 616 } 617 } else if (crp->crp_flags & CRYPTO_F_IOV) { 618 struct uio *uio = NULL; 619 struct iovec *iov = NULL; 620 int iol = 0; 621 622 uio = (struct uio *)crp->crp_buf; 623 iov = (struct iovec *)uio->uio_iov; 624 iol = uio->uio_iovcnt; 625 while (iol > 0) { 626 *nsegs += NLM_CRYPTO_NUM_SEGS_REQD(iov->iov_len); 627 iol--; 628 iov++; 629 } 630 } else { 631 *nsegs = NLM_CRYPTO_NUM_SEGS_REQD(crp->crp_ilen); 632 } 633 return (0); 634} 635 636static int 637xlp_alloc_cmd_params(struct xlp_sec_command *cmd, unsigned int nsegs) 638{ 639 int err = 0; 640 641 if(cmd == NULL) { 642 err = EINVAL; 643 goto error; 644 } 645 if ((cmd->ctrlp = malloc(sizeof(struct nlm_crypto_pkt_ctrl), M_DEVBUF, 646 M_NOWAIT | M_ZERO)) == NULL) { 647 err = ENOMEM; 648 goto error; 649 } 650 if (((uintptr_t)cmd->ctrlp & (XLP_L2L3_CACHELINE_SIZE - 1))) { 651 err = EINVAL; 652 goto error; 653 } 654 /* (nsegs - 1) because one seg is part of the structure already */ 655 if ((cmd->paramp = malloc(sizeof(struct nlm_crypto_pkt_param) + 656 (16 * (nsegs - 1)), M_DEVBUF, M_NOWAIT | M_ZERO)) == NULL) { 657 err = ENOMEM; 658 goto error; 659 } 660 if (((uintptr_t)cmd->paramp & (XLP_L2L3_CACHELINE_SIZE - 1))) { 661 err = EINVAL; 662 goto error; 663 } 664 if ((cmd->iv = malloc(EALG_MAX_BLOCK_LEN, M_DEVBUF, 665 M_NOWAIT | M_ZERO)) == NULL) { 666 err = ENOMEM; 667 goto error; 668 } 669 if ((cmd->hashdest = malloc(HASH_MAX_LEN, M_DEVBUF, 670 M_NOWAIT | M_ZERO)) == NULL) { 671 err = ENOMEM; 672 goto error; 673 } 674error: 675 return (err); 676} 677 678static void 679xlp_free_cmd_params(struct xlp_sec_command *cmd) 680{ 681 if (cmd->ctrlp != NULL) 682 free(cmd->ctrlp, M_DEVBUF); 683 if (cmd->paramp != NULL) 684 free(cmd->paramp, M_DEVBUF); 685 if (cmd->iv != NULL) 686 free(cmd->iv, M_DEVBUF); 687 if (cmd->hashdest != NULL) 688 free(cmd->hashdest, M_DEVBUF); 689 if (cmd != NULL) 690 free(cmd, M_DEVBUF); 691 return; 692} 693 694static int 695xlp_sec_process(device_t dev, struct cryptop *crp, int hint) 696{ 697 struct xlp_sec_softc *sc = device_get_softc(dev); 698 struct xlp_sec_command *cmd = NULL; 699 int session, err = -1, ret = 0; 700 struct cryptodesc *crd1, *crd2; 701 struct xlp_sec_session *ses; 702 unsigned int nsegs = 0; 703 704 if (crp == NULL || crp->crp_callback == NULL) { 705 return (EINVAL); 706 } 707 session = XLP_SEC_SESSION(crp->crp_sid); 708 if (sc == NULL || session >= sc->sc_nsessions) { 709 err = EINVAL; 710 goto errout; 711 } 712 ses = &sc->sc_sessions[session]; 713 714 if ((cmd = malloc(sizeof(struct xlp_sec_command), M_DEVBUF, 715 M_NOWAIT | M_ZERO)) == NULL) { 716 err = ENOMEM; 717 goto errout; 718 } 719 720 cmd->crp = crp; 721 cmd->session_num = session; 722 cmd->hash_dst_len = ses->hs_mlen; 723 724 if ((crd1 = crp->crp_desc) == NULL) { 725 err = EINVAL; 726 goto errout; 727 } 728 crd2 = crd1->crd_next; 729 730 if ((ret = xlp_get_nsegs(crp, &nsegs)) != 0) { 731 err = EINVAL; 732 goto errout; 733 } 734 if (((crd1 != NULL) && (crd1->crd_flags & CRD_F_IV_EXPLICIT)) || 735 ((crd2 != NULL) && (crd2->crd_flags & CRD_F_IV_EXPLICIT))) { 736 /* Since IV is given as separate segment to avoid copy */ 737 nsegs += 1; 738 } 739 cmd->nsegs = nsegs; 740 741 if ((err = xlp_alloc_cmd_params(cmd, nsegs)) != 0) 742 goto errout; 743 744 if ((crd1 != NULL) && (crd2 == NULL)) { 745 if (crd1->crd_alg == CRYPTO_DES_CBC || 746 crd1->crd_alg == CRYPTO_3DES_CBC || 747 crd1->crd_alg == CRYPTO_AES_CBC || 748 crd1->crd_alg == CRYPTO_ARC4) { 749 cmd->enccrd = crd1; 750 cmd->maccrd = NULL; 751 if ((ret = nlm_get_cipher_param(cmd)) != 0) { 752 err = EINVAL; 753 goto errout; 754 } 755 if (crd1->crd_flags & CRD_F_IV_EXPLICIT) 756 cmd->cipheroff = cmd->ivlen; 757 else 758 cmd->cipheroff = cmd->enccrd->crd_skip; 759 cmd->cipherlen = cmd->enccrd->crd_len; 760 if (crd1->crd_flags & CRD_F_IV_PRESENT) 761 cmd->ivoff = 0; 762 else 763 cmd->ivoff = cmd->enccrd->crd_inject; 764 if ((err = xlp_copyiv(sc, cmd, cmd->enccrd)) != 0) 765 goto errout; 766 if ((err = nlm_crypto_do_cipher(sc, cmd)) != 0) 767 goto errout; 768 } else if (crd1->crd_alg == CRYPTO_MD5_HMAC || 769 crd1->crd_alg == CRYPTO_SHA1_HMAC || 770 crd1->crd_alg == CRYPTO_SHA1 || 771 crd1->crd_alg == CRYPTO_MD5) { 772 cmd->enccrd = NULL; 773 cmd->maccrd = crd1; 774 if ((ret = nlm_get_digest_param(cmd)) != 0) { 775 err = EINVAL; 776 goto errout; 777 } 778 cmd->hashoff = cmd->maccrd->crd_skip; 779 cmd->hashlen = cmd->maccrd->crd_len; 780 cmd->hmacpad = 0; 781 cmd->hashsrc = 0; 782 if ((err = nlm_crypto_do_digest(sc, cmd)) != 0) 783 goto errout; 784 } else { 785 err = EINVAL; 786 goto errout; 787 } 788 } else if( (crd1 != NULL) && (crd2 != NULL) ) { 789 if ((crd1->crd_alg == CRYPTO_MD5_HMAC || 790 crd1->crd_alg == CRYPTO_SHA1_HMAC || 791 crd1->crd_alg == CRYPTO_MD5 || 792 crd1->crd_alg == CRYPTO_SHA1) && 793 (crd2->crd_alg == CRYPTO_DES_CBC || 794 crd2->crd_alg == CRYPTO_3DES_CBC || 795 crd2->crd_alg == CRYPTO_AES_CBC || 796 crd2->crd_alg == CRYPTO_ARC4)) { 797 cmd->maccrd = crd1; 798 cmd->enccrd = crd2; 799 } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 800 crd1->crd_alg == CRYPTO_ARC4 || 801 crd1->crd_alg == CRYPTO_3DES_CBC || 802 crd1->crd_alg == CRYPTO_AES_CBC) && 803 (crd2->crd_alg == CRYPTO_MD5_HMAC || 804 crd2->crd_alg == CRYPTO_SHA1_HMAC || 805 crd2->crd_alg == CRYPTO_MD5 || 806 crd2->crd_alg == CRYPTO_SHA1)) { 807 cmd->enccrd = crd1; 808 cmd->maccrd = crd2; 809 } else { 810 err = EINVAL; 811 goto errout; 812 } 813 if ((ret = nlm_get_cipher_param(cmd)) != 0) { 814 err = EINVAL; 815 goto errout; 816 } 817 if ((ret = nlm_get_digest_param(cmd)) != 0) { 818 err = EINVAL; 819 goto errout; 820 } 821 cmd->ivoff = cmd->enccrd->crd_inject; 822 cmd->hashoff = cmd->maccrd->crd_skip; 823 cmd->hashlen = cmd->maccrd->crd_len; 824 cmd->hmacpad = 0; 825 if (cmd->enccrd->crd_flags & CRD_F_ENCRYPT) 826 cmd->hashsrc = 1; 827 else 828 cmd->hashsrc = 0; 829 cmd->cipheroff = cmd->enccrd->crd_skip; 830 cmd->cipherlen = cmd->enccrd->crd_len; 831 if ((err = xlp_copyiv(sc, cmd, cmd->enccrd)) != 0) 832 goto errout; 833 if ((err = nlm_crypto_do_cipher_digest(sc, cmd)) != 0) 834 goto errout; 835 } else { 836 err = EINVAL; 837 goto errout; 838 } 839 return (0); 840errout: 841 xlp_free_cmd_params(cmd); 842 if (err == ERESTART) { 843 sc->sc_needwakeup |= CRYPTO_SYMQ; 844 creditleft = 0; 845 return (err); 846 } 847 crp->crp_etype = err; 848 crypto_done(crp); 849 return (err); 850} 851