sec.c revision 209908
11556Srgrimes/*- 2139969Simp * Copyright (C) 2008-2009 Semihalf, Piotr Ziecik 3139969Simp * All rights reserved. 41556Srgrimes * 51556Srgrimes * Redistribution and use in source and binary forms, with or without 61556Srgrimes * modification, are permitted provided that the following conditions 71556Srgrimes * are met: 81556Srgrimes * 1. Redistributions of source code must retain the above copyright 91556Srgrimes * notice, this list of conditions and the following disclaimer. 101556Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 111556Srgrimes * notice, this list of conditions and the following disclaimer in the 121556Srgrimes * documentation and/or other materials provided with the distribution. 131556Srgrimes * 141556Srgrimes * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 151556Srgrimes * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 161556Srgrimes * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 171556Srgrimes * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 181556Srgrimes * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 191556Srgrimes * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 201556Srgrimes * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 211556Srgrimes * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 221556Srgrimes * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 231556Srgrimes * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 241556Srgrimes */ 251556Srgrimes 261556Srgrimes/* 271556Srgrimes * Freescale integrated Security Engine (SEC) driver. Currently SEC 2.0 and 281556Srgrimes * 3.0 are supported. 291556Srgrimes */ 301556Srgrimes 311556Srgrimes#include <sys/cdefs.h> 321556Srgrimes__FBSDID("$FreeBSD: head/sys/dev/sec/sec.c 209908 2010-07-11 21:08:29Z raj $"); 331556Srgrimes 3417987Speter#include <sys/param.h> 3550471Speter#include <sys/systm.h> 361556Srgrimes#include <sys/bus.h> 3769029Skris#include <sys/endian.h> 3817987Speter#include <sys/kernel.h> 3917987Speter#include <sys/lock.h> 4017987Speter#include <sys/malloc.h> 4117987Speter#include <sys/mbuf.h> 427502Sphk#include <sys/module.h> 43235927Smarcel#include <sys/mutex.h> 44235927Smarcel#include <sys/random.h> 45235927Smarcel#include <sys/rman.h> 46235927Smarcel 47235927Smarcel#include <machine/bus.h> 48235927Smarcel#include <machine/resource.h> 491556Srgrimes 501556Srgrimes#include <opencrypto/cryptodev.h> 511556Srgrimes#include "cryptodev_if.h" 521556Srgrimes 531556Srgrimes#include <dev/ofw/ofw_bus_subr.h> 5490832Simp#include <dev/sec/sec.h> 551556Srgrimes 561556Srgrimesstatic int sec_probe(device_t dev); 571556Srgrimesstatic int sec_attach(device_t dev); 581556Srgrimesstatic int sec_detach(device_t dev); 5917987Speterstatic int sec_suspend(device_t dev); 60235927Smarcelstatic int sec_resume(device_t dev); 61223060Sjillesstatic int sec_shutdown(device_t dev); 621556Srgrimesstatic void sec_primary_intr(void *arg); 631556Srgrimesstatic void sec_secondary_intr(void *arg); 641556Srgrimesstatic int sec_setup_intr(struct sec_softc *sc, struct resource **ires, 65293118Sjilles void **ihand, int *irid, driver_intr_t handler, const char *iname); 661556Srgrimesstatic void sec_release_intr(struct sec_softc *sc, struct resource *ires, 67157413Sstefanf void *ihand, int irid, const char *iname); 68157413Sstefanfstatic int sec_controller_reset(struct sec_softc *sc); 69157413Sstefanfstatic int sec_channel_reset(struct sec_softc *sc, int channel, int full); 70293118Sjillesstatic int sec_init(struct sec_softc *sc); 71157413Sstefanfstatic int sec_alloc_dma_mem(struct sec_softc *sc, 72157413Sstefanf struct sec_dma_mem *dma_mem, bus_size_t size); 731556Srgrimesstatic int sec_desc_map_dma(struct sec_softc *sc, 74293118Sjilles struct sec_dma_mem *dma_mem, void *mem, bus_size_t size, int type, 751556Srgrimes struct sec_desc_map_info *sdmi); 76235927Smarcelstatic void sec_free_dma_mem(struct sec_dma_mem *dma_mem); 771556Srgrimesstatic void sec_enqueue(struct sec_softc *sc); 781556Srgrimesstatic int sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, 791556Srgrimes int channel); 801556Srgrimesstatic int sec_eu_channel(struct sec_softc *sc, int eu); 811556Srgrimesstatic int sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, 821556Srgrimes u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype); 831556Srgrimesstatic int sec_make_pointer_direct(struct sec_softc *sc, 841556Srgrimes struct sec_desc *desc, u_int n, bus_addr_t data, bus_size_t dsize); 851556Srgrimesstatic int sec_alloc_session(struct sec_softc *sc); 861556Srgrimesstatic int sec_newsession(device_t dev, u_int32_t *sidp, 87293118Sjilles struct cryptoini *cri); 881556Srgrimesstatic int sec_freesession(device_t dev, uint64_t tid); 89149015Sstefanfstatic int sec_process(device_t dev, struct cryptop *crp, int hint); 90293118Sjillesstatic int sec_split_cri(struct cryptoini *cri, struct cryptoini **enc, 91223060Sjilles struct cryptoini **mac); 92223060Sjillesstatic int sec_split_crp(struct cryptop *crp, struct cryptodesc **enc, 931556Srgrimes struct cryptodesc **mac); 94static int sec_build_common_ns_desc(struct sec_softc *sc, 95 struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp, 96 struct cryptodesc *enc, int buftype); 97static int sec_build_common_s_desc(struct sec_softc *sc, 98 struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp, 99 struct cryptodesc *enc, struct cryptodesc *mac, int buftype); 100 101static struct sec_session *sec_get_session(struct sec_softc *sc, u_int sid); 102static struct sec_desc *sec_find_desc(struct sec_softc *sc, bus_addr_t paddr); 103 104/* AESU */ 105static int sec_aesu_newsession(struct sec_softc *sc, 106 struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac); 107static int sec_aesu_make_desc(struct sec_softc *sc, 108 struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp, 109 int buftype); 110 111/* DEU */ 112static int sec_deu_newsession(struct sec_softc *sc, 113 struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac); 114static int sec_deu_make_desc(struct sec_softc *sc, 115 struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp, 116 int buftype); 117 118/* MDEU */ 119static int sec_mdeu_can_handle(u_int alg); 120static int sec_mdeu_config(struct cryptodesc *crd, 121 u_int *eu, u_int *mode, u_int *hashlen); 122static int sec_mdeu_newsession(struct sec_softc *sc, 123 struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac); 124static int sec_mdeu_make_desc(struct sec_softc *sc, 125 struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp, 126 int buftype); 127 128static device_method_t sec_methods[] = { 129 /* Device interface */ 130 DEVMETHOD(device_probe, sec_probe), 131 DEVMETHOD(device_attach, sec_attach), 132 DEVMETHOD(device_detach, sec_detach), 133 134 DEVMETHOD(device_suspend, sec_suspend), 135 DEVMETHOD(device_resume, sec_resume), 136 DEVMETHOD(device_shutdown, sec_shutdown), 137 138 /* Bus interface */ 139 DEVMETHOD(bus_print_child, bus_generic_print_child), 140 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 141 142 /* Crypto methods */ 143 DEVMETHOD(cryptodev_newsession, sec_newsession), 144 DEVMETHOD(cryptodev_freesession,sec_freesession), 145 DEVMETHOD(cryptodev_process, sec_process), 146 147 { 0, 0 } 148}; 149static driver_t sec_driver = { 150 "sec", 151 sec_methods, 152 sizeof(struct sec_softc), 153}; 154 155static devclass_t sec_devclass; 156DRIVER_MODULE(sec, simplebus, sec_driver, sec_devclass, 0, 0); 157MODULE_DEPEND(sec, crypto, 1, 1, 1); 158 159static struct sec_eu_methods sec_eus[] = { 160 { 161 sec_aesu_newsession, 162 sec_aesu_make_desc, 163 }, 164 { 165 sec_deu_newsession, 166 sec_deu_make_desc, 167 }, 168 { 169 sec_mdeu_newsession, 170 sec_mdeu_make_desc, 171 }, 172 { NULL, NULL } 173}; 174 175static inline void 176sec_sync_dma_mem(struct sec_dma_mem *dma_mem, bus_dmasync_op_t op) 177{ 178 179 /* Sync only if dma memory is valid */ 180 if (dma_mem->dma_vaddr != NULL) 181 bus_dmamap_sync(dma_mem->dma_tag, dma_mem->dma_map, op); 182} 183 184static inline void 185sec_free_session(struct sec_softc *sc, struct sec_session *ses) 186{ 187 188 SEC_LOCK(sc, sessions); 189 ses->ss_used = 0; 190 SEC_UNLOCK(sc, sessions); 191} 192 193static inline void * 194sec_get_pointer_data(struct sec_desc *desc, u_int n) 195{ 196 197 return (desc->sd_ptr_dmem[n].dma_vaddr); 198} 199 200static int 201sec_probe(device_t dev) 202{ 203 struct sec_softc *sc; 204 uint64_t id; 205 206 if (!ofw_bus_is_compatible(dev, "fsl,sec2.0")) 207 return (ENXIO); 208 209 sc = device_get_softc(dev); 210 211 sc->sc_rrid = 0; 212 sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid, 213 RF_ACTIVE); 214 215 if (sc->sc_rres == NULL) 216 return (ENXIO); 217 218 sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres); 219 sc->sc_bas.bst = rman_get_bustag(sc->sc_rres); 220 221 id = SEC_READ(sc, SEC_ID); 222 223 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); 224 225 switch (id) { 226 case SEC_20_ID: 227 device_set_desc(dev, "Freescale Security Engine 2.0"); 228 sc->sc_version = 2; 229 break; 230 case SEC_30_ID: 231 device_set_desc(dev, "Freescale Security Engine 3.0"); 232 sc->sc_version = 3; 233 break; 234 default: 235 device_printf(dev, "unknown SEC ID 0x%016llx!\n", id); 236 return (ENXIO); 237 } 238 239 return (0); 240} 241 242static int 243sec_attach(device_t dev) 244{ 245 struct sec_softc *sc; 246 struct sec_hw_lt *lt; 247 int error = 0; 248 int i; 249 250 sc = device_get_softc(dev); 251 sc->sc_dev = dev; 252 sc->sc_blocked = 0; 253 sc->sc_shutdown = 0; 254 255 sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); 256 if (sc->sc_cid < 0) { 257 device_printf(dev, "could not get crypto driver ID!\n"); 258 return (ENXIO); 259 } 260 261 /* Init locks */ 262 mtx_init(&sc->sc_controller_lock, device_get_nameunit(dev), 263 "SEC Controller lock", MTX_DEF); 264 mtx_init(&sc->sc_descriptors_lock, device_get_nameunit(dev), 265 "SEC Descriptors lock", MTX_DEF); 266 mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev), 267 "SEC Sessions lock", MTX_DEF); 268 269 /* Allocate I/O memory for SEC registers */ 270 sc->sc_rrid = 0; 271 sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid, 272 RF_ACTIVE); 273 274 if (sc->sc_rres == NULL) { 275 device_printf(dev, "could not allocate I/O memory!\n"); 276 goto fail1; 277 } 278 279 sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres); 280 sc->sc_bas.bst = rman_get_bustag(sc->sc_rres); 281 282 /* Setup interrupts */ 283 sc->sc_pri_irid = 0; 284 error = sec_setup_intr(sc, &sc->sc_pri_ires, &sc->sc_pri_ihand, 285 &sc->sc_pri_irid, sec_primary_intr, "primary"); 286 287 if (error) 288 goto fail2; 289 290 291 if (sc->sc_version == 3) { 292 sc->sc_sec_irid = 1; 293 error = sec_setup_intr(sc, &sc->sc_sec_ires, &sc->sc_sec_ihand, 294 &sc->sc_sec_irid, sec_secondary_intr, "secondary"); 295 296 if (error) 297 goto fail3; 298 } 299 300 /* Alloc DMA memory for descriptors and link tables */ 301 error = sec_alloc_dma_mem(sc, &(sc->sc_desc_dmem), 302 SEC_DESCRIPTORS * sizeof(struct sec_hw_desc)); 303 304 if (error) 305 goto fail4; 306 307 error = sec_alloc_dma_mem(sc, &(sc->sc_lt_dmem), 308 (SEC_LT_ENTRIES + 1) * sizeof(struct sec_hw_lt)); 309 310 if (error) 311 goto fail5; 312 313 /* Fill in descriptors and link tables */ 314 for (i = 0; i < SEC_DESCRIPTORS; i++) { 315 sc->sc_desc[i].sd_desc = 316 (struct sec_hw_desc*)(sc->sc_desc_dmem.dma_vaddr) + i; 317 sc->sc_desc[i].sd_desc_paddr = sc->sc_desc_dmem.dma_paddr + 318 (i * sizeof(struct sec_hw_desc)); 319 } 320 321 for (i = 0; i < SEC_LT_ENTRIES + 1; i++) { 322 sc->sc_lt[i].sl_lt = 323 (struct sec_hw_lt*)(sc->sc_lt_dmem.dma_vaddr) + i; 324 sc->sc_lt[i].sl_lt_paddr = sc->sc_lt_dmem.dma_paddr + 325 (i * sizeof(struct sec_hw_lt)); 326 } 327 328 /* Last entry in link table is used to create a circle */ 329 lt = sc->sc_lt[SEC_LT_ENTRIES].sl_lt; 330 lt->shl_length = 0; 331 lt->shl_r = 0; 332 lt->shl_n = 1; 333 lt->shl_ptr = sc->sc_lt[0].sl_lt_paddr; 334 335 /* Init descriptor and link table queues pointers */ 336 SEC_CNT_INIT(sc, sc_free_desc_get_cnt, SEC_DESCRIPTORS); 337 SEC_CNT_INIT(sc, sc_free_desc_put_cnt, SEC_DESCRIPTORS); 338 SEC_CNT_INIT(sc, sc_ready_desc_get_cnt, SEC_DESCRIPTORS); 339 SEC_CNT_INIT(sc, sc_ready_desc_put_cnt, SEC_DESCRIPTORS); 340 SEC_CNT_INIT(sc, sc_queued_desc_get_cnt, SEC_DESCRIPTORS); 341 SEC_CNT_INIT(sc, sc_queued_desc_put_cnt, SEC_DESCRIPTORS); 342 SEC_CNT_INIT(sc, sc_lt_alloc_cnt, SEC_LT_ENTRIES); 343 SEC_CNT_INIT(sc, sc_lt_free_cnt, SEC_LT_ENTRIES); 344 345 /* Create masks for fast checks */ 346 sc->sc_int_error_mask = 0; 347 for (i = 0; i < SEC_CHANNELS; i++) 348 sc->sc_int_error_mask |= (~0ULL & SEC_INT_CH_ERR(i)); 349 350 switch (sc->sc_version) { 351 case 2: 352 sc->sc_channel_idle_mask = 353 (SEC_CHAN_CSR2_FFLVL_M << SEC_CHAN_CSR2_FFLVL_S) | 354 (SEC_CHAN_CSR2_MSTATE_M << SEC_CHAN_CSR2_MSTATE_S) | 355 (SEC_CHAN_CSR2_PSTATE_M << SEC_CHAN_CSR2_PSTATE_S) | 356 (SEC_CHAN_CSR2_GSTATE_M << SEC_CHAN_CSR2_GSTATE_S); 357 break; 358 case 3: 359 sc->sc_channel_idle_mask = 360 (SEC_CHAN_CSR3_FFLVL_M << SEC_CHAN_CSR3_FFLVL_S) | 361 (SEC_CHAN_CSR3_MSTATE_M << SEC_CHAN_CSR3_MSTATE_S) | 362 (SEC_CHAN_CSR3_PSTATE_M << SEC_CHAN_CSR3_PSTATE_S) | 363 (SEC_CHAN_CSR3_GSTATE_M << SEC_CHAN_CSR3_GSTATE_S); 364 break; 365 } 366 367 /* Init hardware */ 368 error = sec_init(sc); 369 370 if (error) 371 goto fail6; 372 373 /* Register in OCF (AESU) */ 374 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); 375 376 /* Register in OCF (DEU) */ 377 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); 378 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); 379 380 /* Register in OCF (MDEU) */ 381 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0); 382 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 383 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0); 384 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 385 crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0); 386 if (sc->sc_version >= 3) { 387 crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0); 388 crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0); 389 } 390 391 return (0); 392 393fail6: 394 sec_free_dma_mem(&(sc->sc_lt_dmem)); 395fail5: 396 sec_free_dma_mem(&(sc->sc_desc_dmem)); 397fail4: 398 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 399 sc->sc_sec_irid, "secondary"); 400fail3: 401 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 402 sc->sc_pri_irid, "primary"); 403fail2: 404 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); 405fail1: 406 mtx_destroy(&sc->sc_controller_lock); 407 mtx_destroy(&sc->sc_descriptors_lock); 408 mtx_destroy(&sc->sc_sessions_lock); 409 410 return (ENXIO); 411} 412 413static int 414sec_detach(device_t dev) 415{ 416 struct sec_softc *sc = device_get_softc(dev); 417 int i, error, timeout = SEC_TIMEOUT; 418 419 /* Prepare driver to shutdown */ 420 SEC_LOCK(sc, descriptors); 421 sc->sc_shutdown = 1; 422 SEC_UNLOCK(sc, descriptors); 423 424 /* Wait until all queued processing finishes */ 425 while (1) { 426 SEC_LOCK(sc, descriptors); 427 i = SEC_READY_DESC_CNT(sc) + SEC_QUEUED_DESC_CNT(sc); 428 SEC_UNLOCK(sc, descriptors); 429 430 if (i == 0) 431 break; 432 433 if (timeout < 0) { 434 device_printf(dev, "queue flush timeout!\n"); 435 436 /* DMA can be still active - stop it */ 437 for (i = 0; i < SEC_CHANNELS; i++) 438 sec_channel_reset(sc, i, 1); 439 440 break; 441 } 442 443 timeout -= 1000; 444 DELAY(1000); 445 } 446 447 /* Disable interrupts */ 448 SEC_WRITE(sc, SEC_IER, 0); 449 450 /* Unregister from OCF */ 451 crypto_unregister_all(sc->sc_cid); 452 453 /* Free DMA memory */ 454 for (i = 0; i < SEC_DESCRIPTORS; i++) 455 SEC_DESC_FREE_POINTERS(&(sc->sc_desc[i])); 456 457 sec_free_dma_mem(&(sc->sc_lt_dmem)); 458 sec_free_dma_mem(&(sc->sc_desc_dmem)); 459 460 /* Release interrupts */ 461 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 462 sc->sc_pri_irid, "primary"); 463 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 464 sc->sc_sec_irid, "secondary"); 465 466 /* Release memory */ 467 if (sc->sc_rres) { 468 error = bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, 469 sc->sc_rres); 470 if (error) 471 device_printf(dev, "bus_release_resource() failed for" 472 " I/O memory, error %d\n", error); 473 474 sc->sc_rres = NULL; 475 } 476 477 mtx_destroy(&sc->sc_controller_lock); 478 mtx_destroy(&sc->sc_descriptors_lock); 479 mtx_destroy(&sc->sc_sessions_lock); 480 481 return (0); 482} 483 484static int 485sec_suspend(device_t dev) 486{ 487 488 return (0); 489} 490 491static int 492sec_resume(device_t dev) 493{ 494 495 return (0); 496} 497 498static int 499sec_shutdown(device_t dev) 500{ 501 502 return (0); 503} 504 505static int 506sec_setup_intr(struct sec_softc *sc, struct resource **ires, void **ihand, 507 int *irid, driver_intr_t handler, const char *iname) 508{ 509 int error; 510 511 (*ires) = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ, irid, 512 RF_ACTIVE); 513 514 if ((*ires) == NULL) { 515 device_printf(sc->sc_dev, "could not allocate %s IRQ\n", iname); 516 return (ENXIO); 517 } 518 519 error = bus_setup_intr(sc->sc_dev, *ires, INTR_MPSAFE | INTR_TYPE_NET, 520 NULL, handler, sc, ihand); 521 522 if (error) { 523 device_printf(sc->sc_dev, "failed to set up %s IRQ\n", iname); 524 if (bus_release_resource(sc->sc_dev, SYS_RES_IRQ, *irid, *ires)) 525 device_printf(sc->sc_dev, "could not release %s IRQ\n", 526 iname); 527 528 (*ires) = NULL; 529 return (error); 530 } 531 532 return (0); 533} 534 535static void 536sec_release_intr(struct sec_softc *sc, struct resource *ires, void *ihand, 537 int irid, const char *iname) 538{ 539 int error; 540 541 if (ires == NULL) 542 return; 543 544 error = bus_teardown_intr(sc->sc_dev, ires, ihand); 545 if (error) 546 device_printf(sc->sc_dev, "bus_teardown_intr() failed for %s" 547 " IRQ, error %d\n", iname, error); 548 549 error = bus_release_resource(sc->sc_dev, SYS_RES_IRQ, irid, ires); 550 if (error) 551 device_printf(sc->sc_dev, "bus_release_resource() failed for %s" 552 " IRQ, error %d\n", iname, error); 553} 554 555static void 556sec_primary_intr(void *arg) 557{ 558 struct sec_softc *sc = arg; 559 struct sec_desc *desc; 560 uint64_t isr; 561 int i, wakeup = 0; 562 563 SEC_LOCK(sc, controller); 564 565 /* Check for errors */ 566 isr = SEC_READ(sc, SEC_ISR); 567 if (isr & sc->sc_int_error_mask) { 568 /* Check each channel for error */ 569 for (i = 0; i < SEC_CHANNELS; i++) { 570 if ((isr & SEC_INT_CH_ERR(i)) == 0) 571 continue; 572 573 device_printf(sc->sc_dev, 574 "I/O error on channel %i!\n", i); 575 576 /* Find and mark problematic descriptor */ 577 desc = sec_find_desc(sc, SEC_READ(sc, 578 SEC_CHAN_CDPR(i))); 579 580 if (desc != NULL) 581 desc->sd_error = EIO; 582 583 /* Do partial channel reset */ 584 sec_channel_reset(sc, i, 0); 585 } 586 } 587 588 /* ACK interrupt */ 589 SEC_WRITE(sc, SEC_ICR, 0xFFFFFFFFFFFFFFFFULL); 590 591 SEC_UNLOCK(sc, controller); 592 SEC_LOCK(sc, descriptors); 593 594 /* Handle processed descriptors */ 595 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 596 597 while (SEC_QUEUED_DESC_CNT(sc) > 0) { 598 desc = SEC_GET_QUEUED_DESC(sc); 599 600 if (desc->sd_desc->shd_done != 0xFF && desc->sd_error == 0) { 601 SEC_PUT_BACK_QUEUED_DESC(sc); 602 break; 603 } 604 605 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_PREREAD | 606 BUS_DMASYNC_PREWRITE); 607 608 desc->sd_crp->crp_etype = desc->sd_error; 609 crypto_done(desc->sd_crp); 610 611 SEC_DESC_FREE_POINTERS(desc); 612 SEC_DESC_FREE_LT(sc, desc); 613 SEC_DESC_QUEUED2FREE(sc); 614 } 615 616 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 617 618 if (!sc->sc_shutdown) { 619 wakeup = sc->sc_blocked; 620 sc->sc_blocked = 0; 621 } 622 623 SEC_UNLOCK(sc, descriptors); 624 625 /* Enqueue ready descriptors in hardware */ 626 sec_enqueue(sc); 627 628 if (wakeup) 629 crypto_unblock(sc->sc_cid, wakeup); 630} 631 632static void 633sec_secondary_intr(void *arg) 634{ 635 struct sec_softc *sc = arg; 636 637 device_printf(sc->sc_dev, "spurious secondary interrupt!\n"); 638 sec_primary_intr(arg); 639} 640 641static int 642sec_controller_reset(struct sec_softc *sc) 643{ 644 int timeout = SEC_TIMEOUT; 645 646 /* Reset Controller */ 647 SEC_WRITE(sc, SEC_MCR, SEC_MCR_SWR); 648 649 while (SEC_READ(sc, SEC_MCR) & SEC_MCR_SWR) { 650 DELAY(1000); 651 timeout -= 1000; 652 653 if (timeout < 0) { 654 device_printf(sc->sc_dev, "timeout while waiting for " 655 "device reset!\n"); 656 return (ETIMEDOUT); 657 } 658 } 659 660 return (0); 661} 662 663static int 664sec_channel_reset(struct sec_softc *sc, int channel, int full) 665{ 666 int timeout = SEC_TIMEOUT; 667 uint64_t bit = (full) ? SEC_CHAN_CCR_R : SEC_CHAN_CCR_CON; 668 uint64_t reg; 669 670 /* Reset Channel */ 671 reg = SEC_READ(sc, SEC_CHAN_CCR(channel)); 672 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg | bit); 673 674 while (SEC_READ(sc, SEC_CHAN_CCR(channel)) & bit) { 675 DELAY(1000); 676 timeout -= 1000; 677 678 if (timeout < 0) { 679 device_printf(sc->sc_dev, "timeout while waiting for " 680 "channel reset!\n"); 681 return (ETIMEDOUT); 682 } 683 } 684 685 if (full) { 686 reg = SEC_CHAN_CCR_CDIE | SEC_CHAN_CCR_NT | SEC_CHAN_CCR_BS; 687 688 switch(sc->sc_version) { 689 case 2: 690 reg |= SEC_CHAN_CCR_CDWE; 691 break; 692 case 3: 693 reg |= SEC_CHAN_CCR_AWSE | SEC_CHAN_CCR_WGN; 694 break; 695 } 696 697 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg); 698 } 699 700 return (0); 701} 702 703static int 704sec_init(struct sec_softc *sc) 705{ 706 uint64_t reg; 707 int error, i; 708 709 /* Reset controller twice to clear all pending interrupts */ 710 error = sec_controller_reset(sc); 711 if (error) 712 return (error); 713 714 error = sec_controller_reset(sc); 715 if (error) 716 return (error); 717 718 /* Reset channels */ 719 for (i = 0; i < SEC_CHANNELS; i++) { 720 error = sec_channel_reset(sc, i, 1); 721 if (error) 722 return (error); 723 } 724 725 /* Enable Interrupts */ 726 reg = SEC_INT_ITO; 727 for (i = 0; i < SEC_CHANNELS; i++) 728 reg |= SEC_INT_CH_DN(i) | SEC_INT_CH_ERR(i); 729 730 SEC_WRITE(sc, SEC_IER, reg); 731 732 return (error); 733} 734 735static void 736sec_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 737{ 738 struct sec_dma_mem *dma_mem = arg; 739 740 if (error) 741 return; 742 743 KASSERT(nseg == 1, ("Wrong number of segments, should be 1")); 744 dma_mem->dma_paddr = segs->ds_addr; 745} 746 747static void 748sec_dma_map_desc_cb(void *arg, bus_dma_segment_t *segs, int nseg, 749 int error) 750{ 751 struct sec_desc_map_info *sdmi = arg; 752 struct sec_softc *sc = sdmi->sdmi_sc; 753 struct sec_lt *lt = NULL; 754 bus_addr_t addr; 755 bus_size_t size; 756 int i; 757 758 SEC_LOCK_ASSERT(sc, descriptors); 759 760 if (error) 761 return; 762 763 for (i = 0; i < nseg; i++) { 764 addr = segs[i].ds_addr; 765 size = segs[i].ds_len; 766 767 /* Skip requested offset */ 768 if (sdmi->sdmi_offset >= size) { 769 sdmi->sdmi_offset -= size; 770 continue; 771 } 772 773 addr += sdmi->sdmi_offset; 774 size -= sdmi->sdmi_offset; 775 sdmi->sdmi_offset = 0; 776 777 /* Do not link more than requested */ 778 if (sdmi->sdmi_size < size) 779 size = sdmi->sdmi_size; 780 781 lt = SEC_ALLOC_LT_ENTRY(sc); 782 lt->sl_lt->shl_length = size; 783 lt->sl_lt->shl_r = 0; 784 lt->sl_lt->shl_n = 0; 785 lt->sl_lt->shl_ptr = addr; 786 787 if (sdmi->sdmi_lt_first == NULL) 788 sdmi->sdmi_lt_first = lt; 789 790 sdmi->sdmi_lt_used += 1; 791 792 if ((sdmi->sdmi_size -= size) == 0) 793 break; 794 } 795 796 sdmi->sdmi_lt_last = lt; 797} 798 799static void 800sec_dma_map_desc_cb2(void *arg, bus_dma_segment_t *segs, int nseg, 801 bus_size_t size, int error) 802{ 803 804 sec_dma_map_desc_cb(arg, segs, nseg, error); 805} 806 807static int 808sec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem, 809 bus_size_t size) 810{ 811 int error; 812 813 if (dma_mem->dma_vaddr != NULL) 814 return (EBUSY); 815 816 error = bus_dma_tag_create(NULL, /* parent */ 817 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 818 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 819 BUS_SPACE_MAXADDR, /* highaddr */ 820 NULL, NULL, /* filtfunc, filtfuncarg */ 821 size, 1, /* maxsize, nsegments */ 822 size, 0, /* maxsegsz, flags */ 823 NULL, NULL, /* lockfunc, lockfuncarg */ 824 &(dma_mem->dma_tag)); /* dmat */ 825 826 if (error) { 827 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 828 " %i!\n", error); 829 goto err1; 830 } 831 832 error = bus_dmamem_alloc(dma_mem->dma_tag, &(dma_mem->dma_vaddr), 833 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &(dma_mem->dma_map)); 834 835 if (error) { 836 device_printf(sc->sc_dev, "failed to allocate DMA safe" 837 " memory, error %i!\n", error); 838 goto err2; 839 } 840 841 error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map, 842 dma_mem->dma_vaddr, size, sec_alloc_dma_mem_cb, dma_mem, 843 BUS_DMA_NOWAIT); 844 845 if (error) { 846 device_printf(sc->sc_dev, "cannot get address of the DMA" 847 " memory, error %i\n", error); 848 goto err3; 849 } 850 851 dma_mem->dma_is_map = 0; 852 return (0); 853 854err3: 855 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, dma_mem->dma_map); 856err2: 857 bus_dma_tag_destroy(dma_mem->dma_tag); 858err1: 859 dma_mem->dma_vaddr = NULL; 860 return(error); 861} 862 863static int 864sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, void *mem, 865 bus_size_t size, int type, struct sec_desc_map_info *sdmi) 866{ 867 int error; 868 869 if (dma_mem->dma_vaddr != NULL) 870 return (EBUSY); 871 872 switch (type) { 873 case SEC_MEMORY: 874 break; 875 case SEC_UIO: 876 size = SEC_FREE_LT_CNT(sc) * SEC_MAX_DMA_BLOCK_SIZE; 877 break; 878 case SEC_MBUF: 879 size = m_length((struct mbuf*)mem, NULL); 880 break; 881 default: 882 return (EINVAL); 883 } 884 885 error = bus_dma_tag_create(NULL, /* parent */ 886 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 887 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 888 BUS_SPACE_MAXADDR, /* highaddr */ 889 NULL, NULL, /* filtfunc, filtfuncarg */ 890 size, /* maxsize */ 891 SEC_FREE_LT_CNT(sc), /* nsegments */ 892 SEC_MAX_DMA_BLOCK_SIZE, 0, /* maxsegsz, flags */ 893 NULL, NULL, /* lockfunc, lockfuncarg */ 894 &(dma_mem->dma_tag)); /* dmat */ 895 896 if (error) { 897 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 898 " %i!\n", error); 899 dma_mem->dma_vaddr = NULL; 900 return (error); 901 } 902 903 error = bus_dmamap_create(dma_mem->dma_tag, 0, &(dma_mem->dma_map)); 904 905 if (error) { 906 device_printf(sc->sc_dev, "failed to create DMA map, error %i!" 907 "\n", error); 908 bus_dma_tag_destroy(dma_mem->dma_tag); 909 return (error); 910 } 911 912 switch (type) { 913 case SEC_MEMORY: 914 error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map, 915 mem, size, sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT); 916 break; 917 case SEC_UIO: 918 error = bus_dmamap_load_uio(dma_mem->dma_tag, dma_mem->dma_map, 919 mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT); 920 break; 921 case SEC_MBUF: 922 error = bus_dmamap_load_mbuf(dma_mem->dma_tag, dma_mem->dma_map, 923 mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT); 924 break; 925 } 926 927 if (error) { 928 device_printf(sc->sc_dev, "cannot get address of the DMA" 929 " memory, error %i!\n", error); 930 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 931 bus_dma_tag_destroy(dma_mem->dma_tag); 932 return (error); 933 } 934 935 dma_mem->dma_is_map = 1; 936 dma_mem->dma_vaddr = mem; 937 938 return (0); 939} 940 941static void 942sec_free_dma_mem(struct sec_dma_mem *dma_mem) 943{ 944 945 /* Check for double free */ 946 if (dma_mem->dma_vaddr == NULL) 947 return; 948 949 bus_dmamap_unload(dma_mem->dma_tag, dma_mem->dma_map); 950 951 if (dma_mem->dma_is_map) 952 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 953 else 954 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, 955 dma_mem->dma_map); 956 957 bus_dma_tag_destroy(dma_mem->dma_tag); 958 dma_mem->dma_vaddr = NULL; 959} 960 961static int 962sec_eu_channel(struct sec_softc *sc, int eu) 963{ 964 uint64_t reg; 965 int channel = 0; 966 967 SEC_LOCK_ASSERT(sc, controller); 968 969 reg = SEC_READ(sc, SEC_EUASR); 970 971 switch (eu) { 972 case SEC_EU_AFEU: 973 channel = SEC_EUASR_AFEU(reg); 974 break; 975 case SEC_EU_DEU: 976 channel = SEC_EUASR_DEU(reg); 977 break; 978 case SEC_EU_MDEU_A: 979 case SEC_EU_MDEU_B: 980 channel = SEC_EUASR_MDEU(reg); 981 break; 982 case SEC_EU_RNGU: 983 channel = SEC_EUASR_RNGU(reg); 984 break; 985 case SEC_EU_PKEU: 986 channel = SEC_EUASR_PKEU(reg); 987 break; 988 case SEC_EU_AESU: 989 channel = SEC_EUASR_AESU(reg); 990 break; 991 case SEC_EU_KEU: 992 channel = SEC_EUASR_KEU(reg); 993 break; 994 case SEC_EU_CRCU: 995 channel = SEC_EUASR_CRCU(reg); 996 break; 997 } 998 999 return (channel - 1); 1000} 1001 1002static int 1003sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, int channel) 1004{ 1005 u_int fflvl = SEC_MAX_FIFO_LEVEL; 1006 uint64_t reg; 1007 int i; 1008 1009 SEC_LOCK_ASSERT(sc, controller); 1010 1011 /* Find free channel if have not got one */ 1012 if (channel < 0) { 1013 for (i = 0; i < SEC_CHANNELS; i++) { 1014 reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 1015 1016 if ((reg & sc->sc_channel_idle_mask) == 0) { 1017 channel = i; 1018 break; 1019 } 1020 } 1021 } 1022 1023 /* There is no free channel */ 1024 if (channel < 0) 1025 return (-1); 1026 1027 /* Check FIFO level on selected channel */ 1028 reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 1029 1030 switch(sc->sc_version) { 1031 case 2: 1032 fflvl = (reg >> SEC_CHAN_CSR2_FFLVL_S) & SEC_CHAN_CSR2_FFLVL_M; 1033 break; 1034 case 3: 1035 fflvl = (reg >> SEC_CHAN_CSR3_FFLVL_S) & SEC_CHAN_CSR3_FFLVL_M; 1036 break; 1037 } 1038 1039 if (fflvl >= SEC_MAX_FIFO_LEVEL) 1040 return (-1); 1041 1042 /* Enqueue descriptor in channel */ 1043 SEC_WRITE(sc, SEC_CHAN_FF(channel), desc->sd_desc_paddr); 1044 1045 return (channel); 1046} 1047 1048static void 1049sec_enqueue(struct sec_softc *sc) 1050{ 1051 struct sec_desc *desc; 1052 int ch0, ch1; 1053 1054 SEC_LOCK(sc, descriptors); 1055 SEC_LOCK(sc, controller); 1056 1057 while (SEC_READY_DESC_CNT(sc) > 0) { 1058 desc = SEC_GET_READY_DESC(sc); 1059 1060 ch0 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel0); 1061 ch1 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel1); 1062 1063 /* 1064 * Both EU are used by the same channel. 1065 * Enqueue descriptor in channel used by busy EUs. 1066 */ 1067 if (ch0 >= 0 && ch0 == ch1) { 1068 if (sec_enqueue_desc(sc, desc, ch0) >= 0) { 1069 SEC_DESC_READY2QUEUED(sc); 1070 continue; 1071 } 1072 } 1073 1074 /* 1075 * Only one EU is free. 1076 * Enqueue descriptor in channel used by busy EU. 1077 */ 1078 if ((ch0 >= 0 && ch1 < 0) || (ch1 >= 0 && ch0 < 0)) { 1079 if (sec_enqueue_desc(sc, desc, (ch0 >= 0) ? ch0 : ch1) 1080 >= 0) { 1081 SEC_DESC_READY2QUEUED(sc); 1082 continue; 1083 } 1084 } 1085 1086 /* 1087 * Both EU are free. 1088 * Enqueue descriptor in first free channel. 1089 */ 1090 if (ch0 < 0 && ch1 < 0) { 1091 if (sec_enqueue_desc(sc, desc, -1) >= 0) { 1092 SEC_DESC_READY2QUEUED(sc); 1093 continue; 1094 } 1095 } 1096 1097 /* Current descriptor can not be queued at the moment */ 1098 SEC_PUT_BACK_READY_DESC(sc); 1099 break; 1100 } 1101 1102 SEC_UNLOCK(sc, controller); 1103 SEC_UNLOCK(sc, descriptors); 1104} 1105 1106static struct sec_desc * 1107sec_find_desc(struct sec_softc *sc, bus_addr_t paddr) 1108{ 1109 struct sec_desc *desc = NULL; 1110 int i; 1111 1112 SEC_LOCK_ASSERT(sc, descriptors); 1113 1114 for (i = 0; i < SEC_CHANNELS; i++) { 1115 if (sc->sc_desc[i].sd_desc_paddr == paddr) { 1116 desc = &(sc->sc_desc[i]); 1117 break; 1118 } 1119 } 1120 1121 return (desc); 1122} 1123 1124static int 1125sec_make_pointer_direct(struct sec_softc *sc, struct sec_desc *desc, u_int n, 1126 bus_addr_t data, bus_size_t dsize) 1127{ 1128 struct sec_hw_desc_ptr *ptr; 1129 1130 SEC_LOCK_ASSERT(sc, descriptors); 1131 1132 ptr = &(desc->sd_desc->shd_pointer[n]); 1133 ptr->shdp_length = dsize; 1134 ptr->shdp_extent = 0; 1135 ptr->shdp_j = 0; 1136 ptr->shdp_ptr = data; 1137 1138 return (0); 1139} 1140 1141static int 1142sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, 1143 u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype) 1144{ 1145 struct sec_desc_map_info sdmi = { sc, dsize, doffset, NULL, NULL, 0 }; 1146 struct sec_hw_desc_ptr *ptr; 1147 int error; 1148 1149 SEC_LOCK_ASSERT(sc, descriptors); 1150 1151 /* For flat memory map only requested region */ 1152 if (dtype == SEC_MEMORY) { 1153 data = (uint8_t*)(data) + doffset; 1154 sdmi.sdmi_offset = 0; 1155 } 1156 1157 error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), data, dsize, 1158 dtype, &sdmi); 1159 1160 if (error) 1161 return (error); 1162 1163 sdmi.sdmi_lt_last->sl_lt->shl_r = 1; 1164 desc->sd_lt_used += sdmi.sdmi_lt_used; 1165 1166 ptr = &(desc->sd_desc->shd_pointer[n]); 1167 ptr->shdp_length = dsize; 1168 ptr->shdp_extent = 0; 1169 ptr->shdp_j = 1; 1170 ptr->shdp_ptr = sdmi.sdmi_lt_first->sl_lt_paddr; 1171 1172 return (0); 1173} 1174 1175static int 1176sec_split_cri(struct cryptoini *cri, struct cryptoini **enc, 1177 struct cryptoini **mac) 1178{ 1179 struct cryptoini *e, *m; 1180 1181 e = cri; 1182 m = cri->cri_next; 1183 1184 /* We can haldle only two operations */ 1185 if (m && m->cri_next) 1186 return (EINVAL); 1187 1188 if (sec_mdeu_can_handle(e->cri_alg)) { 1189 cri = m; 1190 m = e; 1191 e = cri; 1192 } 1193 1194 if (m && !sec_mdeu_can_handle(m->cri_alg)) 1195 return (EINVAL); 1196 1197 *enc = e; 1198 *mac = m; 1199 1200 return (0); 1201} 1202 1203static int 1204sec_split_crp(struct cryptop *crp, struct cryptodesc **enc, 1205 struct cryptodesc **mac) 1206{ 1207 struct cryptodesc *e, *m, *t; 1208 1209 e = crp->crp_desc; 1210 m = e->crd_next; 1211 1212 /* We can haldle only two operations */ 1213 if (m && m->crd_next) 1214 return (EINVAL); 1215 1216 if (sec_mdeu_can_handle(e->crd_alg)) { 1217 t = m; 1218 m = e; 1219 e = t; 1220 } 1221 1222 if (m && !sec_mdeu_can_handle(m->crd_alg)) 1223 return (EINVAL); 1224 1225 *enc = e; 1226 *mac = m; 1227 1228 return (0); 1229} 1230 1231static int 1232sec_alloc_session(struct sec_softc *sc) 1233{ 1234 struct sec_session *ses = NULL; 1235 int sid = -1; 1236 u_int i; 1237 1238 SEC_LOCK(sc, sessions); 1239 1240 for (i = 0; i < SEC_MAX_SESSIONS; i++) { 1241 if (sc->sc_sessions[i].ss_used == 0) { 1242 ses = &(sc->sc_sessions[i]); 1243 ses->ss_used = 1; 1244 ses->ss_ivlen = 0; 1245 ses->ss_klen = 0; 1246 ses->ss_mklen = 0; 1247 sid = i; 1248 break; 1249 } 1250 } 1251 1252 SEC_UNLOCK(sc, sessions); 1253 1254 return (sid); 1255} 1256 1257static struct sec_session * 1258sec_get_session(struct sec_softc *sc, u_int sid) 1259{ 1260 struct sec_session *ses; 1261 1262 if (sid >= SEC_MAX_SESSIONS) 1263 return (NULL); 1264 1265 SEC_LOCK(sc, sessions); 1266 1267 ses = &(sc->sc_sessions[sid]); 1268 1269 if (ses->ss_used == 0) 1270 ses = NULL; 1271 1272 SEC_UNLOCK(sc, sessions); 1273 1274 return (ses); 1275} 1276 1277static int 1278sec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri) 1279{ 1280 struct sec_softc *sc = device_get_softc(dev); 1281 struct sec_eu_methods *eu = sec_eus; 1282 struct cryptoini *enc = NULL; 1283 struct cryptoini *mac = NULL; 1284 struct sec_session *ses; 1285 int error = -1; 1286 int sid; 1287 1288 error = sec_split_cri(cri, &enc, &mac); 1289 if (error) 1290 return (error); 1291 1292 /* Check key lengths */ 1293 if (enc && enc->cri_key && (enc->cri_klen / 8) > SEC_MAX_KEY_LEN) 1294 return (E2BIG); 1295 1296 if (mac && mac->cri_key && (mac->cri_klen / 8) > SEC_MAX_KEY_LEN) 1297 return (E2BIG); 1298 1299 /* Only SEC 3.0 supports digests larger than 256 bits */ 1300 if (sc->sc_version < 3 && mac && mac->cri_klen > 256) 1301 return (E2BIG); 1302 1303 sid = sec_alloc_session(sc); 1304 if (sid < 0) 1305 return (ENOMEM); 1306 1307 ses = sec_get_session(sc, sid); 1308 1309 /* Find EU for this session */ 1310 while (eu->sem_make_desc != NULL) { 1311 error = eu->sem_newsession(sc, ses, enc, mac); 1312 if (error >= 0) 1313 break; 1314 1315 eu++; 1316 } 1317 1318 /* If not found, return EINVAL */ 1319 if (error < 0) { 1320 sec_free_session(sc, ses); 1321 return (EINVAL); 1322 } 1323 1324 /* Save cipher key */ 1325 if (enc && enc->cri_key) { 1326 ses->ss_klen = enc->cri_klen / 8; 1327 memcpy(ses->ss_key, enc->cri_key, ses->ss_klen); 1328 } 1329 1330 /* Save digest key */ 1331 if (mac && mac->cri_key) { 1332 ses->ss_mklen = mac->cri_klen / 8; 1333 memcpy(ses->ss_mkey, mac->cri_key, ses->ss_mklen); 1334 } 1335 1336 ses->ss_eu = eu; 1337 *sidp = sid; 1338 1339 return (0); 1340} 1341 1342static int 1343sec_freesession(device_t dev, uint64_t tid) 1344{ 1345 struct sec_softc *sc = device_get_softc(dev); 1346 struct sec_session *ses; 1347 int error = 0; 1348 1349 ses = sec_get_session(sc, CRYPTO_SESID2LID(tid)); 1350 if (ses == NULL) 1351 return (EINVAL); 1352 1353 sec_free_session(sc, ses); 1354 1355 return (error); 1356} 1357 1358static int 1359sec_process(device_t dev, struct cryptop *crp, int hint) 1360{ 1361 struct sec_softc *sc = device_get_softc(dev); 1362 struct sec_desc *desc = NULL; 1363 struct cryptodesc *mac, *enc; 1364 struct sec_session *ses; 1365 int buftype, error = 0; 1366 1367 /* Check Session ID */ 1368 ses = sec_get_session(sc, CRYPTO_SESID2LID(crp->crp_sid)); 1369 if (ses == NULL) { 1370 crp->crp_etype = EINVAL; 1371 crypto_done(crp); 1372 return (0); 1373 } 1374 1375 /* Check for input length */ 1376 if (crp->crp_ilen > SEC_MAX_DMA_BLOCK_SIZE) { 1377 crp->crp_etype = E2BIG; 1378 crypto_done(crp); 1379 return (0); 1380 } 1381 1382 /* Get descriptors */ 1383 if (sec_split_crp(crp, &enc, &mac)) { 1384 crp->crp_etype = EINVAL; 1385 crypto_done(crp); 1386 return (0); 1387 } 1388 1389 SEC_LOCK(sc, descriptors); 1390 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1391 1392 /* Block driver if there is no free descriptors or we are going down */ 1393 if (SEC_FREE_DESC_CNT(sc) == 0 || sc->sc_shutdown) { 1394 sc->sc_blocked |= CRYPTO_SYMQ; 1395 SEC_UNLOCK(sc, descriptors); 1396 return (ERESTART); 1397 } 1398 1399 /* Prepare descriptor */ 1400 desc = SEC_GET_FREE_DESC(sc); 1401 desc->sd_lt_used = 0; 1402 desc->sd_error = 0; 1403 desc->sd_crp = crp; 1404 1405 if (crp->crp_flags & CRYPTO_F_IOV) 1406 buftype = SEC_UIO; 1407 else if (crp->crp_flags & CRYPTO_F_IMBUF) 1408 buftype = SEC_MBUF; 1409 else 1410 buftype = SEC_MEMORY; 1411 1412 if (enc && enc->crd_flags & CRD_F_ENCRYPT) { 1413 if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1414 memcpy(desc->sd_desc->shd_iv, enc->crd_iv, 1415 ses->ss_ivlen); 1416 else 1417 arc4rand(desc->sd_desc->shd_iv, ses->ss_ivlen, 0); 1418 1419 if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0) 1420 crypto_copyback(crp->crp_flags, crp->crp_buf, 1421 enc->crd_inject, ses->ss_ivlen, 1422 desc->sd_desc->shd_iv); 1423 } else if (enc) { 1424 if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1425 memcpy(desc->sd_desc->shd_iv, enc->crd_iv, 1426 ses->ss_ivlen); 1427 else 1428 crypto_copydata(crp->crp_flags, crp->crp_buf, 1429 enc->crd_inject, ses->ss_ivlen, 1430 desc->sd_desc->shd_iv); 1431 } 1432 1433 if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) { 1434 if ((enc->crd_klen / 8) <= SEC_MAX_KEY_LEN) { 1435 ses->ss_klen = enc->crd_klen / 8; 1436 memcpy(ses->ss_key, enc->crd_key, ses->ss_klen); 1437 } else 1438 error = E2BIG; 1439 } 1440 1441 if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) { 1442 if ((mac->crd_klen / 8) <= SEC_MAX_KEY_LEN) { 1443 ses->ss_mklen = mac->crd_klen / 8; 1444 memcpy(ses->ss_mkey, mac->crd_key, ses->ss_mklen); 1445 } else 1446 error = E2BIG; 1447 } 1448 1449 if (!error) { 1450 memcpy(desc->sd_desc->shd_key, ses->ss_key, ses->ss_klen); 1451 memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, ses->ss_mklen); 1452 1453 error = ses->ss_eu->sem_make_desc(sc, ses, desc, crp, buftype); 1454 } 1455 1456 if (error) { 1457 SEC_DESC_FREE_POINTERS(desc); 1458 SEC_DESC_PUT_BACK_LT(sc, desc); 1459 SEC_PUT_BACK_FREE_DESC(sc); 1460 SEC_UNLOCK(sc, descriptors); 1461 crp->crp_etype = error; 1462 crypto_done(crp); 1463 return (0); 1464 } 1465 1466 /* 1467 * Skip DONE interrupt if this is not last request in burst, but only 1468 * if we are running on SEC 3.X. On SEC 2.X we have to enable DONE 1469 * signaling on each descriptor. 1470 */ 1471 if ((hint & CRYPTO_HINT_MORE) && sc->sc_version == 3) 1472 desc->sd_desc->shd_dn = 0; 1473 else 1474 desc->sd_desc->shd_dn = 1; 1475 1476 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1477 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_POSTREAD | 1478 BUS_DMASYNC_POSTWRITE); 1479 SEC_DESC_FREE2READY(sc); 1480 SEC_UNLOCK(sc, descriptors); 1481 1482 /* Enqueue ready descriptors in hardware */ 1483 sec_enqueue(sc); 1484 1485 return (0); 1486} 1487 1488static int 1489sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc, 1490 struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc, 1491 int buftype) 1492{ 1493 struct sec_hw_desc *hd = desc->sd_desc; 1494 int error; 1495 1496 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 1497 hd->shd_eu_sel1 = SEC_EU_NONE; 1498 hd->shd_mode1 = 0; 1499 1500 /* Pointer 0: NULL */ 1501 error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 1502 if (error) 1503 return (error); 1504 1505 /* Pointer 1: IV IN */ 1506 error = sec_make_pointer_direct(sc, desc, 1, desc->sd_desc_paddr + 1507 offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen); 1508 if (error) 1509 return (error); 1510 1511 /* Pointer 2: Cipher Key */ 1512 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 1513 offsetof(struct sec_hw_desc, shd_key), ses->ss_klen); 1514 if (error) 1515 return (error); 1516 1517 /* Pointer 3: Data IN */ 1518 error = sec_make_pointer(sc, desc, 3, crp->crp_buf, enc->crd_skip, 1519 enc->crd_len, buftype); 1520 if (error) 1521 return (error); 1522 1523 /* Pointer 4: Data OUT */ 1524 error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip, 1525 enc->crd_len, buftype); 1526 if (error) 1527 return (error); 1528 1529 /* Pointer 5: IV OUT (Not used: NULL) */ 1530 error = sec_make_pointer_direct(sc, desc, 5, 0, 0); 1531 if (error) 1532 return (error); 1533 1534 /* Pointer 6: NULL */ 1535 error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 1536 1537 return (error); 1538} 1539 1540static int 1541sec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc, 1542 struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc, 1543 struct cryptodesc *mac, int buftype) 1544{ 1545 struct sec_hw_desc *hd = desc->sd_desc; 1546 u_int eu, mode, hashlen; 1547 int error; 1548 1549 if (mac->crd_len < enc->crd_len) 1550 return (EINVAL); 1551 1552 if (mac->crd_skip + mac->crd_len != enc->crd_skip + enc->crd_len) 1553 return (EINVAL); 1554 1555 error = sec_mdeu_config(mac, &eu, &mode, &hashlen); 1556 if (error) 1557 return (error); 1558 1559 hd->shd_desc_type = SEC_DT_HMAC_SNOOP; 1560 hd->shd_eu_sel1 = eu; 1561 hd->shd_mode1 = mode; 1562 1563 /* Pointer 0: HMAC Key */ 1564 error = sec_make_pointer_direct(sc, desc, 0, desc->sd_desc_paddr + 1565 offsetof(struct sec_hw_desc, shd_mkey), ses->ss_mklen); 1566 if (error) 1567 return (error); 1568 1569 /* Pointer 1: HMAC-Only Data IN */ 1570 error = sec_make_pointer(sc, desc, 1, crp->crp_buf, mac->crd_skip, 1571 mac->crd_len - enc->crd_len, buftype); 1572 if (error) 1573 return (error); 1574 1575 /* Pointer 2: Cipher Key */ 1576 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 1577 offsetof(struct sec_hw_desc, shd_key), ses->ss_klen); 1578 if (error) 1579 return (error); 1580 1581 /* Pointer 3: IV IN */ 1582 error = sec_make_pointer_direct(sc, desc, 3, desc->sd_desc_paddr + 1583 offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen); 1584 if (error) 1585 return (error); 1586 1587 /* Pointer 4: Data IN */ 1588 error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip, 1589 enc->crd_len, buftype); 1590 if (error) 1591 return (error); 1592 1593 /* Pointer 5: Data OUT */ 1594 error = sec_make_pointer(sc, desc, 5, crp->crp_buf, enc->crd_skip, 1595 enc->crd_len, buftype); 1596 if (error) 1597 return (error); 1598 1599 /* Pointer 6: HMAC OUT */ 1600 error = sec_make_pointer(sc, desc, 6, crp->crp_buf, mac->crd_inject, 1601 hashlen, buftype); 1602 1603 return (error); 1604} 1605 1606/* AESU */ 1607 1608static int 1609sec_aesu_newsession(struct sec_softc *sc, struct sec_session *ses, 1610 struct cryptoini *enc, struct cryptoini *mac) 1611{ 1612 1613 if (enc == NULL) 1614 return (-1); 1615 1616 if (enc->cri_alg != CRYPTO_AES_CBC) 1617 return (-1); 1618 1619 ses->ss_ivlen = AES_BLOCK_LEN; 1620 1621 return (0); 1622} 1623 1624static int 1625sec_aesu_make_desc(struct sec_softc *sc, struct sec_session *ses, 1626 struct sec_desc *desc, struct cryptop *crp, int buftype) 1627{ 1628 struct sec_hw_desc *hd = desc->sd_desc; 1629 struct cryptodesc *enc, *mac; 1630 int error; 1631 1632 error = sec_split_crp(crp, &enc, &mac); 1633 if (error) 1634 return (error); 1635 1636 if (!enc) 1637 return (EINVAL); 1638 1639 hd->shd_eu_sel0 = SEC_EU_AESU; 1640 hd->shd_mode0 = SEC_AESU_MODE_CBC; 1641 1642 if (enc->crd_alg != CRYPTO_AES_CBC) 1643 return (EINVAL); 1644 1645 if (enc->crd_flags & CRD_F_ENCRYPT) { 1646 hd->shd_mode0 |= SEC_AESU_MODE_ED; 1647 hd->shd_dir = 0; 1648 } else 1649 hd->shd_dir = 1; 1650 1651 if (mac) 1652 error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac, 1653 buftype); 1654 else 1655 error = sec_build_common_ns_desc(sc, desc, ses, crp, enc, 1656 buftype); 1657 1658 return (error); 1659} 1660 1661/* DEU */ 1662 1663static int 1664sec_deu_newsession(struct sec_softc *sc, struct sec_session *ses, 1665 struct cryptoini *enc, struct cryptoini *mac) 1666{ 1667 1668 if (enc == NULL) 1669 return (-1); 1670 1671 switch (enc->cri_alg) { 1672 case CRYPTO_DES_CBC: 1673 case CRYPTO_3DES_CBC: 1674 break; 1675 default: 1676 return (-1); 1677 } 1678 1679 ses->ss_ivlen = DES_BLOCK_LEN; 1680 1681 return (0); 1682} 1683 1684static int 1685sec_deu_make_desc(struct sec_softc *sc, struct sec_session *ses, 1686 struct sec_desc *desc, struct cryptop *crp, int buftype) 1687{ 1688 struct sec_hw_desc *hd = desc->sd_desc; 1689 struct cryptodesc *enc, *mac; 1690 int error; 1691 1692 error = sec_split_crp(crp, &enc, &mac); 1693 if (error) 1694 return (error); 1695 1696 if (!enc) 1697 return (EINVAL); 1698 1699 hd->shd_eu_sel0 = SEC_EU_DEU; 1700 hd->shd_mode0 = SEC_DEU_MODE_CBC; 1701 1702 switch (enc->crd_alg) { 1703 case CRYPTO_3DES_CBC: 1704 hd->shd_mode0 |= SEC_DEU_MODE_TS; 1705 break; 1706 case CRYPTO_DES_CBC: 1707 break; 1708 default: 1709 return (EINVAL); 1710 } 1711 1712 if (enc->crd_flags & CRD_F_ENCRYPT) { 1713 hd->shd_mode0 |= SEC_DEU_MODE_ED; 1714 hd->shd_dir = 0; 1715 } else 1716 hd->shd_dir = 1; 1717 1718 if (mac) 1719 error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac, 1720 buftype); 1721 else 1722 error = sec_build_common_ns_desc(sc, desc, ses, crp, enc, 1723 buftype); 1724 1725 return (error); 1726} 1727 1728/* MDEU */ 1729 1730static int 1731sec_mdeu_can_handle(u_int alg) 1732{ 1733 switch (alg) { 1734 case CRYPTO_MD5: 1735 case CRYPTO_SHA1: 1736 case CRYPTO_MD5_HMAC: 1737 case CRYPTO_SHA1_HMAC: 1738 case CRYPTO_SHA2_256_HMAC: 1739 case CRYPTO_SHA2_384_HMAC: 1740 case CRYPTO_SHA2_512_HMAC: 1741 return (1); 1742 default: 1743 return (0); 1744 } 1745} 1746 1747static int 1748sec_mdeu_config(struct cryptodesc *crd, u_int *eu, u_int *mode, u_int *hashlen) 1749{ 1750 1751 *mode = SEC_MDEU_MODE_PD | SEC_MDEU_MODE_INIT; 1752 *eu = SEC_EU_NONE; 1753 1754 switch (crd->crd_alg) { 1755 case CRYPTO_MD5_HMAC: 1756 *mode |= SEC_MDEU_MODE_HMAC; 1757 /* FALLTHROUGH */ 1758 case CRYPTO_MD5: 1759 *eu = SEC_EU_MDEU_A; 1760 *mode |= SEC_MDEU_MODE_MD5; 1761 *hashlen = MD5_HASH_LEN; 1762 break; 1763 case CRYPTO_SHA1_HMAC: 1764 *mode |= SEC_MDEU_MODE_HMAC; 1765 /* FALLTHROUGH */ 1766 case CRYPTO_SHA1: 1767 *eu = SEC_EU_MDEU_A; 1768 *mode |= SEC_MDEU_MODE_SHA1; 1769 *hashlen = SHA1_HASH_LEN; 1770 break; 1771 case CRYPTO_SHA2_256_HMAC: 1772 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA256; 1773 *eu = SEC_EU_MDEU_A; 1774 break; 1775 case CRYPTO_SHA2_384_HMAC: 1776 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA384; 1777 *eu = SEC_EU_MDEU_B; 1778 break; 1779 case CRYPTO_SHA2_512_HMAC: 1780 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA512; 1781 *eu = SEC_EU_MDEU_B; 1782 break; 1783 default: 1784 return (EINVAL); 1785 } 1786 1787 if (*mode & SEC_MDEU_MODE_HMAC) 1788 *hashlen = SEC_HMAC_HASH_LEN; 1789 1790 return (0); 1791} 1792 1793static int 1794sec_mdeu_newsession(struct sec_softc *sc, struct sec_session *ses, 1795 struct cryptoini *enc, struct cryptoini *mac) 1796{ 1797 1798 if (mac && sec_mdeu_can_handle(mac->cri_alg)) 1799 return (0); 1800 1801 return (-1); 1802} 1803 1804static int 1805sec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses, 1806 struct sec_desc *desc, struct cryptop *crp, int buftype) 1807{ 1808 struct cryptodesc *enc, *mac; 1809 struct sec_hw_desc *hd = desc->sd_desc; 1810 u_int eu, mode, hashlen; 1811 int error; 1812 1813 error = sec_split_crp(crp, &enc, &mac); 1814 if (error) 1815 return (error); 1816 1817 if (enc) 1818 return (EINVAL); 1819 1820 error = sec_mdeu_config(mac, &eu, &mode, &hashlen); 1821 if (error) 1822 return (error); 1823 1824 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 1825 hd->shd_eu_sel0 = eu; 1826 hd->shd_mode0 = mode; 1827 hd->shd_eu_sel1 = SEC_EU_NONE; 1828 hd->shd_mode1 = 0; 1829 1830 /* Pointer 0: NULL */ 1831 error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 1832 if (error) 1833 return (error); 1834 1835 /* Pointer 1: Context In (Not used: NULL) */ 1836 error = sec_make_pointer_direct(sc, desc, 1, 0, 0); 1837 if (error) 1838 return (error); 1839 1840 /* Pointer 2: HMAC Key (or NULL, depending on digest type) */ 1841 if (hd->shd_mode0 & SEC_MDEU_MODE_HMAC) 1842 error = sec_make_pointer_direct(sc, desc, 2, 1843 desc->sd_desc_paddr + offsetof(struct sec_hw_desc, 1844 shd_mkey), ses->ss_mklen); 1845 else 1846 error = sec_make_pointer_direct(sc, desc, 2, 0, 0); 1847 1848 if (error) 1849 return (error); 1850 1851 /* Pointer 3: Input Data */ 1852 error = sec_make_pointer(sc, desc, 3, crp->crp_buf, mac->crd_skip, 1853 mac->crd_len, buftype); 1854 if (error) 1855 return (error); 1856 1857 /* Pointer 4: NULL */ 1858 error = sec_make_pointer_direct(sc, desc, 4, 0, 0); 1859 if (error) 1860 return (error); 1861 1862 /* Pointer 5: Hash out */ 1863 error = sec_make_pointer(sc, desc, 5, crp->crp_buf, 1864 mac->crd_inject, hashlen, buftype); 1865 if (error) 1866 return (error); 1867 1868 /* Pointer 6: NULL */ 1869 error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 1870 1871 return (0); 1872} 1873