305 /* Alloc DMA memory for descriptors and link tables */ 306 error = sec_alloc_dma_mem(sc, &(sc->sc_desc_dmem), 307 SEC_DESCRIPTORS * sizeof(struct sec_hw_desc)); 308 309 if (error) 310 goto fail4; 311 312 error = sec_alloc_dma_mem(sc, &(sc->sc_lt_dmem), 313 (SEC_LT_ENTRIES + 1) * sizeof(struct sec_hw_lt)); 314 315 if (error) 316 goto fail5; 317 318 /* Fill in descriptors and link tables */ 319 for (i = 0; i < SEC_DESCRIPTORS; i++) { 320 sc->sc_desc[i].sd_desc = 321 (struct sec_hw_desc*)(sc->sc_desc_dmem.dma_vaddr) + i; 322 sc->sc_desc[i].sd_desc_paddr = sc->sc_desc_dmem.dma_paddr + 323 (i * sizeof(struct sec_hw_desc)); 324 } 325 326 for (i = 0; i < SEC_LT_ENTRIES + 1; i++) { 327 sc->sc_lt[i].sl_lt = 328 (struct sec_hw_lt*)(sc->sc_lt_dmem.dma_vaddr) + i; 329 sc->sc_lt[i].sl_lt_paddr = sc->sc_lt_dmem.dma_paddr + 330 (i * sizeof(struct sec_hw_lt)); 331 } 332 333 /* Last entry in link table is used to create a circle */ 334 lt = sc->sc_lt[SEC_LT_ENTRIES].sl_lt; 335 lt->shl_length = 0; 336 lt->shl_r = 0; 337 lt->shl_n = 1; 338 lt->shl_ptr = sc->sc_lt[0].sl_lt_paddr; 339 340 /* Init descriptor and link table queues pointers */ 341 SEC_CNT_INIT(sc, sc_free_desc_get_cnt, SEC_DESCRIPTORS); 342 SEC_CNT_INIT(sc, sc_free_desc_put_cnt, SEC_DESCRIPTORS); 343 SEC_CNT_INIT(sc, sc_ready_desc_get_cnt, SEC_DESCRIPTORS); 344 SEC_CNT_INIT(sc, sc_ready_desc_put_cnt, SEC_DESCRIPTORS); 345 SEC_CNT_INIT(sc, sc_queued_desc_get_cnt, SEC_DESCRIPTORS); 346 SEC_CNT_INIT(sc, sc_queued_desc_put_cnt, SEC_DESCRIPTORS); 347 SEC_CNT_INIT(sc, sc_lt_alloc_cnt, SEC_LT_ENTRIES); 348 SEC_CNT_INIT(sc, sc_lt_free_cnt, SEC_LT_ENTRIES); 349 350 /* Create masks for fast checks */ 351 sc->sc_int_error_mask = 0; 352 for (i = 0; i < SEC_CHANNELS; i++) 353 sc->sc_int_error_mask |= (~0ULL & SEC_INT_CH_ERR(i)); 354 355 switch (sc->sc_version) { 356 case 2: 357 sc->sc_channel_idle_mask = 358 (SEC_CHAN_CSR2_FFLVL_M << SEC_CHAN_CSR2_FFLVL_S) | 359 (SEC_CHAN_CSR2_MSTATE_M << SEC_CHAN_CSR2_MSTATE_S) | 360 (SEC_CHAN_CSR2_PSTATE_M << SEC_CHAN_CSR2_PSTATE_S) | 361 (SEC_CHAN_CSR2_GSTATE_M << SEC_CHAN_CSR2_GSTATE_S); 362 break; 363 case 3: 364 sc->sc_channel_idle_mask = 365 (SEC_CHAN_CSR3_FFLVL_M << SEC_CHAN_CSR3_FFLVL_S) | 366 (SEC_CHAN_CSR3_MSTATE_M << SEC_CHAN_CSR3_MSTATE_S) | 367 (SEC_CHAN_CSR3_PSTATE_M << SEC_CHAN_CSR3_PSTATE_S) | 368 (SEC_CHAN_CSR3_GSTATE_M << SEC_CHAN_CSR3_GSTATE_S); 369 break; 370 } 371 372 /* Init hardware */ 373 error = sec_init(sc); 374 375 if (error) 376 goto fail6; 377 378 /* Register in OCF (AESU) */ 379 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); 380 381 /* Register in OCF (DEU) */ 382 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); 383 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); 384 385 /* Register in OCF (MDEU) */ 386 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0); 387 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 388 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0); 389 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 390 crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0); 391 if (sc->sc_version >= 3) { 392 crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0); 393 crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0); 394 } 395 396 return (0); 397 398fail6: 399 sec_free_dma_mem(&(sc->sc_lt_dmem)); 400fail5: 401 sec_free_dma_mem(&(sc->sc_desc_dmem)); 402fail4: 403 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 404 sc->sc_sec_irid, "secondary"); 405fail3: 406 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 407 sc->sc_pri_irid, "primary"); 408fail2: 409 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); 410fail1: 411 mtx_destroy(&sc->sc_controller_lock); 412 mtx_destroy(&sc->sc_descriptors_lock); 413 mtx_destroy(&sc->sc_sessions_lock); 414 415 return (ENXIO); 416} 417 418static int 419sec_detach(device_t dev) 420{ 421 struct sec_softc *sc = device_get_softc(dev); 422 int i, error, timeout = SEC_TIMEOUT; 423 424 /* Prepare driver to shutdown */ 425 SEC_LOCK(sc, descriptors); 426 sc->sc_shutdown = 1; 427 SEC_UNLOCK(sc, descriptors); 428 429 /* Wait until all queued processing finishes */ 430 while (1) { 431 SEC_LOCK(sc, descriptors); 432 i = SEC_READY_DESC_CNT(sc) + SEC_QUEUED_DESC_CNT(sc); 433 SEC_UNLOCK(sc, descriptors); 434 435 if (i == 0) 436 break; 437 438 if (timeout < 0) { 439 device_printf(dev, "queue flush timeout!\n"); 440 441 /* DMA can be still active - stop it */ 442 for (i = 0; i < SEC_CHANNELS; i++) 443 sec_channel_reset(sc, i, 1); 444 445 break; 446 } 447 448 timeout -= 1000; 449 DELAY(1000); 450 } 451 452 /* Disable interrupts */ 453 SEC_WRITE(sc, SEC_IER, 0); 454 455 /* Unregister from OCF */ 456 crypto_unregister_all(sc->sc_cid); 457 458 /* Free DMA memory */ 459 for (i = 0; i < SEC_DESCRIPTORS; i++) 460 SEC_DESC_FREE_POINTERS(&(sc->sc_desc[i])); 461 462 sec_free_dma_mem(&(sc->sc_lt_dmem)); 463 sec_free_dma_mem(&(sc->sc_desc_dmem)); 464 465 /* Release interrupts */ 466 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 467 sc->sc_pri_irid, "primary"); 468 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 469 sc->sc_sec_irid, "secondary"); 470 471 /* Release memory */ 472 if (sc->sc_rres) { 473 error = bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, 474 sc->sc_rres); 475 if (error) 476 device_printf(dev, "bus_release_resource() failed for" 477 " I/O memory, error %d\n", error); 478 479 sc->sc_rres = NULL; 480 } 481 482 mtx_destroy(&sc->sc_controller_lock); 483 mtx_destroy(&sc->sc_descriptors_lock); 484 mtx_destroy(&sc->sc_sessions_lock); 485 486 return (0); 487} 488 489static int 490sec_suspend(device_t dev) 491{ 492 493 return (0); 494} 495 496static int 497sec_resume(device_t dev) 498{ 499 500 return (0); 501} 502 503static int 504sec_shutdown(device_t dev) 505{ 506 507 return (0); 508} 509 510static int 511sec_setup_intr(struct sec_softc *sc, struct resource **ires, void **ihand, 512 int *irid, driver_intr_t handler, const char *iname) 513{ 514 int error; 515 516 (*ires) = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ, irid, 517 RF_ACTIVE); 518 519 if ((*ires) == NULL) { 520 device_printf(sc->sc_dev, "could not allocate %s IRQ\n", iname); 521 return (ENXIO); 522 } 523 524 error = bus_setup_intr(sc->sc_dev, *ires, INTR_MPSAFE | INTR_TYPE_NET, 525 NULL, handler, sc, ihand); 526 527 if (error) { 528 device_printf(sc->sc_dev, "failed to set up %s IRQ\n", iname); 529 if (bus_release_resource(sc->sc_dev, SYS_RES_IRQ, *irid, *ires)) 530 device_printf(sc->sc_dev, "could not release %s IRQ\n", 531 iname); 532 533 (*ires) = NULL; 534 return (error); 535 } 536 537 return (0); 538} 539 540static void 541sec_release_intr(struct sec_softc *sc, struct resource *ires, void *ihand, 542 int irid, const char *iname) 543{ 544 int error; 545 546 if (ires == NULL) 547 return; 548 549 error = bus_teardown_intr(sc->sc_dev, ires, ihand); 550 if (error) 551 device_printf(sc->sc_dev, "bus_teardown_intr() failed for %s" 552 " IRQ, error %d\n", iname, error); 553 554 error = bus_release_resource(sc->sc_dev, SYS_RES_IRQ, irid, ires); 555 if (error) 556 device_printf(sc->sc_dev, "bus_release_resource() failed for %s" 557 " IRQ, error %d\n", iname, error); 558} 559 560static void 561sec_primary_intr(void *arg) 562{ 563 struct sec_softc *sc = arg; 564 struct sec_desc *desc; 565 uint64_t isr; 566 int i, wakeup = 0; 567 568 SEC_LOCK(sc, controller); 569 570 /* Check for errors */ 571 isr = SEC_READ(sc, SEC_ISR); 572 if (isr & sc->sc_int_error_mask) { 573 /* Check each channel for error */ 574 for (i = 0; i < SEC_CHANNELS; i++) { 575 if ((isr & SEC_INT_CH_ERR(i)) == 0) 576 continue; 577 578 device_printf(sc->sc_dev, 579 "I/O error on channel %i!\n", i); 580 581 /* Find and mark problematic descriptor */ 582 desc = sec_find_desc(sc, SEC_READ(sc, 583 SEC_CHAN_CDPR(i))); 584 585 if (desc != NULL) 586 desc->sd_error = EIO; 587 588 /* Do partial channel reset */ 589 sec_channel_reset(sc, i, 0); 590 } 591 } 592 593 /* ACK interrupt */ 594 SEC_WRITE(sc, SEC_ICR, 0xFFFFFFFFFFFFFFFFULL); 595 596 SEC_UNLOCK(sc, controller); 597 SEC_LOCK(sc, descriptors); 598 599 /* Handle processed descriptors */ 600 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 601 602 while (SEC_QUEUED_DESC_CNT(sc) > 0) { 603 desc = SEC_GET_QUEUED_DESC(sc); 604 605 if (desc->sd_desc->shd_done != 0xFF && desc->sd_error == 0) { 606 SEC_PUT_BACK_QUEUED_DESC(sc); 607 break; 608 } 609 610 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_PREREAD | 611 BUS_DMASYNC_PREWRITE); 612 613 desc->sd_crp->crp_etype = desc->sd_error; 614 crypto_done(desc->sd_crp); 615 616 SEC_DESC_FREE_POINTERS(desc); 617 SEC_DESC_FREE_LT(sc, desc); 618 SEC_DESC_QUEUED2FREE(sc); 619 } 620 621 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 622 623 if (!sc->sc_shutdown) { 624 wakeup = sc->sc_blocked; 625 sc->sc_blocked = 0; 626 } 627 628 SEC_UNLOCK(sc, descriptors); 629 630 /* Enqueue ready descriptors in hardware */ 631 sec_enqueue(sc); 632 633 if (wakeup) 634 crypto_unblock(sc->sc_cid, wakeup); 635} 636 637static void 638sec_secondary_intr(void *arg) 639{ 640 struct sec_softc *sc = arg; 641 642 device_printf(sc->sc_dev, "spurious secondary interrupt!\n"); 643 sec_primary_intr(arg); 644} 645 646static int 647sec_controller_reset(struct sec_softc *sc) 648{ 649 int timeout = SEC_TIMEOUT; 650 651 /* Reset Controller */ 652 SEC_WRITE(sc, SEC_MCR, SEC_MCR_SWR); 653 654 while (SEC_READ(sc, SEC_MCR) & SEC_MCR_SWR) { 655 DELAY(1000); 656 timeout -= 1000; 657 658 if (timeout < 0) { 659 device_printf(sc->sc_dev, "timeout while waiting for " 660 "device reset!\n"); 661 return (ETIMEDOUT); 662 } 663 } 664 665 return (0); 666} 667 668static int 669sec_channel_reset(struct sec_softc *sc, int channel, int full) 670{ 671 int timeout = SEC_TIMEOUT; 672 uint64_t bit = (full) ? SEC_CHAN_CCR_R : SEC_CHAN_CCR_CON; 673 uint64_t reg; 674 675 /* Reset Channel */ 676 reg = SEC_READ(sc, SEC_CHAN_CCR(channel)); 677 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg | bit); 678 679 while (SEC_READ(sc, SEC_CHAN_CCR(channel)) & bit) { 680 DELAY(1000); 681 timeout -= 1000; 682 683 if (timeout < 0) { 684 device_printf(sc->sc_dev, "timeout while waiting for " 685 "channel reset!\n"); 686 return (ETIMEDOUT); 687 } 688 } 689 690 if (full) { 691 reg = SEC_CHAN_CCR_CDIE | SEC_CHAN_CCR_NT | SEC_CHAN_CCR_BS; 692 693 switch(sc->sc_version) { 694 case 2: 695 reg |= SEC_CHAN_CCR_CDWE; 696 break; 697 case 3: 698 reg |= SEC_CHAN_CCR_AWSE | SEC_CHAN_CCR_WGN; 699 break; 700 } 701 702 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg); 703 } 704 705 return (0); 706} 707 708static int 709sec_init(struct sec_softc *sc) 710{ 711 uint64_t reg; 712 int error, i; 713 714 /* Reset controller twice to clear all pending interrupts */ 715 error = sec_controller_reset(sc); 716 if (error) 717 return (error); 718 719 error = sec_controller_reset(sc); 720 if (error) 721 return (error); 722 723 /* Reset channels */ 724 for (i = 0; i < SEC_CHANNELS; i++) { 725 error = sec_channel_reset(sc, i, 1); 726 if (error) 727 return (error); 728 } 729 730 /* Enable Interrupts */ 731 reg = SEC_INT_ITO; 732 for (i = 0; i < SEC_CHANNELS; i++) 733 reg |= SEC_INT_CH_DN(i) | SEC_INT_CH_ERR(i); 734 735 SEC_WRITE(sc, SEC_IER, reg); 736 737 return (error); 738} 739 740static void 741sec_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 742{ 743 struct sec_dma_mem *dma_mem = arg; 744 745 if (error) 746 return; 747 748 KASSERT(nseg == 1, ("Wrong number of segments, should be 1")); 749 dma_mem->dma_paddr = segs->ds_addr; 750} 751 752static void 753sec_dma_map_desc_cb(void *arg, bus_dma_segment_t *segs, int nseg, 754 int error) 755{ 756 struct sec_desc_map_info *sdmi = arg; 757 struct sec_softc *sc = sdmi->sdmi_sc; 758 struct sec_lt *lt = NULL; 759 bus_addr_t addr; 760 bus_size_t size; 761 int i; 762 763 SEC_LOCK_ASSERT(sc, descriptors); 764 765 if (error) 766 return; 767 768 for (i = 0; i < nseg; i++) { 769 addr = segs[i].ds_addr; 770 size = segs[i].ds_len; 771 772 /* Skip requested offset */ 773 if (sdmi->sdmi_offset >= size) { 774 sdmi->sdmi_offset -= size; 775 continue; 776 } 777 778 addr += sdmi->sdmi_offset; 779 size -= sdmi->sdmi_offset; 780 sdmi->sdmi_offset = 0; 781 782 /* Do not link more than requested */ 783 if (sdmi->sdmi_size < size) 784 size = sdmi->sdmi_size; 785 786 lt = SEC_ALLOC_LT_ENTRY(sc); 787 lt->sl_lt->shl_length = size; 788 lt->sl_lt->shl_r = 0; 789 lt->sl_lt->shl_n = 0; 790 lt->sl_lt->shl_ptr = addr; 791 792 if (sdmi->sdmi_lt_first == NULL) 793 sdmi->sdmi_lt_first = lt; 794 795 sdmi->sdmi_lt_used += 1; 796 797 if ((sdmi->sdmi_size -= size) == 0) 798 break; 799 } 800 801 sdmi->sdmi_lt_last = lt; 802} 803 804static void 805sec_dma_map_desc_cb2(void *arg, bus_dma_segment_t *segs, int nseg, 806 bus_size_t size, int error) 807{ 808 809 sec_dma_map_desc_cb(arg, segs, nseg, error); 810} 811 812static int 813sec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem, 814 bus_size_t size) 815{ 816 int error; 817 818 if (dma_mem->dma_vaddr != NULL) 819 return (EBUSY); 820 821 error = bus_dma_tag_create(NULL, /* parent */ 822 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 823 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 824 BUS_SPACE_MAXADDR, /* highaddr */ 825 NULL, NULL, /* filtfunc, filtfuncarg */ 826 size, 1, /* maxsize, nsegments */ 827 size, 0, /* maxsegsz, flags */ 828 NULL, NULL, /* lockfunc, lockfuncarg */ 829 &(dma_mem->dma_tag)); /* dmat */ 830 831 if (error) { 832 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 833 " %i!\n", error); 834 goto err1; 835 } 836 837 error = bus_dmamem_alloc(dma_mem->dma_tag, &(dma_mem->dma_vaddr), 838 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &(dma_mem->dma_map)); 839 840 if (error) { 841 device_printf(sc->sc_dev, "failed to allocate DMA safe" 842 " memory, error %i!\n", error); 843 goto err2; 844 } 845 846 error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map, 847 dma_mem->dma_vaddr, size, sec_alloc_dma_mem_cb, dma_mem, 848 BUS_DMA_NOWAIT); 849 850 if (error) { 851 device_printf(sc->sc_dev, "cannot get address of the DMA" 852 " memory, error %i\n", error); 853 goto err3; 854 } 855 856 dma_mem->dma_is_map = 0; 857 return (0); 858 859err3: 860 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, dma_mem->dma_map); 861err2: 862 bus_dma_tag_destroy(dma_mem->dma_tag); 863err1: 864 dma_mem->dma_vaddr = NULL; 865 return(error); 866} 867 868static int 869sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, void *mem, 870 bus_size_t size, int type, struct sec_desc_map_info *sdmi) 871{ 872 int error; 873 874 if (dma_mem->dma_vaddr != NULL) 875 return (EBUSY); 876 877 switch (type) { 878 case SEC_MEMORY: 879 break; 880 case SEC_UIO: 881 size = SEC_FREE_LT_CNT(sc) * SEC_MAX_DMA_BLOCK_SIZE; 882 break; 883 case SEC_MBUF: 884 size = m_length((struct mbuf*)mem, NULL); 885 break; 886 default: 887 return (EINVAL); 888 } 889 890 error = bus_dma_tag_create(NULL, /* parent */ 891 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 892 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 893 BUS_SPACE_MAXADDR, /* highaddr */ 894 NULL, NULL, /* filtfunc, filtfuncarg */ 895 size, /* maxsize */ 896 SEC_FREE_LT_CNT(sc), /* nsegments */ 897 SEC_MAX_DMA_BLOCK_SIZE, 0, /* maxsegsz, flags */ 898 NULL, NULL, /* lockfunc, lockfuncarg */ 899 &(dma_mem->dma_tag)); /* dmat */ 900 901 if (error) { 902 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 903 " %i!\n", error); 904 dma_mem->dma_vaddr = NULL; 905 return (error); 906 } 907 908 error = bus_dmamap_create(dma_mem->dma_tag, 0, &(dma_mem->dma_map)); 909 910 if (error) { 911 device_printf(sc->sc_dev, "failed to create DMA map, error %i!" 912 "\n", error); 913 bus_dma_tag_destroy(dma_mem->dma_tag); 914 return (error); 915 } 916 917 switch (type) { 918 case SEC_MEMORY: 919 error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map, 920 mem, size, sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT); 921 break; 922 case SEC_UIO: 923 error = bus_dmamap_load_uio(dma_mem->dma_tag, dma_mem->dma_map, 924 mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT); 925 break; 926 case SEC_MBUF: 927 error = bus_dmamap_load_mbuf(dma_mem->dma_tag, dma_mem->dma_map, 928 mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT); 929 break; 930 } 931 932 if (error) { 933 device_printf(sc->sc_dev, "cannot get address of the DMA" 934 " memory, error %i!\n", error); 935 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 936 bus_dma_tag_destroy(dma_mem->dma_tag); 937 return (error); 938 } 939 940 dma_mem->dma_is_map = 1; 941 dma_mem->dma_vaddr = mem; 942 943 return (0); 944} 945 946static void 947sec_free_dma_mem(struct sec_dma_mem *dma_mem) 948{ 949 950 /* Check for double free */ 951 if (dma_mem->dma_vaddr == NULL) 952 return; 953 954 bus_dmamap_unload(dma_mem->dma_tag, dma_mem->dma_map); 955 956 if (dma_mem->dma_is_map) 957 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 958 else 959 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, 960 dma_mem->dma_map); 961 962 bus_dma_tag_destroy(dma_mem->dma_tag); 963 dma_mem->dma_vaddr = NULL; 964} 965 966static int 967sec_eu_channel(struct sec_softc *sc, int eu) 968{ 969 uint64_t reg; 970 int channel = 0; 971 972 SEC_LOCK_ASSERT(sc, controller); 973 974 reg = SEC_READ(sc, SEC_EUASR); 975 976 switch (eu) { 977 case SEC_EU_AFEU: 978 channel = SEC_EUASR_AFEU(reg); 979 break; 980 case SEC_EU_DEU: 981 channel = SEC_EUASR_DEU(reg); 982 break; 983 case SEC_EU_MDEU_A: 984 case SEC_EU_MDEU_B: 985 channel = SEC_EUASR_MDEU(reg); 986 break; 987 case SEC_EU_RNGU: 988 channel = SEC_EUASR_RNGU(reg); 989 break; 990 case SEC_EU_PKEU: 991 channel = SEC_EUASR_PKEU(reg); 992 break; 993 case SEC_EU_AESU: 994 channel = SEC_EUASR_AESU(reg); 995 break; 996 case SEC_EU_KEU: 997 channel = SEC_EUASR_KEU(reg); 998 break; 999 case SEC_EU_CRCU: 1000 channel = SEC_EUASR_CRCU(reg); 1001 break; 1002 } 1003 1004 return (channel - 1); 1005} 1006 1007static int 1008sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, int channel) 1009{ 1010 u_int fflvl = SEC_MAX_FIFO_LEVEL; 1011 uint64_t reg; 1012 int i; 1013 1014 SEC_LOCK_ASSERT(sc, controller); 1015 1016 /* Find free channel if have not got one */ 1017 if (channel < 0) { 1018 for (i = 0; i < SEC_CHANNELS; i++) { 1019 reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 1020 1021 if ((reg & sc->sc_channel_idle_mask) == 0) { 1022 channel = i; 1023 break; 1024 } 1025 } 1026 } 1027 1028 /* There is no free channel */ 1029 if (channel < 0) 1030 return (-1); 1031 1032 /* Check FIFO level on selected channel */ 1033 reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 1034 1035 switch(sc->sc_version) { 1036 case 2: 1037 fflvl = (reg >> SEC_CHAN_CSR2_FFLVL_S) & SEC_CHAN_CSR2_FFLVL_M; 1038 break; 1039 case 3: 1040 fflvl = (reg >> SEC_CHAN_CSR3_FFLVL_S) & SEC_CHAN_CSR3_FFLVL_M; 1041 break; 1042 } 1043 1044 if (fflvl >= SEC_MAX_FIFO_LEVEL) 1045 return (-1); 1046 1047 /* Enqueue descriptor in channel */ 1048 SEC_WRITE(sc, SEC_CHAN_FF(channel), desc->sd_desc_paddr); 1049 1050 return (channel); 1051} 1052 1053static void 1054sec_enqueue(struct sec_softc *sc) 1055{ 1056 struct sec_desc *desc; 1057 int ch0, ch1; 1058 1059 SEC_LOCK(sc, descriptors); 1060 SEC_LOCK(sc, controller); 1061 1062 while (SEC_READY_DESC_CNT(sc) > 0) { 1063 desc = SEC_GET_READY_DESC(sc); 1064 1065 ch0 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel0); 1066 ch1 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel1); 1067 1068 /* 1069 * Both EU are used by the same channel. 1070 * Enqueue descriptor in channel used by busy EUs. 1071 */ 1072 if (ch0 >= 0 && ch0 == ch1) { 1073 if (sec_enqueue_desc(sc, desc, ch0) >= 0) { 1074 SEC_DESC_READY2QUEUED(sc); 1075 continue; 1076 } 1077 } 1078 1079 /* 1080 * Only one EU is free. 1081 * Enqueue descriptor in channel used by busy EU. 1082 */ 1083 if ((ch0 >= 0 && ch1 < 0) || (ch1 >= 0 && ch0 < 0)) { 1084 if (sec_enqueue_desc(sc, desc, (ch0 >= 0) ? ch0 : ch1) 1085 >= 0) { 1086 SEC_DESC_READY2QUEUED(sc); 1087 continue; 1088 } 1089 } 1090 1091 /* 1092 * Both EU are free. 1093 * Enqueue descriptor in first free channel. 1094 */ 1095 if (ch0 < 0 && ch1 < 0) { 1096 if (sec_enqueue_desc(sc, desc, -1) >= 0) { 1097 SEC_DESC_READY2QUEUED(sc); 1098 continue; 1099 } 1100 } 1101 1102 /* Current descriptor can not be queued at the moment */ 1103 SEC_PUT_BACK_READY_DESC(sc); 1104 break; 1105 } 1106 1107 SEC_UNLOCK(sc, controller); 1108 SEC_UNLOCK(sc, descriptors); 1109} 1110 1111static struct sec_desc * 1112sec_find_desc(struct sec_softc *sc, bus_addr_t paddr) 1113{ 1114 struct sec_desc *desc = NULL; 1115 int i; 1116 1117 SEC_LOCK_ASSERT(sc, descriptors); 1118 1119 for (i = 0; i < SEC_CHANNELS; i++) { 1120 if (sc->sc_desc[i].sd_desc_paddr == paddr) { 1121 desc = &(sc->sc_desc[i]); 1122 break; 1123 } 1124 } 1125 1126 return (desc); 1127} 1128 1129static int 1130sec_make_pointer_direct(struct sec_softc *sc, struct sec_desc *desc, u_int n, 1131 bus_addr_t data, bus_size_t dsize) 1132{ 1133 struct sec_hw_desc_ptr *ptr; 1134 1135 SEC_LOCK_ASSERT(sc, descriptors); 1136 1137 ptr = &(desc->sd_desc->shd_pointer[n]); 1138 ptr->shdp_length = dsize; 1139 ptr->shdp_extent = 0; 1140 ptr->shdp_j = 0; 1141 ptr->shdp_ptr = data; 1142 1143 return (0); 1144} 1145 1146static int 1147sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, 1148 u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype) 1149{ 1150 struct sec_desc_map_info sdmi = { sc, dsize, doffset, NULL, NULL, 0 }; 1151 struct sec_hw_desc_ptr *ptr; 1152 int error; 1153 1154 SEC_LOCK_ASSERT(sc, descriptors); 1155 1156 /* For flat memory map only requested region */ 1157 if (dtype == SEC_MEMORY) { 1158 data = (uint8_t*)(data) + doffset; 1159 sdmi.sdmi_offset = 0; 1160 } 1161 1162 error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), data, dsize, 1163 dtype, &sdmi); 1164 1165 if (error) 1166 return (error); 1167 1168 sdmi.sdmi_lt_last->sl_lt->shl_r = 1; 1169 desc->sd_lt_used += sdmi.sdmi_lt_used; 1170 1171 ptr = &(desc->sd_desc->shd_pointer[n]); 1172 ptr->shdp_length = dsize; 1173 ptr->shdp_extent = 0; 1174 ptr->shdp_j = 1; 1175 ptr->shdp_ptr = sdmi.sdmi_lt_first->sl_lt_paddr; 1176 1177 return (0); 1178} 1179 1180static int 1181sec_split_cri(struct cryptoini *cri, struct cryptoini **enc, 1182 struct cryptoini **mac) 1183{ 1184 struct cryptoini *e, *m; 1185 1186 e = cri; 1187 m = cri->cri_next; 1188 1189 /* We can haldle only two operations */ 1190 if (m && m->cri_next) 1191 return (EINVAL); 1192 1193 if (sec_mdeu_can_handle(e->cri_alg)) { 1194 cri = m; 1195 m = e; 1196 e = cri; 1197 } 1198 1199 if (m && !sec_mdeu_can_handle(m->cri_alg)) 1200 return (EINVAL); 1201 1202 *enc = e; 1203 *mac = m; 1204 1205 return (0); 1206} 1207 1208static int 1209sec_split_crp(struct cryptop *crp, struct cryptodesc **enc, 1210 struct cryptodesc **mac) 1211{ 1212 struct cryptodesc *e, *m, *t; 1213 1214 e = crp->crp_desc; 1215 m = e->crd_next; 1216 1217 /* We can haldle only two operations */ 1218 if (m && m->crd_next) 1219 return (EINVAL); 1220 1221 if (sec_mdeu_can_handle(e->crd_alg)) { 1222 t = m; 1223 m = e; 1224 e = t; 1225 } 1226 1227 if (m && !sec_mdeu_can_handle(m->crd_alg)) 1228 return (EINVAL); 1229 1230 *enc = e; 1231 *mac = m; 1232 1233 return (0); 1234} 1235 1236static int 1237sec_alloc_session(struct sec_softc *sc) 1238{ 1239 struct sec_session *ses = NULL; 1240 int sid = -1; 1241 u_int i; 1242 1243 SEC_LOCK(sc, sessions); 1244 1245 for (i = 0; i < SEC_MAX_SESSIONS; i++) { 1246 if (sc->sc_sessions[i].ss_used == 0) { 1247 ses = &(sc->sc_sessions[i]); 1248 ses->ss_used = 1; 1249 ses->ss_ivlen = 0; 1250 ses->ss_klen = 0; 1251 ses->ss_mklen = 0; 1252 sid = i; 1253 break; 1254 } 1255 } 1256 1257 SEC_UNLOCK(sc, sessions); 1258 1259 return (sid); 1260} 1261 1262static struct sec_session * 1263sec_get_session(struct sec_softc *sc, u_int sid) 1264{ 1265 struct sec_session *ses; 1266 1267 if (sid >= SEC_MAX_SESSIONS) 1268 return (NULL); 1269 1270 SEC_LOCK(sc, sessions); 1271 1272 ses = &(sc->sc_sessions[sid]); 1273 1274 if (ses->ss_used == 0) 1275 ses = NULL; 1276 1277 SEC_UNLOCK(sc, sessions); 1278 1279 return (ses); 1280} 1281 1282static int 1283sec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri) 1284{ 1285 struct sec_softc *sc = device_get_softc(dev); 1286 struct sec_eu_methods *eu = sec_eus; 1287 struct cryptoini *enc = NULL; 1288 struct cryptoini *mac = NULL; 1289 struct sec_session *ses; 1290 int error = -1; 1291 int sid; 1292 1293 error = sec_split_cri(cri, &enc, &mac); 1294 if (error) 1295 return (error); 1296 1297 /* Check key lengths */ 1298 if (enc && enc->cri_key && (enc->cri_klen / 8) > SEC_MAX_KEY_LEN) 1299 return (E2BIG); 1300 1301 if (mac && mac->cri_key && (mac->cri_klen / 8) > SEC_MAX_KEY_LEN) 1302 return (E2BIG); 1303 1304 /* Only SEC 3.0 supports digests larger than 256 bits */ 1305 if (sc->sc_version < 3 && mac && mac->cri_klen > 256) 1306 return (E2BIG); 1307 1308 sid = sec_alloc_session(sc); 1309 if (sid < 0) 1310 return (ENOMEM); 1311 1312 ses = sec_get_session(sc, sid); 1313 1314 /* Find EU for this session */ 1315 while (eu->sem_make_desc != NULL) { 1316 error = eu->sem_newsession(sc, ses, enc, mac); 1317 if (error >= 0) 1318 break; 1319 1320 eu++; 1321 } 1322 1323 /* If not found, return EINVAL */ 1324 if (error < 0) { 1325 sec_free_session(sc, ses); 1326 return (EINVAL); 1327 } 1328 1329 /* Save cipher key */ 1330 if (enc && enc->cri_key) { 1331 ses->ss_klen = enc->cri_klen / 8; 1332 memcpy(ses->ss_key, enc->cri_key, ses->ss_klen); 1333 } 1334 1335 /* Save digest key */ 1336 if (mac && mac->cri_key) { 1337 ses->ss_mklen = mac->cri_klen / 8; 1338 memcpy(ses->ss_mkey, mac->cri_key, ses->ss_mklen); 1339 } 1340 1341 ses->ss_eu = eu; 1342 *sidp = sid; 1343 1344 return (0); 1345} 1346 1347static int 1348sec_freesession(device_t dev, uint64_t tid) 1349{ 1350 struct sec_softc *sc = device_get_softc(dev); 1351 struct sec_session *ses; 1352 int error = 0; 1353 1354 ses = sec_get_session(sc, CRYPTO_SESID2LID(tid)); 1355 if (ses == NULL) 1356 return (EINVAL); 1357 1358 sec_free_session(sc, ses); 1359 1360 return (error); 1361} 1362 1363static int 1364sec_process(device_t dev, struct cryptop *crp, int hint) 1365{ 1366 struct sec_softc *sc = device_get_softc(dev); 1367 struct sec_desc *desc = NULL; 1368 struct cryptodesc *mac, *enc; 1369 struct sec_session *ses; 1370 int buftype, error = 0; 1371 1372 /* Check Session ID */ 1373 ses = sec_get_session(sc, CRYPTO_SESID2LID(crp->crp_sid)); 1374 if (ses == NULL) { 1375 crp->crp_etype = EINVAL; 1376 crypto_done(crp); 1377 return (0); 1378 } 1379 1380 /* Check for input length */ 1381 if (crp->crp_ilen > SEC_MAX_DMA_BLOCK_SIZE) { 1382 crp->crp_etype = E2BIG; 1383 crypto_done(crp); 1384 return (0); 1385 } 1386 1387 /* Get descriptors */ 1388 if (sec_split_crp(crp, &enc, &mac)) { 1389 crp->crp_etype = EINVAL; 1390 crypto_done(crp); 1391 return (0); 1392 } 1393 1394 SEC_LOCK(sc, descriptors); 1395 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1396 1397 /* Block driver if there is no free descriptors or we are going down */ 1398 if (SEC_FREE_DESC_CNT(sc) == 0 || sc->sc_shutdown) { 1399 sc->sc_blocked |= CRYPTO_SYMQ; 1400 SEC_UNLOCK(sc, descriptors); 1401 return (ERESTART); 1402 } 1403 1404 /* Prepare descriptor */ 1405 desc = SEC_GET_FREE_DESC(sc); 1406 desc->sd_lt_used = 0; 1407 desc->sd_error = 0; 1408 desc->sd_crp = crp; 1409 1410 if (crp->crp_flags & CRYPTO_F_IOV) 1411 buftype = SEC_UIO; 1412 else if (crp->crp_flags & CRYPTO_F_IMBUF) 1413 buftype = SEC_MBUF; 1414 else 1415 buftype = SEC_MEMORY; 1416 1417 if (enc && enc->crd_flags & CRD_F_ENCRYPT) { 1418 if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1419 memcpy(desc->sd_desc->shd_iv, enc->crd_iv, 1420 ses->ss_ivlen); 1421 else 1422 arc4rand(desc->sd_desc->shd_iv, ses->ss_ivlen, 0); 1423 1424 if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0) 1425 crypto_copyback(crp->crp_flags, crp->crp_buf, 1426 enc->crd_inject, ses->ss_ivlen, 1427 desc->sd_desc->shd_iv); 1428 } else if (enc) { 1429 if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1430 memcpy(desc->sd_desc->shd_iv, enc->crd_iv, 1431 ses->ss_ivlen); 1432 else 1433 crypto_copydata(crp->crp_flags, crp->crp_buf, 1434 enc->crd_inject, ses->ss_ivlen, 1435 desc->sd_desc->shd_iv); 1436 } 1437 1438 if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) { 1439 if ((enc->crd_klen / 8) <= SEC_MAX_KEY_LEN) { 1440 ses->ss_klen = enc->crd_klen / 8; 1441 memcpy(ses->ss_key, enc->crd_key, ses->ss_klen); 1442 } else 1443 error = E2BIG; 1444 } 1445 1446 if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) { 1447 if ((mac->crd_klen / 8) <= SEC_MAX_KEY_LEN) { 1448 ses->ss_mklen = mac->crd_klen / 8; 1449 memcpy(ses->ss_mkey, mac->crd_key, ses->ss_mklen); 1450 } else 1451 error = E2BIG; 1452 } 1453 1454 if (!error) { 1455 memcpy(desc->sd_desc->shd_key, ses->ss_key, ses->ss_klen); 1456 memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, ses->ss_mklen); 1457 1458 error = ses->ss_eu->sem_make_desc(sc, ses, desc, crp, buftype); 1459 } 1460 1461 if (error) { 1462 SEC_DESC_FREE_POINTERS(desc); 1463 SEC_DESC_PUT_BACK_LT(sc, desc); 1464 SEC_PUT_BACK_FREE_DESC(sc); 1465 SEC_UNLOCK(sc, descriptors); 1466 crp->crp_etype = error; 1467 crypto_done(crp); 1468 return (0); 1469 } 1470 1471 /* 1472 * Skip DONE interrupt if this is not last request in burst, but only 1473 * if we are running on SEC 3.X. On SEC 2.X we have to enable DONE 1474 * signaling on each descriptor. 1475 */ 1476 if ((hint & CRYPTO_HINT_MORE) && sc->sc_version == 3) 1477 desc->sd_desc->shd_dn = 0; 1478 else 1479 desc->sd_desc->shd_dn = 1; 1480 1481 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1482 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_POSTREAD | 1483 BUS_DMASYNC_POSTWRITE); 1484 SEC_DESC_FREE2READY(sc); 1485 SEC_UNLOCK(sc, descriptors); 1486 1487 /* Enqueue ready descriptors in hardware */ 1488 sec_enqueue(sc); 1489 1490 return (0); 1491} 1492 1493static int 1494sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc, 1495 struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc, 1496 int buftype) 1497{ 1498 struct sec_hw_desc *hd = desc->sd_desc; 1499 int error; 1500 1501 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 1502 hd->shd_eu_sel1 = SEC_EU_NONE; 1503 hd->shd_mode1 = 0; 1504 1505 /* Pointer 0: NULL */ 1506 error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 1507 if (error) 1508 return (error); 1509 1510 /* Pointer 1: IV IN */ 1511 error = sec_make_pointer_direct(sc, desc, 1, desc->sd_desc_paddr + 1512 offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen); 1513 if (error) 1514 return (error); 1515 1516 /* Pointer 2: Cipher Key */ 1517 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 1518 offsetof(struct sec_hw_desc, shd_key), ses->ss_klen); 1519 if (error) 1520 return (error); 1521 1522 /* Pointer 3: Data IN */ 1523 error = sec_make_pointer(sc, desc, 3, crp->crp_buf, enc->crd_skip, 1524 enc->crd_len, buftype); 1525 if (error) 1526 return (error); 1527 1528 /* Pointer 4: Data OUT */ 1529 error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip, 1530 enc->crd_len, buftype); 1531 if (error) 1532 return (error); 1533 1534 /* Pointer 5: IV OUT (Not used: NULL) */ 1535 error = sec_make_pointer_direct(sc, desc, 5, 0, 0); 1536 if (error) 1537 return (error); 1538 1539 /* Pointer 6: NULL */ 1540 error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 1541 1542 return (error); 1543} 1544 1545static int 1546sec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc, 1547 struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc, 1548 struct cryptodesc *mac, int buftype) 1549{ 1550 struct sec_hw_desc *hd = desc->sd_desc; 1551 u_int eu, mode, hashlen; 1552 int error; 1553 1554 if (mac->crd_len < enc->crd_len) 1555 return (EINVAL); 1556 1557 if (mac->crd_skip + mac->crd_len != enc->crd_skip + enc->crd_len) 1558 return (EINVAL); 1559 1560 error = sec_mdeu_config(mac, &eu, &mode, &hashlen); 1561 if (error) 1562 return (error); 1563 1564 hd->shd_desc_type = SEC_DT_HMAC_SNOOP; 1565 hd->shd_eu_sel1 = eu; 1566 hd->shd_mode1 = mode; 1567 1568 /* Pointer 0: HMAC Key */ 1569 error = sec_make_pointer_direct(sc, desc, 0, desc->sd_desc_paddr + 1570 offsetof(struct sec_hw_desc, shd_mkey), ses->ss_mklen); 1571 if (error) 1572 return (error); 1573 1574 /* Pointer 1: HMAC-Only Data IN */ 1575 error = sec_make_pointer(sc, desc, 1, crp->crp_buf, mac->crd_skip, 1576 mac->crd_len - enc->crd_len, buftype); 1577 if (error) 1578 return (error); 1579 1580 /* Pointer 2: Cipher Key */ 1581 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 1582 offsetof(struct sec_hw_desc, shd_key), ses->ss_klen); 1583 if (error) 1584 return (error); 1585 1586 /* Pointer 3: IV IN */ 1587 error = sec_make_pointer_direct(sc, desc, 3, desc->sd_desc_paddr + 1588 offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen); 1589 if (error) 1590 return (error); 1591 1592 /* Pointer 4: Data IN */ 1593 error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip, 1594 enc->crd_len, buftype); 1595 if (error) 1596 return (error); 1597 1598 /* Pointer 5: Data OUT */ 1599 error = sec_make_pointer(sc, desc, 5, crp->crp_buf, enc->crd_skip, 1600 enc->crd_len, buftype); 1601 if (error) 1602 return (error); 1603 1604 /* Pointer 6: HMAC OUT */ 1605 error = sec_make_pointer(sc, desc, 6, crp->crp_buf, mac->crd_inject, 1606 hashlen, buftype); 1607 1608 return (error); 1609} 1610 1611/* AESU */ 1612 1613static int 1614sec_aesu_newsession(struct sec_softc *sc, struct sec_session *ses, 1615 struct cryptoini *enc, struct cryptoini *mac) 1616{ 1617 1618 if (enc == NULL) 1619 return (-1); 1620 1621 if (enc->cri_alg != CRYPTO_AES_CBC) 1622 return (-1); 1623 1624 ses->ss_ivlen = AES_BLOCK_LEN; 1625 1626 return (0); 1627} 1628 1629static int 1630sec_aesu_make_desc(struct sec_softc *sc, struct sec_session *ses, 1631 struct sec_desc *desc, struct cryptop *crp, int buftype) 1632{ 1633 struct sec_hw_desc *hd = desc->sd_desc; 1634 struct cryptodesc *enc, *mac; 1635 int error; 1636 1637 error = sec_split_crp(crp, &enc, &mac); 1638 if (error) 1639 return (error); 1640 1641 if (!enc) 1642 return (EINVAL); 1643 1644 hd->shd_eu_sel0 = SEC_EU_AESU; 1645 hd->shd_mode0 = SEC_AESU_MODE_CBC; 1646 1647 if (enc->crd_alg != CRYPTO_AES_CBC) 1648 return (EINVAL); 1649 1650 if (enc->crd_flags & CRD_F_ENCRYPT) { 1651 hd->shd_mode0 |= SEC_AESU_MODE_ED; 1652 hd->shd_dir = 0; 1653 } else 1654 hd->shd_dir = 1; 1655 1656 if (mac) 1657 error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac, 1658 buftype); 1659 else 1660 error = sec_build_common_ns_desc(sc, desc, ses, crp, enc, 1661 buftype); 1662 1663 return (error); 1664} 1665 1666/* DEU */ 1667 1668static int 1669sec_deu_newsession(struct sec_softc *sc, struct sec_session *ses, 1670 struct cryptoini *enc, struct cryptoini *mac) 1671{ 1672 1673 if (enc == NULL) 1674 return (-1); 1675 1676 switch (enc->cri_alg) { 1677 case CRYPTO_DES_CBC: 1678 case CRYPTO_3DES_CBC: 1679 break; 1680 default: 1681 return (-1); 1682 } 1683 1684 ses->ss_ivlen = DES_BLOCK_LEN; 1685 1686 return (0); 1687} 1688 1689static int 1690sec_deu_make_desc(struct sec_softc *sc, struct sec_session *ses, 1691 struct sec_desc *desc, struct cryptop *crp, int buftype) 1692{ 1693 struct sec_hw_desc *hd = desc->sd_desc; 1694 struct cryptodesc *enc, *mac; 1695 int error; 1696 1697 error = sec_split_crp(crp, &enc, &mac); 1698 if (error) 1699 return (error); 1700 1701 if (!enc) 1702 return (EINVAL); 1703 1704 hd->shd_eu_sel0 = SEC_EU_DEU; 1705 hd->shd_mode0 = SEC_DEU_MODE_CBC; 1706 1707 switch (enc->crd_alg) { 1708 case CRYPTO_3DES_CBC: 1709 hd->shd_mode0 |= SEC_DEU_MODE_TS; 1710 break; 1711 case CRYPTO_DES_CBC: 1712 break; 1713 default: 1714 return (EINVAL); 1715 } 1716 1717 if (enc->crd_flags & CRD_F_ENCRYPT) { 1718 hd->shd_mode0 |= SEC_DEU_MODE_ED; 1719 hd->shd_dir = 0; 1720 } else 1721 hd->shd_dir = 1; 1722 1723 if (mac) 1724 error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac, 1725 buftype); 1726 else 1727 error = sec_build_common_ns_desc(sc, desc, ses, crp, enc, 1728 buftype); 1729 1730 return (error); 1731} 1732 1733/* MDEU */ 1734 1735static int 1736sec_mdeu_can_handle(u_int alg) 1737{ 1738 switch (alg) { 1739 case CRYPTO_MD5: 1740 case CRYPTO_SHA1: 1741 case CRYPTO_MD5_HMAC: 1742 case CRYPTO_SHA1_HMAC: 1743 case CRYPTO_SHA2_256_HMAC: 1744 case CRYPTO_SHA2_384_HMAC: 1745 case CRYPTO_SHA2_512_HMAC: 1746 return (1); 1747 default: 1748 return (0); 1749 } 1750} 1751 1752static int 1753sec_mdeu_config(struct cryptodesc *crd, u_int *eu, u_int *mode, u_int *hashlen) 1754{ 1755 1756 *mode = SEC_MDEU_MODE_PD | SEC_MDEU_MODE_INIT; 1757 *eu = SEC_EU_NONE; 1758 1759 switch (crd->crd_alg) { 1760 case CRYPTO_MD5_HMAC: 1761 *mode |= SEC_MDEU_MODE_HMAC; 1762 /* FALLTHROUGH */ 1763 case CRYPTO_MD5: 1764 *eu = SEC_EU_MDEU_A; 1765 *mode |= SEC_MDEU_MODE_MD5; 1766 *hashlen = MD5_HASH_LEN; 1767 break; 1768 case CRYPTO_SHA1_HMAC: 1769 *mode |= SEC_MDEU_MODE_HMAC; 1770 /* FALLTHROUGH */ 1771 case CRYPTO_SHA1: 1772 *eu = SEC_EU_MDEU_A; 1773 *mode |= SEC_MDEU_MODE_SHA1; 1774 *hashlen = SHA1_HASH_LEN; 1775 break; 1776 case CRYPTO_SHA2_256_HMAC: 1777 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA256; 1778 *eu = SEC_EU_MDEU_A; 1779 break; 1780 case CRYPTO_SHA2_384_HMAC: 1781 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA384; 1782 *eu = SEC_EU_MDEU_B; 1783 break; 1784 case CRYPTO_SHA2_512_HMAC: 1785 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA512; 1786 *eu = SEC_EU_MDEU_B; 1787 break; 1788 default: 1789 return (EINVAL); 1790 } 1791 1792 if (*mode & SEC_MDEU_MODE_HMAC) 1793 *hashlen = SEC_HMAC_HASH_LEN; 1794 1795 return (0); 1796} 1797 1798static int 1799sec_mdeu_newsession(struct sec_softc *sc, struct sec_session *ses, 1800 struct cryptoini *enc, struct cryptoini *mac) 1801{ 1802 1803 if (mac && sec_mdeu_can_handle(mac->cri_alg)) 1804 return (0); 1805 1806 return (-1); 1807} 1808 1809static int 1810sec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses, 1811 struct sec_desc *desc, struct cryptop *crp, int buftype) 1812{ 1813 struct cryptodesc *enc, *mac; 1814 struct sec_hw_desc *hd = desc->sd_desc; 1815 u_int eu, mode, hashlen; 1816 int error; 1817 1818 error = sec_split_crp(crp, &enc, &mac); 1819 if (error) 1820 return (error); 1821 1822 if (enc) 1823 return (EINVAL); 1824 1825 error = sec_mdeu_config(mac, &eu, &mode, &hashlen); 1826 if (error) 1827 return (error); 1828 1829 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 1830 hd->shd_eu_sel0 = eu; 1831 hd->shd_mode0 = mode; 1832 hd->shd_eu_sel1 = SEC_EU_NONE; 1833 hd->shd_mode1 = 0; 1834 1835 /* Pointer 0: NULL */ 1836 error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 1837 if (error) 1838 return (error); 1839 1840 /* Pointer 1: Context In (Not used: NULL) */ 1841 error = sec_make_pointer_direct(sc, desc, 1, 0, 0); 1842 if (error) 1843 return (error); 1844 1845 /* Pointer 2: HMAC Key (or NULL, depending on digest type) */ 1846 if (hd->shd_mode0 & SEC_MDEU_MODE_HMAC) 1847 error = sec_make_pointer_direct(sc, desc, 2, 1848 desc->sd_desc_paddr + offsetof(struct sec_hw_desc, 1849 shd_mkey), ses->ss_mklen); 1850 else 1851 error = sec_make_pointer_direct(sc, desc, 2, 0, 0); 1852 1853 if (error) 1854 return (error); 1855 1856 /* Pointer 3: Input Data */ 1857 error = sec_make_pointer(sc, desc, 3, crp->crp_buf, mac->crd_skip, 1858 mac->crd_len, buftype); 1859 if (error) 1860 return (error); 1861 1862 /* Pointer 4: NULL */ 1863 error = sec_make_pointer_direct(sc, desc, 4, 0, 0); 1864 if (error) 1865 return (error); 1866 1867 /* Pointer 5: Hash out */ 1868 error = sec_make_pointer(sc, desc, 5, crp->crp_buf, 1869 mac->crd_inject, hashlen, buftype); 1870 if (error) 1871 return (error); 1872 1873 /* Pointer 6: NULL */ 1874 error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 1875 1876 return (0); 1877}
| 300 /* Alloc DMA memory for descriptors and link tables */ 301 error = sec_alloc_dma_mem(sc, &(sc->sc_desc_dmem), 302 SEC_DESCRIPTORS * sizeof(struct sec_hw_desc)); 303 304 if (error) 305 goto fail4; 306 307 error = sec_alloc_dma_mem(sc, &(sc->sc_lt_dmem), 308 (SEC_LT_ENTRIES + 1) * sizeof(struct sec_hw_lt)); 309 310 if (error) 311 goto fail5; 312 313 /* Fill in descriptors and link tables */ 314 for (i = 0; i < SEC_DESCRIPTORS; i++) { 315 sc->sc_desc[i].sd_desc = 316 (struct sec_hw_desc*)(sc->sc_desc_dmem.dma_vaddr) + i; 317 sc->sc_desc[i].sd_desc_paddr = sc->sc_desc_dmem.dma_paddr + 318 (i * sizeof(struct sec_hw_desc)); 319 } 320 321 for (i = 0; i < SEC_LT_ENTRIES + 1; i++) { 322 sc->sc_lt[i].sl_lt = 323 (struct sec_hw_lt*)(sc->sc_lt_dmem.dma_vaddr) + i; 324 sc->sc_lt[i].sl_lt_paddr = sc->sc_lt_dmem.dma_paddr + 325 (i * sizeof(struct sec_hw_lt)); 326 } 327 328 /* Last entry in link table is used to create a circle */ 329 lt = sc->sc_lt[SEC_LT_ENTRIES].sl_lt; 330 lt->shl_length = 0; 331 lt->shl_r = 0; 332 lt->shl_n = 1; 333 lt->shl_ptr = sc->sc_lt[0].sl_lt_paddr; 334 335 /* Init descriptor and link table queues pointers */ 336 SEC_CNT_INIT(sc, sc_free_desc_get_cnt, SEC_DESCRIPTORS); 337 SEC_CNT_INIT(sc, sc_free_desc_put_cnt, SEC_DESCRIPTORS); 338 SEC_CNT_INIT(sc, sc_ready_desc_get_cnt, SEC_DESCRIPTORS); 339 SEC_CNT_INIT(sc, sc_ready_desc_put_cnt, SEC_DESCRIPTORS); 340 SEC_CNT_INIT(sc, sc_queued_desc_get_cnt, SEC_DESCRIPTORS); 341 SEC_CNT_INIT(sc, sc_queued_desc_put_cnt, SEC_DESCRIPTORS); 342 SEC_CNT_INIT(sc, sc_lt_alloc_cnt, SEC_LT_ENTRIES); 343 SEC_CNT_INIT(sc, sc_lt_free_cnt, SEC_LT_ENTRIES); 344 345 /* Create masks for fast checks */ 346 sc->sc_int_error_mask = 0; 347 for (i = 0; i < SEC_CHANNELS; i++) 348 sc->sc_int_error_mask |= (~0ULL & SEC_INT_CH_ERR(i)); 349 350 switch (sc->sc_version) { 351 case 2: 352 sc->sc_channel_idle_mask = 353 (SEC_CHAN_CSR2_FFLVL_M << SEC_CHAN_CSR2_FFLVL_S) | 354 (SEC_CHAN_CSR2_MSTATE_M << SEC_CHAN_CSR2_MSTATE_S) | 355 (SEC_CHAN_CSR2_PSTATE_M << SEC_CHAN_CSR2_PSTATE_S) | 356 (SEC_CHAN_CSR2_GSTATE_M << SEC_CHAN_CSR2_GSTATE_S); 357 break; 358 case 3: 359 sc->sc_channel_idle_mask = 360 (SEC_CHAN_CSR3_FFLVL_M << SEC_CHAN_CSR3_FFLVL_S) | 361 (SEC_CHAN_CSR3_MSTATE_M << SEC_CHAN_CSR3_MSTATE_S) | 362 (SEC_CHAN_CSR3_PSTATE_M << SEC_CHAN_CSR3_PSTATE_S) | 363 (SEC_CHAN_CSR3_GSTATE_M << SEC_CHAN_CSR3_GSTATE_S); 364 break; 365 } 366 367 /* Init hardware */ 368 error = sec_init(sc); 369 370 if (error) 371 goto fail6; 372 373 /* Register in OCF (AESU) */ 374 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); 375 376 /* Register in OCF (DEU) */ 377 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); 378 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); 379 380 /* Register in OCF (MDEU) */ 381 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0); 382 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 383 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0); 384 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 385 crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0); 386 if (sc->sc_version >= 3) { 387 crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0); 388 crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0); 389 } 390 391 return (0); 392 393fail6: 394 sec_free_dma_mem(&(sc->sc_lt_dmem)); 395fail5: 396 sec_free_dma_mem(&(sc->sc_desc_dmem)); 397fail4: 398 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 399 sc->sc_sec_irid, "secondary"); 400fail3: 401 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 402 sc->sc_pri_irid, "primary"); 403fail2: 404 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); 405fail1: 406 mtx_destroy(&sc->sc_controller_lock); 407 mtx_destroy(&sc->sc_descriptors_lock); 408 mtx_destroy(&sc->sc_sessions_lock); 409 410 return (ENXIO); 411} 412 413static int 414sec_detach(device_t dev) 415{ 416 struct sec_softc *sc = device_get_softc(dev); 417 int i, error, timeout = SEC_TIMEOUT; 418 419 /* Prepare driver to shutdown */ 420 SEC_LOCK(sc, descriptors); 421 sc->sc_shutdown = 1; 422 SEC_UNLOCK(sc, descriptors); 423 424 /* Wait until all queued processing finishes */ 425 while (1) { 426 SEC_LOCK(sc, descriptors); 427 i = SEC_READY_DESC_CNT(sc) + SEC_QUEUED_DESC_CNT(sc); 428 SEC_UNLOCK(sc, descriptors); 429 430 if (i == 0) 431 break; 432 433 if (timeout < 0) { 434 device_printf(dev, "queue flush timeout!\n"); 435 436 /* DMA can be still active - stop it */ 437 for (i = 0; i < SEC_CHANNELS; i++) 438 sec_channel_reset(sc, i, 1); 439 440 break; 441 } 442 443 timeout -= 1000; 444 DELAY(1000); 445 } 446 447 /* Disable interrupts */ 448 SEC_WRITE(sc, SEC_IER, 0); 449 450 /* Unregister from OCF */ 451 crypto_unregister_all(sc->sc_cid); 452 453 /* Free DMA memory */ 454 for (i = 0; i < SEC_DESCRIPTORS; i++) 455 SEC_DESC_FREE_POINTERS(&(sc->sc_desc[i])); 456 457 sec_free_dma_mem(&(sc->sc_lt_dmem)); 458 sec_free_dma_mem(&(sc->sc_desc_dmem)); 459 460 /* Release interrupts */ 461 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 462 sc->sc_pri_irid, "primary"); 463 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 464 sc->sc_sec_irid, "secondary"); 465 466 /* Release memory */ 467 if (sc->sc_rres) { 468 error = bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, 469 sc->sc_rres); 470 if (error) 471 device_printf(dev, "bus_release_resource() failed for" 472 " I/O memory, error %d\n", error); 473 474 sc->sc_rres = NULL; 475 } 476 477 mtx_destroy(&sc->sc_controller_lock); 478 mtx_destroy(&sc->sc_descriptors_lock); 479 mtx_destroy(&sc->sc_sessions_lock); 480 481 return (0); 482} 483 484static int 485sec_suspend(device_t dev) 486{ 487 488 return (0); 489} 490 491static int 492sec_resume(device_t dev) 493{ 494 495 return (0); 496} 497 498static int 499sec_shutdown(device_t dev) 500{ 501 502 return (0); 503} 504 505static int 506sec_setup_intr(struct sec_softc *sc, struct resource **ires, void **ihand, 507 int *irid, driver_intr_t handler, const char *iname) 508{ 509 int error; 510 511 (*ires) = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ, irid, 512 RF_ACTIVE); 513 514 if ((*ires) == NULL) { 515 device_printf(sc->sc_dev, "could not allocate %s IRQ\n", iname); 516 return (ENXIO); 517 } 518 519 error = bus_setup_intr(sc->sc_dev, *ires, INTR_MPSAFE | INTR_TYPE_NET, 520 NULL, handler, sc, ihand); 521 522 if (error) { 523 device_printf(sc->sc_dev, "failed to set up %s IRQ\n", iname); 524 if (bus_release_resource(sc->sc_dev, SYS_RES_IRQ, *irid, *ires)) 525 device_printf(sc->sc_dev, "could not release %s IRQ\n", 526 iname); 527 528 (*ires) = NULL; 529 return (error); 530 } 531 532 return (0); 533} 534 535static void 536sec_release_intr(struct sec_softc *sc, struct resource *ires, void *ihand, 537 int irid, const char *iname) 538{ 539 int error; 540 541 if (ires == NULL) 542 return; 543 544 error = bus_teardown_intr(sc->sc_dev, ires, ihand); 545 if (error) 546 device_printf(sc->sc_dev, "bus_teardown_intr() failed for %s" 547 " IRQ, error %d\n", iname, error); 548 549 error = bus_release_resource(sc->sc_dev, SYS_RES_IRQ, irid, ires); 550 if (error) 551 device_printf(sc->sc_dev, "bus_release_resource() failed for %s" 552 " IRQ, error %d\n", iname, error); 553} 554 555static void 556sec_primary_intr(void *arg) 557{ 558 struct sec_softc *sc = arg; 559 struct sec_desc *desc; 560 uint64_t isr; 561 int i, wakeup = 0; 562 563 SEC_LOCK(sc, controller); 564 565 /* Check for errors */ 566 isr = SEC_READ(sc, SEC_ISR); 567 if (isr & sc->sc_int_error_mask) { 568 /* Check each channel for error */ 569 for (i = 0; i < SEC_CHANNELS; i++) { 570 if ((isr & SEC_INT_CH_ERR(i)) == 0) 571 continue; 572 573 device_printf(sc->sc_dev, 574 "I/O error on channel %i!\n", i); 575 576 /* Find and mark problematic descriptor */ 577 desc = sec_find_desc(sc, SEC_READ(sc, 578 SEC_CHAN_CDPR(i))); 579 580 if (desc != NULL) 581 desc->sd_error = EIO; 582 583 /* Do partial channel reset */ 584 sec_channel_reset(sc, i, 0); 585 } 586 } 587 588 /* ACK interrupt */ 589 SEC_WRITE(sc, SEC_ICR, 0xFFFFFFFFFFFFFFFFULL); 590 591 SEC_UNLOCK(sc, controller); 592 SEC_LOCK(sc, descriptors); 593 594 /* Handle processed descriptors */ 595 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 596 597 while (SEC_QUEUED_DESC_CNT(sc) > 0) { 598 desc = SEC_GET_QUEUED_DESC(sc); 599 600 if (desc->sd_desc->shd_done != 0xFF && desc->sd_error == 0) { 601 SEC_PUT_BACK_QUEUED_DESC(sc); 602 break; 603 } 604 605 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_PREREAD | 606 BUS_DMASYNC_PREWRITE); 607 608 desc->sd_crp->crp_etype = desc->sd_error; 609 crypto_done(desc->sd_crp); 610 611 SEC_DESC_FREE_POINTERS(desc); 612 SEC_DESC_FREE_LT(sc, desc); 613 SEC_DESC_QUEUED2FREE(sc); 614 } 615 616 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 617 618 if (!sc->sc_shutdown) { 619 wakeup = sc->sc_blocked; 620 sc->sc_blocked = 0; 621 } 622 623 SEC_UNLOCK(sc, descriptors); 624 625 /* Enqueue ready descriptors in hardware */ 626 sec_enqueue(sc); 627 628 if (wakeup) 629 crypto_unblock(sc->sc_cid, wakeup); 630} 631 632static void 633sec_secondary_intr(void *arg) 634{ 635 struct sec_softc *sc = arg; 636 637 device_printf(sc->sc_dev, "spurious secondary interrupt!\n"); 638 sec_primary_intr(arg); 639} 640 641static int 642sec_controller_reset(struct sec_softc *sc) 643{ 644 int timeout = SEC_TIMEOUT; 645 646 /* Reset Controller */ 647 SEC_WRITE(sc, SEC_MCR, SEC_MCR_SWR); 648 649 while (SEC_READ(sc, SEC_MCR) & SEC_MCR_SWR) { 650 DELAY(1000); 651 timeout -= 1000; 652 653 if (timeout < 0) { 654 device_printf(sc->sc_dev, "timeout while waiting for " 655 "device reset!\n"); 656 return (ETIMEDOUT); 657 } 658 } 659 660 return (0); 661} 662 663static int 664sec_channel_reset(struct sec_softc *sc, int channel, int full) 665{ 666 int timeout = SEC_TIMEOUT; 667 uint64_t bit = (full) ? SEC_CHAN_CCR_R : SEC_CHAN_CCR_CON; 668 uint64_t reg; 669 670 /* Reset Channel */ 671 reg = SEC_READ(sc, SEC_CHAN_CCR(channel)); 672 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg | bit); 673 674 while (SEC_READ(sc, SEC_CHAN_CCR(channel)) & bit) { 675 DELAY(1000); 676 timeout -= 1000; 677 678 if (timeout < 0) { 679 device_printf(sc->sc_dev, "timeout while waiting for " 680 "channel reset!\n"); 681 return (ETIMEDOUT); 682 } 683 } 684 685 if (full) { 686 reg = SEC_CHAN_CCR_CDIE | SEC_CHAN_CCR_NT | SEC_CHAN_CCR_BS; 687 688 switch(sc->sc_version) { 689 case 2: 690 reg |= SEC_CHAN_CCR_CDWE; 691 break; 692 case 3: 693 reg |= SEC_CHAN_CCR_AWSE | SEC_CHAN_CCR_WGN; 694 break; 695 } 696 697 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg); 698 } 699 700 return (0); 701} 702 703static int 704sec_init(struct sec_softc *sc) 705{ 706 uint64_t reg; 707 int error, i; 708 709 /* Reset controller twice to clear all pending interrupts */ 710 error = sec_controller_reset(sc); 711 if (error) 712 return (error); 713 714 error = sec_controller_reset(sc); 715 if (error) 716 return (error); 717 718 /* Reset channels */ 719 for (i = 0; i < SEC_CHANNELS; i++) { 720 error = sec_channel_reset(sc, i, 1); 721 if (error) 722 return (error); 723 } 724 725 /* Enable Interrupts */ 726 reg = SEC_INT_ITO; 727 for (i = 0; i < SEC_CHANNELS; i++) 728 reg |= SEC_INT_CH_DN(i) | SEC_INT_CH_ERR(i); 729 730 SEC_WRITE(sc, SEC_IER, reg); 731 732 return (error); 733} 734 735static void 736sec_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 737{ 738 struct sec_dma_mem *dma_mem = arg; 739 740 if (error) 741 return; 742 743 KASSERT(nseg == 1, ("Wrong number of segments, should be 1")); 744 dma_mem->dma_paddr = segs->ds_addr; 745} 746 747static void 748sec_dma_map_desc_cb(void *arg, bus_dma_segment_t *segs, int nseg, 749 int error) 750{ 751 struct sec_desc_map_info *sdmi = arg; 752 struct sec_softc *sc = sdmi->sdmi_sc; 753 struct sec_lt *lt = NULL; 754 bus_addr_t addr; 755 bus_size_t size; 756 int i; 757 758 SEC_LOCK_ASSERT(sc, descriptors); 759 760 if (error) 761 return; 762 763 for (i = 0; i < nseg; i++) { 764 addr = segs[i].ds_addr; 765 size = segs[i].ds_len; 766 767 /* Skip requested offset */ 768 if (sdmi->sdmi_offset >= size) { 769 sdmi->sdmi_offset -= size; 770 continue; 771 } 772 773 addr += sdmi->sdmi_offset; 774 size -= sdmi->sdmi_offset; 775 sdmi->sdmi_offset = 0; 776 777 /* Do not link more than requested */ 778 if (sdmi->sdmi_size < size) 779 size = sdmi->sdmi_size; 780 781 lt = SEC_ALLOC_LT_ENTRY(sc); 782 lt->sl_lt->shl_length = size; 783 lt->sl_lt->shl_r = 0; 784 lt->sl_lt->shl_n = 0; 785 lt->sl_lt->shl_ptr = addr; 786 787 if (sdmi->sdmi_lt_first == NULL) 788 sdmi->sdmi_lt_first = lt; 789 790 sdmi->sdmi_lt_used += 1; 791 792 if ((sdmi->sdmi_size -= size) == 0) 793 break; 794 } 795 796 sdmi->sdmi_lt_last = lt; 797} 798 799static void 800sec_dma_map_desc_cb2(void *arg, bus_dma_segment_t *segs, int nseg, 801 bus_size_t size, int error) 802{ 803 804 sec_dma_map_desc_cb(arg, segs, nseg, error); 805} 806 807static int 808sec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem, 809 bus_size_t size) 810{ 811 int error; 812 813 if (dma_mem->dma_vaddr != NULL) 814 return (EBUSY); 815 816 error = bus_dma_tag_create(NULL, /* parent */ 817 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 818 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 819 BUS_SPACE_MAXADDR, /* highaddr */ 820 NULL, NULL, /* filtfunc, filtfuncarg */ 821 size, 1, /* maxsize, nsegments */ 822 size, 0, /* maxsegsz, flags */ 823 NULL, NULL, /* lockfunc, lockfuncarg */ 824 &(dma_mem->dma_tag)); /* dmat */ 825 826 if (error) { 827 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 828 " %i!\n", error); 829 goto err1; 830 } 831 832 error = bus_dmamem_alloc(dma_mem->dma_tag, &(dma_mem->dma_vaddr), 833 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &(dma_mem->dma_map)); 834 835 if (error) { 836 device_printf(sc->sc_dev, "failed to allocate DMA safe" 837 " memory, error %i!\n", error); 838 goto err2; 839 } 840 841 error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map, 842 dma_mem->dma_vaddr, size, sec_alloc_dma_mem_cb, dma_mem, 843 BUS_DMA_NOWAIT); 844 845 if (error) { 846 device_printf(sc->sc_dev, "cannot get address of the DMA" 847 " memory, error %i\n", error); 848 goto err3; 849 } 850 851 dma_mem->dma_is_map = 0; 852 return (0); 853 854err3: 855 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, dma_mem->dma_map); 856err2: 857 bus_dma_tag_destroy(dma_mem->dma_tag); 858err1: 859 dma_mem->dma_vaddr = NULL; 860 return(error); 861} 862 863static int 864sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, void *mem, 865 bus_size_t size, int type, struct sec_desc_map_info *sdmi) 866{ 867 int error; 868 869 if (dma_mem->dma_vaddr != NULL) 870 return (EBUSY); 871 872 switch (type) { 873 case SEC_MEMORY: 874 break; 875 case SEC_UIO: 876 size = SEC_FREE_LT_CNT(sc) * SEC_MAX_DMA_BLOCK_SIZE; 877 break; 878 case SEC_MBUF: 879 size = m_length((struct mbuf*)mem, NULL); 880 break; 881 default: 882 return (EINVAL); 883 } 884 885 error = bus_dma_tag_create(NULL, /* parent */ 886 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 887 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 888 BUS_SPACE_MAXADDR, /* highaddr */ 889 NULL, NULL, /* filtfunc, filtfuncarg */ 890 size, /* maxsize */ 891 SEC_FREE_LT_CNT(sc), /* nsegments */ 892 SEC_MAX_DMA_BLOCK_SIZE, 0, /* maxsegsz, flags */ 893 NULL, NULL, /* lockfunc, lockfuncarg */ 894 &(dma_mem->dma_tag)); /* dmat */ 895 896 if (error) { 897 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 898 " %i!\n", error); 899 dma_mem->dma_vaddr = NULL; 900 return (error); 901 } 902 903 error = bus_dmamap_create(dma_mem->dma_tag, 0, &(dma_mem->dma_map)); 904 905 if (error) { 906 device_printf(sc->sc_dev, "failed to create DMA map, error %i!" 907 "\n", error); 908 bus_dma_tag_destroy(dma_mem->dma_tag); 909 return (error); 910 } 911 912 switch (type) { 913 case SEC_MEMORY: 914 error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map, 915 mem, size, sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT); 916 break; 917 case SEC_UIO: 918 error = bus_dmamap_load_uio(dma_mem->dma_tag, dma_mem->dma_map, 919 mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT); 920 break; 921 case SEC_MBUF: 922 error = bus_dmamap_load_mbuf(dma_mem->dma_tag, dma_mem->dma_map, 923 mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT); 924 break; 925 } 926 927 if (error) { 928 device_printf(sc->sc_dev, "cannot get address of the DMA" 929 " memory, error %i!\n", error); 930 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 931 bus_dma_tag_destroy(dma_mem->dma_tag); 932 return (error); 933 } 934 935 dma_mem->dma_is_map = 1; 936 dma_mem->dma_vaddr = mem; 937 938 return (0); 939} 940 941static void 942sec_free_dma_mem(struct sec_dma_mem *dma_mem) 943{ 944 945 /* Check for double free */ 946 if (dma_mem->dma_vaddr == NULL) 947 return; 948 949 bus_dmamap_unload(dma_mem->dma_tag, dma_mem->dma_map); 950 951 if (dma_mem->dma_is_map) 952 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 953 else 954 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, 955 dma_mem->dma_map); 956 957 bus_dma_tag_destroy(dma_mem->dma_tag); 958 dma_mem->dma_vaddr = NULL; 959} 960 961static int 962sec_eu_channel(struct sec_softc *sc, int eu) 963{ 964 uint64_t reg; 965 int channel = 0; 966 967 SEC_LOCK_ASSERT(sc, controller); 968 969 reg = SEC_READ(sc, SEC_EUASR); 970 971 switch (eu) { 972 case SEC_EU_AFEU: 973 channel = SEC_EUASR_AFEU(reg); 974 break; 975 case SEC_EU_DEU: 976 channel = SEC_EUASR_DEU(reg); 977 break; 978 case SEC_EU_MDEU_A: 979 case SEC_EU_MDEU_B: 980 channel = SEC_EUASR_MDEU(reg); 981 break; 982 case SEC_EU_RNGU: 983 channel = SEC_EUASR_RNGU(reg); 984 break; 985 case SEC_EU_PKEU: 986 channel = SEC_EUASR_PKEU(reg); 987 break; 988 case SEC_EU_AESU: 989 channel = SEC_EUASR_AESU(reg); 990 break; 991 case SEC_EU_KEU: 992 channel = SEC_EUASR_KEU(reg); 993 break; 994 case SEC_EU_CRCU: 995 channel = SEC_EUASR_CRCU(reg); 996 break; 997 } 998 999 return (channel - 1); 1000} 1001 1002static int 1003sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, int channel) 1004{ 1005 u_int fflvl = SEC_MAX_FIFO_LEVEL; 1006 uint64_t reg; 1007 int i; 1008 1009 SEC_LOCK_ASSERT(sc, controller); 1010 1011 /* Find free channel if have not got one */ 1012 if (channel < 0) { 1013 for (i = 0; i < SEC_CHANNELS; i++) { 1014 reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 1015 1016 if ((reg & sc->sc_channel_idle_mask) == 0) { 1017 channel = i; 1018 break; 1019 } 1020 } 1021 } 1022 1023 /* There is no free channel */ 1024 if (channel < 0) 1025 return (-1); 1026 1027 /* Check FIFO level on selected channel */ 1028 reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 1029 1030 switch(sc->sc_version) { 1031 case 2: 1032 fflvl = (reg >> SEC_CHAN_CSR2_FFLVL_S) & SEC_CHAN_CSR2_FFLVL_M; 1033 break; 1034 case 3: 1035 fflvl = (reg >> SEC_CHAN_CSR3_FFLVL_S) & SEC_CHAN_CSR3_FFLVL_M; 1036 break; 1037 } 1038 1039 if (fflvl >= SEC_MAX_FIFO_LEVEL) 1040 return (-1); 1041 1042 /* Enqueue descriptor in channel */ 1043 SEC_WRITE(sc, SEC_CHAN_FF(channel), desc->sd_desc_paddr); 1044 1045 return (channel); 1046} 1047 1048static void 1049sec_enqueue(struct sec_softc *sc) 1050{ 1051 struct sec_desc *desc; 1052 int ch0, ch1; 1053 1054 SEC_LOCK(sc, descriptors); 1055 SEC_LOCK(sc, controller); 1056 1057 while (SEC_READY_DESC_CNT(sc) > 0) { 1058 desc = SEC_GET_READY_DESC(sc); 1059 1060 ch0 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel0); 1061 ch1 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel1); 1062 1063 /* 1064 * Both EU are used by the same channel. 1065 * Enqueue descriptor in channel used by busy EUs. 1066 */ 1067 if (ch0 >= 0 && ch0 == ch1) { 1068 if (sec_enqueue_desc(sc, desc, ch0) >= 0) { 1069 SEC_DESC_READY2QUEUED(sc); 1070 continue; 1071 } 1072 } 1073 1074 /* 1075 * Only one EU is free. 1076 * Enqueue descriptor in channel used by busy EU. 1077 */ 1078 if ((ch0 >= 0 && ch1 < 0) || (ch1 >= 0 && ch0 < 0)) { 1079 if (sec_enqueue_desc(sc, desc, (ch0 >= 0) ? ch0 : ch1) 1080 >= 0) { 1081 SEC_DESC_READY2QUEUED(sc); 1082 continue; 1083 } 1084 } 1085 1086 /* 1087 * Both EU are free. 1088 * Enqueue descriptor in first free channel. 1089 */ 1090 if (ch0 < 0 && ch1 < 0) { 1091 if (sec_enqueue_desc(sc, desc, -1) >= 0) { 1092 SEC_DESC_READY2QUEUED(sc); 1093 continue; 1094 } 1095 } 1096 1097 /* Current descriptor can not be queued at the moment */ 1098 SEC_PUT_BACK_READY_DESC(sc); 1099 break; 1100 } 1101 1102 SEC_UNLOCK(sc, controller); 1103 SEC_UNLOCK(sc, descriptors); 1104} 1105 1106static struct sec_desc * 1107sec_find_desc(struct sec_softc *sc, bus_addr_t paddr) 1108{ 1109 struct sec_desc *desc = NULL; 1110 int i; 1111 1112 SEC_LOCK_ASSERT(sc, descriptors); 1113 1114 for (i = 0; i < SEC_CHANNELS; i++) { 1115 if (sc->sc_desc[i].sd_desc_paddr == paddr) { 1116 desc = &(sc->sc_desc[i]); 1117 break; 1118 } 1119 } 1120 1121 return (desc); 1122} 1123 1124static int 1125sec_make_pointer_direct(struct sec_softc *sc, struct sec_desc *desc, u_int n, 1126 bus_addr_t data, bus_size_t dsize) 1127{ 1128 struct sec_hw_desc_ptr *ptr; 1129 1130 SEC_LOCK_ASSERT(sc, descriptors); 1131 1132 ptr = &(desc->sd_desc->shd_pointer[n]); 1133 ptr->shdp_length = dsize; 1134 ptr->shdp_extent = 0; 1135 ptr->shdp_j = 0; 1136 ptr->shdp_ptr = data; 1137 1138 return (0); 1139} 1140 1141static int 1142sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, 1143 u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype) 1144{ 1145 struct sec_desc_map_info sdmi = { sc, dsize, doffset, NULL, NULL, 0 }; 1146 struct sec_hw_desc_ptr *ptr; 1147 int error; 1148 1149 SEC_LOCK_ASSERT(sc, descriptors); 1150 1151 /* For flat memory map only requested region */ 1152 if (dtype == SEC_MEMORY) { 1153 data = (uint8_t*)(data) + doffset; 1154 sdmi.sdmi_offset = 0; 1155 } 1156 1157 error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), data, dsize, 1158 dtype, &sdmi); 1159 1160 if (error) 1161 return (error); 1162 1163 sdmi.sdmi_lt_last->sl_lt->shl_r = 1; 1164 desc->sd_lt_used += sdmi.sdmi_lt_used; 1165 1166 ptr = &(desc->sd_desc->shd_pointer[n]); 1167 ptr->shdp_length = dsize; 1168 ptr->shdp_extent = 0; 1169 ptr->shdp_j = 1; 1170 ptr->shdp_ptr = sdmi.sdmi_lt_first->sl_lt_paddr; 1171 1172 return (0); 1173} 1174 1175static int 1176sec_split_cri(struct cryptoini *cri, struct cryptoini **enc, 1177 struct cryptoini **mac) 1178{ 1179 struct cryptoini *e, *m; 1180 1181 e = cri; 1182 m = cri->cri_next; 1183 1184 /* We can haldle only two operations */ 1185 if (m && m->cri_next) 1186 return (EINVAL); 1187 1188 if (sec_mdeu_can_handle(e->cri_alg)) { 1189 cri = m; 1190 m = e; 1191 e = cri; 1192 } 1193 1194 if (m && !sec_mdeu_can_handle(m->cri_alg)) 1195 return (EINVAL); 1196 1197 *enc = e; 1198 *mac = m; 1199 1200 return (0); 1201} 1202 1203static int 1204sec_split_crp(struct cryptop *crp, struct cryptodesc **enc, 1205 struct cryptodesc **mac) 1206{ 1207 struct cryptodesc *e, *m, *t; 1208 1209 e = crp->crp_desc; 1210 m = e->crd_next; 1211 1212 /* We can haldle only two operations */ 1213 if (m && m->crd_next) 1214 return (EINVAL); 1215 1216 if (sec_mdeu_can_handle(e->crd_alg)) { 1217 t = m; 1218 m = e; 1219 e = t; 1220 } 1221 1222 if (m && !sec_mdeu_can_handle(m->crd_alg)) 1223 return (EINVAL); 1224 1225 *enc = e; 1226 *mac = m; 1227 1228 return (0); 1229} 1230 1231static int 1232sec_alloc_session(struct sec_softc *sc) 1233{ 1234 struct sec_session *ses = NULL; 1235 int sid = -1; 1236 u_int i; 1237 1238 SEC_LOCK(sc, sessions); 1239 1240 for (i = 0; i < SEC_MAX_SESSIONS; i++) { 1241 if (sc->sc_sessions[i].ss_used == 0) { 1242 ses = &(sc->sc_sessions[i]); 1243 ses->ss_used = 1; 1244 ses->ss_ivlen = 0; 1245 ses->ss_klen = 0; 1246 ses->ss_mklen = 0; 1247 sid = i; 1248 break; 1249 } 1250 } 1251 1252 SEC_UNLOCK(sc, sessions); 1253 1254 return (sid); 1255} 1256 1257static struct sec_session * 1258sec_get_session(struct sec_softc *sc, u_int sid) 1259{ 1260 struct sec_session *ses; 1261 1262 if (sid >= SEC_MAX_SESSIONS) 1263 return (NULL); 1264 1265 SEC_LOCK(sc, sessions); 1266 1267 ses = &(sc->sc_sessions[sid]); 1268 1269 if (ses->ss_used == 0) 1270 ses = NULL; 1271 1272 SEC_UNLOCK(sc, sessions); 1273 1274 return (ses); 1275} 1276 1277static int 1278sec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri) 1279{ 1280 struct sec_softc *sc = device_get_softc(dev); 1281 struct sec_eu_methods *eu = sec_eus; 1282 struct cryptoini *enc = NULL; 1283 struct cryptoini *mac = NULL; 1284 struct sec_session *ses; 1285 int error = -1; 1286 int sid; 1287 1288 error = sec_split_cri(cri, &enc, &mac); 1289 if (error) 1290 return (error); 1291 1292 /* Check key lengths */ 1293 if (enc && enc->cri_key && (enc->cri_klen / 8) > SEC_MAX_KEY_LEN) 1294 return (E2BIG); 1295 1296 if (mac && mac->cri_key && (mac->cri_klen / 8) > SEC_MAX_KEY_LEN) 1297 return (E2BIG); 1298 1299 /* Only SEC 3.0 supports digests larger than 256 bits */ 1300 if (sc->sc_version < 3 && mac && mac->cri_klen > 256) 1301 return (E2BIG); 1302 1303 sid = sec_alloc_session(sc); 1304 if (sid < 0) 1305 return (ENOMEM); 1306 1307 ses = sec_get_session(sc, sid); 1308 1309 /* Find EU for this session */ 1310 while (eu->sem_make_desc != NULL) { 1311 error = eu->sem_newsession(sc, ses, enc, mac); 1312 if (error >= 0) 1313 break; 1314 1315 eu++; 1316 } 1317 1318 /* If not found, return EINVAL */ 1319 if (error < 0) { 1320 sec_free_session(sc, ses); 1321 return (EINVAL); 1322 } 1323 1324 /* Save cipher key */ 1325 if (enc && enc->cri_key) { 1326 ses->ss_klen = enc->cri_klen / 8; 1327 memcpy(ses->ss_key, enc->cri_key, ses->ss_klen); 1328 } 1329 1330 /* Save digest key */ 1331 if (mac && mac->cri_key) { 1332 ses->ss_mklen = mac->cri_klen / 8; 1333 memcpy(ses->ss_mkey, mac->cri_key, ses->ss_mklen); 1334 } 1335 1336 ses->ss_eu = eu; 1337 *sidp = sid; 1338 1339 return (0); 1340} 1341 1342static int 1343sec_freesession(device_t dev, uint64_t tid) 1344{ 1345 struct sec_softc *sc = device_get_softc(dev); 1346 struct sec_session *ses; 1347 int error = 0; 1348 1349 ses = sec_get_session(sc, CRYPTO_SESID2LID(tid)); 1350 if (ses == NULL) 1351 return (EINVAL); 1352 1353 sec_free_session(sc, ses); 1354 1355 return (error); 1356} 1357 1358static int 1359sec_process(device_t dev, struct cryptop *crp, int hint) 1360{ 1361 struct sec_softc *sc = device_get_softc(dev); 1362 struct sec_desc *desc = NULL; 1363 struct cryptodesc *mac, *enc; 1364 struct sec_session *ses; 1365 int buftype, error = 0; 1366 1367 /* Check Session ID */ 1368 ses = sec_get_session(sc, CRYPTO_SESID2LID(crp->crp_sid)); 1369 if (ses == NULL) { 1370 crp->crp_etype = EINVAL; 1371 crypto_done(crp); 1372 return (0); 1373 } 1374 1375 /* Check for input length */ 1376 if (crp->crp_ilen > SEC_MAX_DMA_BLOCK_SIZE) { 1377 crp->crp_etype = E2BIG; 1378 crypto_done(crp); 1379 return (0); 1380 } 1381 1382 /* Get descriptors */ 1383 if (sec_split_crp(crp, &enc, &mac)) { 1384 crp->crp_etype = EINVAL; 1385 crypto_done(crp); 1386 return (0); 1387 } 1388 1389 SEC_LOCK(sc, descriptors); 1390 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1391 1392 /* Block driver if there is no free descriptors or we are going down */ 1393 if (SEC_FREE_DESC_CNT(sc) == 0 || sc->sc_shutdown) { 1394 sc->sc_blocked |= CRYPTO_SYMQ; 1395 SEC_UNLOCK(sc, descriptors); 1396 return (ERESTART); 1397 } 1398 1399 /* Prepare descriptor */ 1400 desc = SEC_GET_FREE_DESC(sc); 1401 desc->sd_lt_used = 0; 1402 desc->sd_error = 0; 1403 desc->sd_crp = crp; 1404 1405 if (crp->crp_flags & CRYPTO_F_IOV) 1406 buftype = SEC_UIO; 1407 else if (crp->crp_flags & CRYPTO_F_IMBUF) 1408 buftype = SEC_MBUF; 1409 else 1410 buftype = SEC_MEMORY; 1411 1412 if (enc && enc->crd_flags & CRD_F_ENCRYPT) { 1413 if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1414 memcpy(desc->sd_desc->shd_iv, enc->crd_iv, 1415 ses->ss_ivlen); 1416 else 1417 arc4rand(desc->sd_desc->shd_iv, ses->ss_ivlen, 0); 1418 1419 if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0) 1420 crypto_copyback(crp->crp_flags, crp->crp_buf, 1421 enc->crd_inject, ses->ss_ivlen, 1422 desc->sd_desc->shd_iv); 1423 } else if (enc) { 1424 if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1425 memcpy(desc->sd_desc->shd_iv, enc->crd_iv, 1426 ses->ss_ivlen); 1427 else 1428 crypto_copydata(crp->crp_flags, crp->crp_buf, 1429 enc->crd_inject, ses->ss_ivlen, 1430 desc->sd_desc->shd_iv); 1431 } 1432 1433 if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) { 1434 if ((enc->crd_klen / 8) <= SEC_MAX_KEY_LEN) { 1435 ses->ss_klen = enc->crd_klen / 8; 1436 memcpy(ses->ss_key, enc->crd_key, ses->ss_klen); 1437 } else 1438 error = E2BIG; 1439 } 1440 1441 if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) { 1442 if ((mac->crd_klen / 8) <= SEC_MAX_KEY_LEN) { 1443 ses->ss_mklen = mac->crd_klen / 8; 1444 memcpy(ses->ss_mkey, mac->crd_key, ses->ss_mklen); 1445 } else 1446 error = E2BIG; 1447 } 1448 1449 if (!error) { 1450 memcpy(desc->sd_desc->shd_key, ses->ss_key, ses->ss_klen); 1451 memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, ses->ss_mklen); 1452 1453 error = ses->ss_eu->sem_make_desc(sc, ses, desc, crp, buftype); 1454 } 1455 1456 if (error) { 1457 SEC_DESC_FREE_POINTERS(desc); 1458 SEC_DESC_PUT_BACK_LT(sc, desc); 1459 SEC_PUT_BACK_FREE_DESC(sc); 1460 SEC_UNLOCK(sc, descriptors); 1461 crp->crp_etype = error; 1462 crypto_done(crp); 1463 return (0); 1464 } 1465 1466 /* 1467 * Skip DONE interrupt if this is not last request in burst, but only 1468 * if we are running on SEC 3.X. On SEC 2.X we have to enable DONE 1469 * signaling on each descriptor. 1470 */ 1471 if ((hint & CRYPTO_HINT_MORE) && sc->sc_version == 3) 1472 desc->sd_desc->shd_dn = 0; 1473 else 1474 desc->sd_desc->shd_dn = 1; 1475 1476 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1477 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_POSTREAD | 1478 BUS_DMASYNC_POSTWRITE); 1479 SEC_DESC_FREE2READY(sc); 1480 SEC_UNLOCK(sc, descriptors); 1481 1482 /* Enqueue ready descriptors in hardware */ 1483 sec_enqueue(sc); 1484 1485 return (0); 1486} 1487 1488static int 1489sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc, 1490 struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc, 1491 int buftype) 1492{ 1493 struct sec_hw_desc *hd = desc->sd_desc; 1494 int error; 1495 1496 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 1497 hd->shd_eu_sel1 = SEC_EU_NONE; 1498 hd->shd_mode1 = 0; 1499 1500 /* Pointer 0: NULL */ 1501 error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 1502 if (error) 1503 return (error); 1504 1505 /* Pointer 1: IV IN */ 1506 error = sec_make_pointer_direct(sc, desc, 1, desc->sd_desc_paddr + 1507 offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen); 1508 if (error) 1509 return (error); 1510 1511 /* Pointer 2: Cipher Key */ 1512 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 1513 offsetof(struct sec_hw_desc, shd_key), ses->ss_klen); 1514 if (error) 1515 return (error); 1516 1517 /* Pointer 3: Data IN */ 1518 error = sec_make_pointer(sc, desc, 3, crp->crp_buf, enc->crd_skip, 1519 enc->crd_len, buftype); 1520 if (error) 1521 return (error); 1522 1523 /* Pointer 4: Data OUT */ 1524 error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip, 1525 enc->crd_len, buftype); 1526 if (error) 1527 return (error); 1528 1529 /* Pointer 5: IV OUT (Not used: NULL) */ 1530 error = sec_make_pointer_direct(sc, desc, 5, 0, 0); 1531 if (error) 1532 return (error); 1533 1534 /* Pointer 6: NULL */ 1535 error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 1536 1537 return (error); 1538} 1539 1540static int 1541sec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc, 1542 struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc, 1543 struct cryptodesc *mac, int buftype) 1544{ 1545 struct sec_hw_desc *hd = desc->sd_desc; 1546 u_int eu, mode, hashlen; 1547 int error; 1548 1549 if (mac->crd_len < enc->crd_len) 1550 return (EINVAL); 1551 1552 if (mac->crd_skip + mac->crd_len != enc->crd_skip + enc->crd_len) 1553 return (EINVAL); 1554 1555 error = sec_mdeu_config(mac, &eu, &mode, &hashlen); 1556 if (error) 1557 return (error); 1558 1559 hd->shd_desc_type = SEC_DT_HMAC_SNOOP; 1560 hd->shd_eu_sel1 = eu; 1561 hd->shd_mode1 = mode; 1562 1563 /* Pointer 0: HMAC Key */ 1564 error = sec_make_pointer_direct(sc, desc, 0, desc->sd_desc_paddr + 1565 offsetof(struct sec_hw_desc, shd_mkey), ses->ss_mklen); 1566 if (error) 1567 return (error); 1568 1569 /* Pointer 1: HMAC-Only Data IN */ 1570 error = sec_make_pointer(sc, desc, 1, crp->crp_buf, mac->crd_skip, 1571 mac->crd_len - enc->crd_len, buftype); 1572 if (error) 1573 return (error); 1574 1575 /* Pointer 2: Cipher Key */ 1576 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 1577 offsetof(struct sec_hw_desc, shd_key), ses->ss_klen); 1578 if (error) 1579 return (error); 1580 1581 /* Pointer 3: IV IN */ 1582 error = sec_make_pointer_direct(sc, desc, 3, desc->sd_desc_paddr + 1583 offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen); 1584 if (error) 1585 return (error); 1586 1587 /* Pointer 4: Data IN */ 1588 error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip, 1589 enc->crd_len, buftype); 1590 if (error) 1591 return (error); 1592 1593 /* Pointer 5: Data OUT */ 1594 error = sec_make_pointer(sc, desc, 5, crp->crp_buf, enc->crd_skip, 1595 enc->crd_len, buftype); 1596 if (error) 1597 return (error); 1598 1599 /* Pointer 6: HMAC OUT */ 1600 error = sec_make_pointer(sc, desc, 6, crp->crp_buf, mac->crd_inject, 1601 hashlen, buftype); 1602 1603 return (error); 1604} 1605 1606/* AESU */ 1607 1608static int 1609sec_aesu_newsession(struct sec_softc *sc, struct sec_session *ses, 1610 struct cryptoini *enc, struct cryptoini *mac) 1611{ 1612 1613 if (enc == NULL) 1614 return (-1); 1615 1616 if (enc->cri_alg != CRYPTO_AES_CBC) 1617 return (-1); 1618 1619 ses->ss_ivlen = AES_BLOCK_LEN; 1620 1621 return (0); 1622} 1623 1624static int 1625sec_aesu_make_desc(struct sec_softc *sc, struct sec_session *ses, 1626 struct sec_desc *desc, struct cryptop *crp, int buftype) 1627{ 1628 struct sec_hw_desc *hd = desc->sd_desc; 1629 struct cryptodesc *enc, *mac; 1630 int error; 1631 1632 error = sec_split_crp(crp, &enc, &mac); 1633 if (error) 1634 return (error); 1635 1636 if (!enc) 1637 return (EINVAL); 1638 1639 hd->shd_eu_sel0 = SEC_EU_AESU; 1640 hd->shd_mode0 = SEC_AESU_MODE_CBC; 1641 1642 if (enc->crd_alg != CRYPTO_AES_CBC) 1643 return (EINVAL); 1644 1645 if (enc->crd_flags & CRD_F_ENCRYPT) { 1646 hd->shd_mode0 |= SEC_AESU_MODE_ED; 1647 hd->shd_dir = 0; 1648 } else 1649 hd->shd_dir = 1; 1650 1651 if (mac) 1652 error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac, 1653 buftype); 1654 else 1655 error = sec_build_common_ns_desc(sc, desc, ses, crp, enc, 1656 buftype); 1657 1658 return (error); 1659} 1660 1661/* DEU */ 1662 1663static int 1664sec_deu_newsession(struct sec_softc *sc, struct sec_session *ses, 1665 struct cryptoini *enc, struct cryptoini *mac) 1666{ 1667 1668 if (enc == NULL) 1669 return (-1); 1670 1671 switch (enc->cri_alg) { 1672 case CRYPTO_DES_CBC: 1673 case CRYPTO_3DES_CBC: 1674 break; 1675 default: 1676 return (-1); 1677 } 1678 1679 ses->ss_ivlen = DES_BLOCK_LEN; 1680 1681 return (0); 1682} 1683 1684static int 1685sec_deu_make_desc(struct sec_softc *sc, struct sec_session *ses, 1686 struct sec_desc *desc, struct cryptop *crp, int buftype) 1687{ 1688 struct sec_hw_desc *hd = desc->sd_desc; 1689 struct cryptodesc *enc, *mac; 1690 int error; 1691 1692 error = sec_split_crp(crp, &enc, &mac); 1693 if (error) 1694 return (error); 1695 1696 if (!enc) 1697 return (EINVAL); 1698 1699 hd->shd_eu_sel0 = SEC_EU_DEU; 1700 hd->shd_mode0 = SEC_DEU_MODE_CBC; 1701 1702 switch (enc->crd_alg) { 1703 case CRYPTO_3DES_CBC: 1704 hd->shd_mode0 |= SEC_DEU_MODE_TS; 1705 break; 1706 case CRYPTO_DES_CBC: 1707 break; 1708 default: 1709 return (EINVAL); 1710 } 1711 1712 if (enc->crd_flags & CRD_F_ENCRYPT) { 1713 hd->shd_mode0 |= SEC_DEU_MODE_ED; 1714 hd->shd_dir = 0; 1715 } else 1716 hd->shd_dir = 1; 1717 1718 if (mac) 1719 error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac, 1720 buftype); 1721 else 1722 error = sec_build_common_ns_desc(sc, desc, ses, crp, enc, 1723 buftype); 1724 1725 return (error); 1726} 1727 1728/* MDEU */ 1729 1730static int 1731sec_mdeu_can_handle(u_int alg) 1732{ 1733 switch (alg) { 1734 case CRYPTO_MD5: 1735 case CRYPTO_SHA1: 1736 case CRYPTO_MD5_HMAC: 1737 case CRYPTO_SHA1_HMAC: 1738 case CRYPTO_SHA2_256_HMAC: 1739 case CRYPTO_SHA2_384_HMAC: 1740 case CRYPTO_SHA2_512_HMAC: 1741 return (1); 1742 default: 1743 return (0); 1744 } 1745} 1746 1747static int 1748sec_mdeu_config(struct cryptodesc *crd, u_int *eu, u_int *mode, u_int *hashlen) 1749{ 1750 1751 *mode = SEC_MDEU_MODE_PD | SEC_MDEU_MODE_INIT; 1752 *eu = SEC_EU_NONE; 1753 1754 switch (crd->crd_alg) { 1755 case CRYPTO_MD5_HMAC: 1756 *mode |= SEC_MDEU_MODE_HMAC; 1757 /* FALLTHROUGH */ 1758 case CRYPTO_MD5: 1759 *eu = SEC_EU_MDEU_A; 1760 *mode |= SEC_MDEU_MODE_MD5; 1761 *hashlen = MD5_HASH_LEN; 1762 break; 1763 case CRYPTO_SHA1_HMAC: 1764 *mode |= SEC_MDEU_MODE_HMAC; 1765 /* FALLTHROUGH */ 1766 case CRYPTO_SHA1: 1767 *eu = SEC_EU_MDEU_A; 1768 *mode |= SEC_MDEU_MODE_SHA1; 1769 *hashlen = SHA1_HASH_LEN; 1770 break; 1771 case CRYPTO_SHA2_256_HMAC: 1772 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA256; 1773 *eu = SEC_EU_MDEU_A; 1774 break; 1775 case CRYPTO_SHA2_384_HMAC: 1776 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA384; 1777 *eu = SEC_EU_MDEU_B; 1778 break; 1779 case CRYPTO_SHA2_512_HMAC: 1780 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA512; 1781 *eu = SEC_EU_MDEU_B; 1782 break; 1783 default: 1784 return (EINVAL); 1785 } 1786 1787 if (*mode & SEC_MDEU_MODE_HMAC) 1788 *hashlen = SEC_HMAC_HASH_LEN; 1789 1790 return (0); 1791} 1792 1793static int 1794sec_mdeu_newsession(struct sec_softc *sc, struct sec_session *ses, 1795 struct cryptoini *enc, struct cryptoini *mac) 1796{ 1797 1798 if (mac && sec_mdeu_can_handle(mac->cri_alg)) 1799 return (0); 1800 1801 return (-1); 1802} 1803 1804static int 1805sec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses, 1806 struct sec_desc *desc, struct cryptop *crp, int buftype) 1807{ 1808 struct cryptodesc *enc, *mac; 1809 struct sec_hw_desc *hd = desc->sd_desc; 1810 u_int eu, mode, hashlen; 1811 int error; 1812 1813 error = sec_split_crp(crp, &enc, &mac); 1814 if (error) 1815 return (error); 1816 1817 if (enc) 1818 return (EINVAL); 1819 1820 error = sec_mdeu_config(mac, &eu, &mode, &hashlen); 1821 if (error) 1822 return (error); 1823 1824 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 1825 hd->shd_eu_sel0 = eu; 1826 hd->shd_mode0 = mode; 1827 hd->shd_eu_sel1 = SEC_EU_NONE; 1828 hd->shd_mode1 = 0; 1829 1830 /* Pointer 0: NULL */ 1831 error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 1832 if (error) 1833 return (error); 1834 1835 /* Pointer 1: Context In (Not used: NULL) */ 1836 error = sec_make_pointer_direct(sc, desc, 1, 0, 0); 1837 if (error) 1838 return (error); 1839 1840 /* Pointer 2: HMAC Key (or NULL, depending on digest type) */ 1841 if (hd->shd_mode0 & SEC_MDEU_MODE_HMAC) 1842 error = sec_make_pointer_direct(sc, desc, 2, 1843 desc->sd_desc_paddr + offsetof(struct sec_hw_desc, 1844 shd_mkey), ses->ss_mklen); 1845 else 1846 error = sec_make_pointer_direct(sc, desc, 2, 0, 0); 1847 1848 if (error) 1849 return (error); 1850 1851 /* Pointer 3: Input Data */ 1852 error = sec_make_pointer(sc, desc, 3, crp->crp_buf, mac->crd_skip, 1853 mac->crd_len, buftype); 1854 if (error) 1855 return (error); 1856 1857 /* Pointer 4: NULL */ 1858 error = sec_make_pointer_direct(sc, desc, 4, 0, 0); 1859 if (error) 1860 return (error); 1861 1862 /* Pointer 5: Hash out */ 1863 error = sec_make_pointer(sc, desc, 5, crp->crp_buf, 1864 mac->crd_inject, hashlen, buftype); 1865 if (error) 1866 return (error); 1867 1868 /* Pointer 6: NULL */ 1869 error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 1870 1871 return (0); 1872}
|