Deleted Added
sdiff udiff text old ( 108466 ) new ( 108471 )
full compact
ubsec.c (108466) ubsec.c (108471)
1/* $FreeBSD: head/sys/dev/ubsec/ubsec.c 108466 2002-12-30 20:22:40Z sam $ */
1/* $FreeBSD: head/sys/dev/ubsec/ubsec.c 108471 2002-12-30 22:16:45Z sam $ */
2/* $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $ */
3
4/*
5 * Copyright (c) 2000 Jason L. Wright (jason@thought.net)
6 * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org)
7 * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com)
8 *
9 * All rights reserved.

--- 166 unchanged lines hidden (view full) ---

176
177#define SWAP32(x) (x) = htole32(ntohl((x)))
178#define HTOLE32(x) (x) = htole32(x)
179
180
181struct ubsec_stats ubsecstats;
182SYSCTL_STRUCT(_kern, OID_AUTO, ubsec_stats, CTLFLAG_RD, &ubsecstats,
183 ubsec_stats, "Broadcom driver statistics");
2/* $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $ */
3
4/*
5 * Copyright (c) 2000 Jason L. Wright (jason@thought.net)
6 * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org)
7 * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com)
8 *
9 * All rights reserved.

--- 166 unchanged lines hidden (view full) ---

176
177#define SWAP32(x) (x) = htole32(ntohl((x)))
178#define HTOLE32(x) (x) = htole32(x)
179
180
181struct ubsec_stats ubsecstats;
182SYSCTL_STRUCT(_kern, OID_AUTO, ubsec_stats, CTLFLAG_RD, &ubsecstats,
183 ubsec_stats, "Broadcom driver statistics");
184static int ubsec_maxbatch = 2; /* XXX tune based on part+sys speed */
184/*
185 * ubsec_maxbatch controls the number of crypto ops to voluntarily
186 * collect into one submission to the hardware. This batching happens
187 * when ops are dispatched from the crypto subsystem with a hint that
188 * more are to follow immediately. These ops must also not be marked
189 * with a ``no delay'' flag.
190 */
191static int ubsec_maxbatch = 1;
185SYSCTL_INT(_kern, OID_AUTO, ubsec_maxbatch, CTLFLAG_RW, &ubsec_maxbatch,
186 0, "Broadcom driver: max ops to batch w/o interrupt");
192SYSCTL_INT(_kern, OID_AUTO, ubsec_maxbatch, CTLFLAG_RW, &ubsec_maxbatch,
193 0, "Broadcom driver: max ops to batch w/o interrupt");
194/*
195 * ubsec_maxaggr controls the number of crypto ops to submit to the
196 * hardware as a unit. This aggregation reduces the number of interrupts
197 * to the host at the expense of increased latency (for all but the last
198 * operation). For network traffic setting this to one yields the highest
199 * performance but at the expense of more interrupt processing.
200 */
201static int ubsec_maxaggr = 1;
202SYSCTL_INT(_kern, OID_AUTO, ubsec_maxaggr, CTLFLAG_RW, &ubsec_maxaggr,
203 0, "Broadcom driver: max ops to aggregate under one interrupt");
187
188static int
189ubsec_probe(device_t dev)
190{
191 if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
192 (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 ||
193 pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601))
194 return (0);

--- 361 unchanged lines hidden (view full) ---

556 dmap = q->q_dma;
557
558 if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0)
559 break;
560
561 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next);
562
563 npkts = q->q_nstacked_mcrs;
204
205static int
206ubsec_probe(device_t dev)
207{
208 if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
209 (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 ||
210 pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601))
211 return (0);

--- 361 unchanged lines hidden (view full) ---

573 dmap = q->q_dma;
574
575 if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0)
576 break;
577
578 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next);
579
580 npkts = q->q_nstacked_mcrs;
581 sc->sc_nqchip -= 1+npkts;
564 /*
565 * search for further sc_qchip ubsec_q's that share
566 * the same MCR, and complete them too, they must be
567 * at the top.
568 */
569 for (i = 0; i < npkts; i++) {
570 if(q->q_stacked_mcr[i]) {
571 ubsec_callback(sc, q->q_stacked_mcr[i]);

--- 84 unchanged lines hidden (view full) ---

656ubsec_feed(struct ubsec_softc *sc)
657{
658 struct ubsec_q *q, *q2;
659 int npkts, i;
660 void *v;
661 u_int32_t stat;
662
663 npkts = sc->sc_nqueue;
582 /*
583 * search for further sc_qchip ubsec_q's that share
584 * the same MCR, and complete them too, they must be
585 * at the top.
586 */
587 for (i = 0; i < npkts; i++) {
588 if(q->q_stacked_mcr[i]) {
589 ubsec_callback(sc, q->q_stacked_mcr[i]);

--- 84 unchanged lines hidden (view full) ---

674ubsec_feed(struct ubsec_softc *sc)
675{
676 struct ubsec_q *q, *q2;
677 int npkts, i;
678 void *v;
679 u_int32_t stat;
680
681 npkts = sc->sc_nqueue;
682 if (npkts > ubsecstats.hst_maxqueue)
683 ubsecstats.hst_maxqueue = npkts;
684 /*
685 * Decide how many ops to combine in a single MCR. We cannot
686 * aggregate more than UBS_MAX_AGGR because this is the number
687 * of slots defined in the data structure. Otherwise we clamp
688 * based on the tunable parameter ubsec_maxaggr. Note that
689 * aggregation can happen in two ways: either by batching ops
690 * from above or because the h/w backs up and throttles us.
691 * Aggregating ops reduces the number of interrupts to the host
692 * but also (potentially) increases the latency for processing
693 * completed ops as we only get an interrupt when all aggregated
694 * ops have completed.
695 */
664 if (npkts > UBS_MAX_AGGR)
665 npkts = UBS_MAX_AGGR;
696 if (npkts > UBS_MAX_AGGR)
697 npkts = UBS_MAX_AGGR;
698 if (npkts > ubsec_maxaggr)
699 npkts = ubsec_maxaggr;
666 if (npkts > ubsecstats.hst_maxbatch)
667 ubsecstats.hst_maxbatch = npkts;
668 if (npkts < 2)
669 goto feed1;
670 ubsecstats.hst_totbatch += npkts-1;
671
672 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
700 if (npkts > ubsecstats.hst_maxbatch)
701 ubsecstats.hst_maxbatch = npkts;
702 if (npkts < 2)
703 goto feed1;
704 ubsecstats.hst_totbatch += npkts-1;
705
706 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
673 if(stat & BS_STAT_DMAERR) {
707 if (stat & BS_STAT_DMAERR) {
674 ubsec_totalreset(sc);
675 ubsecstats.hst_dmaerr++;
708 ubsec_totalreset(sc);
709 ubsecstats.hst_dmaerr++;
676 }
710 } else
711 ubsecstats.hst_mcr1full++;
677 return (0);
678 }
679
680#ifdef UBSEC_DEBUG
681 if (ubsec_debug)
682 printf("merging %d records\n", npkts);
683#endif /* UBSEC_DEBUG */
684

--- 19 unchanged lines hidden (view full) ---

704
705 v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) -
706 sizeof(struct ubsec_mcr_add));
707 bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add));
708 q->q_stacked_mcr[i] = q2;
709 }
710 q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts);
711 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
712 return (0);
713 }
714
715#ifdef UBSEC_DEBUG
716 if (ubsec_debug)
717 printf("merging %d records\n", npkts);
718#endif /* UBSEC_DEBUG */
719

--- 19 unchanged lines hidden (view full) ---

739
740 v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) -
741 sizeof(struct ubsec_mcr_add));
742 bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add));
743 q->q_stacked_mcr[i] = q2;
744 }
745 q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts);
746 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
747 sc->sc_nqchip += npkts;
748 if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
749 ubsecstats.hst_maxqchip = sc->sc_nqchip;
712 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map,
713 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
714 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
715 offsetof(struct ubsec_dmachunk, d_mcr));
716 return (0);
717
718feed1:
719 while (!SIMPLEQ_EMPTY(&sc->sc_queue)) {
720 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
750 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map,
751 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
752 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
753 offsetof(struct ubsec_dmachunk, d_mcr));
754 return (0);
755
756feed1:
757 while (!SIMPLEQ_EMPTY(&sc->sc_queue)) {
758 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
721 if(stat & BS_STAT_DMAERR) {
759 if (stat & BS_STAT_DMAERR) {
722 ubsec_totalreset(sc);
723 ubsecstats.hst_dmaerr++;
760 ubsec_totalreset(sc);
761 ubsecstats.hst_dmaerr++;
724 }
762 } else
763 ubsecstats.hst_mcr1full++;
725 break;
726 }
727
728 q = SIMPLEQ_FIRST(&sc->sc_queue);
729
730 bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
731 BUS_DMASYNC_PREWRITE);
732 if (q->q_dst_map != NULL)

--- 8 unchanged lines hidden (view full) ---

741 if (ubsec_debug)
742 printf("feed: q->chip %p %08x stat %08x\n",
743 q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr),
744 stat);
745#endif /* UBSEC_DEBUG */
746 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next);
747 --sc->sc_nqueue;
748 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
764 break;
765 }
766
767 q = SIMPLEQ_FIRST(&sc->sc_queue);
768
769 bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
770 BUS_DMASYNC_PREWRITE);
771 if (q->q_dst_map != NULL)

--- 8 unchanged lines hidden (view full) ---

780 if (ubsec_debug)
781 printf("feed: q->chip %p %08x stat %08x\n",
782 q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr),
783 stat);
784#endif /* UBSEC_DEBUG */
785 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next);
786 --sc->sc_nqueue;
787 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
788 sc->sc_nqchip++;
749 }
789 }
790 if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
791 ubsecstats.hst_maxqchip = sc->sc_nqchip;
750 return (0);
751}
752
753/*
754 * Allocate a new 'session' and return an encoded session id. 'sidp'
755 * contains our registration id, and should contain an encoded session
756 * id on successful allocation.
757 */

--- 187 unchanged lines hidden (view full) ---

945 struct ubsec_pktctx ctx;
946 struct ubsec_dma *dmap = NULL;
947
948 if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
949 ubsecstats.hst_invalid++;
950 return (EINVAL);
951 }
952 if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
792 return (0);
793}
794
795/*
796 * Allocate a new 'session' and return an encoded session id. 'sidp'
797 * contains our registration id, and should contain an encoded session
798 * id on successful allocation.
799 */

--- 187 unchanged lines hidden (view full) ---

987 struct ubsec_pktctx ctx;
988 struct ubsec_dma *dmap = NULL;
989
990 if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
991 ubsecstats.hst_invalid++;
992 return (EINVAL);
993 }
994 if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
953 ubsecstats.hst_invalid++;
995 ubsecstats.hst_badsession++;
954 return (EINVAL);
955 }
956
957 UBSEC_LOCK(sc);
958
959 if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
960 ubsecstats.hst_queuefull++;
961 sc->sc_needwakeup |= CRYPTO_SYMQ;

--- 14 unchanged lines hidden (view full) ---

976
977 if (crp->crp_flags & CRYPTO_F_IMBUF) {
978 q->q_src_m = (struct mbuf *)crp->crp_buf;
979 q->q_dst_m = (struct mbuf *)crp->crp_buf;
980 } else if (crp->crp_flags & CRYPTO_F_IOV) {
981 q->q_src_io = (struct uio *)crp->crp_buf;
982 q->q_dst_io = (struct uio *)crp->crp_buf;
983 } else {
996 return (EINVAL);
997 }
998
999 UBSEC_LOCK(sc);
1000
1001 if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
1002 ubsecstats.hst_queuefull++;
1003 sc->sc_needwakeup |= CRYPTO_SYMQ;

--- 14 unchanged lines hidden (view full) ---

1018
1019 if (crp->crp_flags & CRYPTO_F_IMBUF) {
1020 q->q_src_m = (struct mbuf *)crp->crp_buf;
1021 q->q_dst_m = (struct mbuf *)crp->crp_buf;
1022 } else if (crp->crp_flags & CRYPTO_F_IOV) {
1023 q->q_src_io = (struct uio *)crp->crp_buf;
1024 q->q_dst_io = (struct uio *)crp->crp_buf;
1025 } else {
984 ubsecstats.hst_invalid++;
1026 ubsecstats.hst_badflags++;
985 err = EINVAL;
986 goto errout; /* XXX we don't handle contiguous blocks! */
987 }
988
989 bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr));
990
991 dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
992 dmap->d_dma->d_mcr.mcr_flags = 0;
993 q->q_crp = crp;
994
995 crd1 = crp->crp_desc;
996 if (crd1 == NULL) {
1027 err = EINVAL;
1028 goto errout; /* XXX we don't handle contiguous blocks! */
1029 }
1030
1031 bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr));
1032
1033 dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
1034 dmap->d_dma->d_mcr.mcr_flags = 0;
1035 q->q_crp = crp;
1036
1037 crd1 = crp->crp_desc;
1038 if (crd1 == NULL) {
997 ubsecstats.hst_invalid++;
1039 ubsecstats.hst_nodesc++;
998 err = EINVAL;
999 goto errout;
1000 }
1001 crd2 = crd1->crd_next;
1002
1003 if (crd2 == NULL) {
1004 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
1005 crd1->crd_alg == CRYPTO_SHA1_HMAC) {
1006 maccrd = crd1;
1007 enccrd = NULL;
1008 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
1009 crd1->crd_alg == CRYPTO_3DES_CBC) {
1010 maccrd = NULL;
1011 enccrd = crd1;
1012 } else {
1040 err = EINVAL;
1041 goto errout;
1042 }
1043 crd2 = crd1->crd_next;
1044
1045 if (crd2 == NULL) {
1046 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
1047 crd1->crd_alg == CRYPTO_SHA1_HMAC) {
1048 maccrd = crd1;
1049 enccrd = NULL;
1050 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
1051 crd1->crd_alg == CRYPTO_3DES_CBC) {
1052 maccrd = NULL;
1053 enccrd = crd1;
1054 } else {
1013 ubsecstats.hst_invalid++;
1055 ubsecstats.hst_badalg++;
1014 err = EINVAL;
1015 goto errout;
1016 }
1017 } else {
1018 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
1019 crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
1020 (crd2->crd_alg == CRYPTO_DES_CBC ||
1021 crd2->crd_alg == CRYPTO_3DES_CBC) &&

--- 6 unchanged lines hidden (view full) ---

1028 crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
1029 (crd1->crd_flags & CRD_F_ENCRYPT)) {
1030 enccrd = crd1;
1031 maccrd = crd2;
1032 } else {
1033 /*
1034 * We cannot order the ubsec as requested
1035 */
1056 err = EINVAL;
1057 goto errout;
1058 }
1059 } else {
1060 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
1061 crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
1062 (crd2->crd_alg == CRYPTO_DES_CBC ||
1063 crd2->crd_alg == CRYPTO_3DES_CBC) &&

--- 6 unchanged lines hidden (view full) ---

1070 crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
1071 (crd1->crd_flags & CRD_F_ENCRYPT)) {
1072 enccrd = crd1;
1073 maccrd = crd2;
1074 } else {
1075 /*
1076 * We cannot order the ubsec as requested
1077 */
1036 ubsecstats.hst_invalid++;
1078 ubsecstats.hst_badalg++;
1037 err = EINVAL;
1038 goto errout;
1039 }
1040 }
1041
1042 if (enccrd) {
1043 encoffset = enccrd->crd_skip;
1044 ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES);

--- 295 unchanged lines hidden (view full) ---

1340 q->q_dst_map);
1341 q->q_dst_map = NULL;
1342 ubsecstats.hst_noload++;
1343 err = ENOMEM;
1344 goto errout;
1345 }
1346 }
1347 } else {
1079 err = EINVAL;
1080 goto errout;
1081 }
1082 }
1083
1084 if (enccrd) {
1085 encoffset = enccrd->crd_skip;
1086 ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES);

--- 295 unchanged lines hidden (view full) ---

1382 q->q_dst_map);
1383 q->q_dst_map = NULL;
1384 ubsecstats.hst_noload++;
1385 err = ENOMEM;
1386 goto errout;
1387 }
1388 }
1389 } else {
1348 ubsecstats.hst_invalid++;
1390 ubsecstats.hst_badflags++;
1349 err = EINVAL;
1350 goto errout;
1351 }
1352
1353#ifdef UBSEC_DEBUG
1354 if (ubsec_debug)
1355 printf("dst skip: %d\n", dskip);
1356#endif

--- 527 unchanged lines hidden (view full) ---

1884{
1885 struct ubsec_q *q;
1886
1887 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
1888 q = SIMPLEQ_FIRST(&sc->sc_qchip);
1889 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next);
1890 ubsec_free_q(sc, q);
1891 }
1391 err = EINVAL;
1392 goto errout;
1393 }
1394
1395#ifdef UBSEC_DEBUG
1396 if (ubsec_debug)
1397 printf("dst skip: %d\n", dskip);
1398#endif

--- 527 unchanged lines hidden (view full) ---

1926{
1927 struct ubsec_q *q;
1928
1929 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
1930 q = SIMPLEQ_FIRST(&sc->sc_qchip);
1931 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next);
1932 ubsec_free_q(sc, q);
1933 }
1934 sc->sc_nqchip = 0;
1892}
1893
1894/*
1895 * free a ubsec_q
1896 * It is assumed that the caller is within spimp()
1897 */
1898static int
1899ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q)

--- 866 unchanged lines hidden --- 		1935}
1936
1937/*
1938 * free a ubsec_q
1939 * It is assumed that the caller is within spimp()
1940 */
1941static int
1942ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q)

--- 866 unchanged lines hidden ---