ubsec.c revision 108533
1/* $FreeBSD: head/sys/dev/ubsec/ubsec.c 108533 2003-01-01 18:49:04Z schweikh $ */ 2/* $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $ */ 3 4/* 5 * Copyright (c) 2000 Jason L. Wright (jason@thought.net) 6 * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org) 7 * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com) 8 * 9 * All rights reserved. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by Jason L. Wright 22 * 4. The name of the author may not be used to endorse or promote products 23 * derived from this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 26 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 27 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 28 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 29 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 30 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 31 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 33 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 34 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 * 37 * Effort sponsored in part by the Defense Advanced Research Projects 38 * Agency (DARPA) and Air Force Research Laboratory, Air Force 39 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 40 * 41 */ 42 43#define UBSEC_DEBUG 44 45/* 46 * uBsec 5[56]01, 58xx hardware crypto accelerator 47 */ 48 49#include <sys/param.h> 50#include <sys/systm.h> 51#include <sys/proc.h> 52#include <sys/errno.h> 53#include <sys/malloc.h> 54#include <sys/kernel.h> 55#include <sys/mbuf.h> 56#include <sys/lock.h> 57#include <sys/mutex.h> 58#include <sys/stdint.h> 59#include <sys/sysctl.h> 60#include <sys/endian.h> 61 62#include <vm/vm.h> 63#include <vm/pmap.h> 64 65#include <machine/clock.h> 66#include <machine/bus.h> 67#include <machine/resource.h> 68#include <sys/bus.h> 69#include <sys/rman.h> 70 71#include <crypto/sha1.h> 72#include <opencrypto/cryptodev.h> 73#include <opencrypto/cryptosoft.h> 74#include <sys/md5.h> 75#include <sys/random.h> 76 77#include <pci/pcivar.h> 78#include <pci/pcireg.h> 79 80/* grr, #defines for gratuitous incompatibility in queue.h */ 81#define SIMPLEQ_HEAD STAILQ_HEAD 82#define SIMPLEQ_ENTRY STAILQ_ENTRY 83#define SIMPLEQ_INIT STAILQ_INIT 84#define SIMPLEQ_INSERT_TAIL STAILQ_INSERT_TAIL 85#define SIMPLEQ_EMPTY STAILQ_EMPTY 86#define SIMPLEQ_FIRST STAILQ_FIRST 87#define SIMPLEQ_REMOVE_HEAD STAILQ_REMOVE_HEAD_UNTIL 88/* ditto for endian.h */ 89#define letoh16(x) le16toh(x) 90#define letoh32(x) le32toh(x) 91 92#include <dev/ubsec/ubsecreg.h> 93#include <dev/ubsec/ubsecvar.h> 94 95/* 96 * Prototypes and count for the pci_device structure 97 */ 98static int ubsec_probe(device_t); 99static int ubsec_attach(device_t); 100static int ubsec_detach(device_t); 101static int ubsec_suspend(device_t); 102static int ubsec_resume(device_t); 103static void ubsec_shutdown(device_t); 104 105static device_method_t ubsec_methods[] = { 106 /* Device interface */ 107 DEVMETHOD(device_probe, ubsec_probe), 108 DEVMETHOD(device_attach, ubsec_attach), 109 DEVMETHOD(device_detach, ubsec_detach), 110 DEVMETHOD(device_suspend, ubsec_suspend), 111 DEVMETHOD(device_resume, ubsec_resume), 112 DEVMETHOD(device_shutdown, ubsec_shutdown), 113 114 /* bus interface */ 115 DEVMETHOD(bus_print_child, bus_generic_print_child), 116 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 117 118 { 0, 0 } 119}; 120static driver_t ubsec_driver = { 121 "ubsec", 122 ubsec_methods, 123 sizeof (struct ubsec_softc) 124}; 125static devclass_t ubsec_devclass; 126 127DRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0); 128MODULE_DEPEND(ubsec, crypto, 1, 1, 1); 129 130static void ubsec_intr(void *); 131static int ubsec_newsession(void *, u_int32_t *, struct cryptoini *); 132static int ubsec_freesession(void *, u_int64_t); 133static int ubsec_process(void *, struct cryptop *, int); 134static void ubsec_callback(struct ubsec_softc *, struct ubsec_q *); 135static int ubsec_feed(struct ubsec_softc *); 136static void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int); 137static void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *); 138static int ubsec_feed2(struct ubsec_softc *); 139static void ubsec_rng(void *); 140static int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t, 141 struct ubsec_dma_alloc *, int); 142static void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *); 143static int ubsec_dmamap_aligned(struct ubsec_operand *op); 144 145static void ubsec_reset_board(struct ubsec_softc *sc); 146static void ubsec_init_board(struct ubsec_softc *sc); 147static void ubsec_init_pciregs(device_t dev); 148static void ubsec_totalreset(struct ubsec_softc *sc); 149 150static int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q); 151 152static int ubsec_kprocess(void*, struct cryptkop *, int); 153static int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int); 154static int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int); 155static int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int); 156static void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *); 157static int ubsec_ksigbits(struct crparam *); 158static void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 159static void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 160 161#ifdef UBSEC_DEBUG 162static void ubsec_dump_pb(volatile struct ubsec_pktbuf *); 163static void ubsec_dump_mcr(struct ubsec_mcr *); 164static void ubsec_dump_ctx2(struct ubsec_ctx_keyop *); 165 166static int ubsec_debug = 0; 167SYSCTL_INT(_debug, OID_AUTO, ubsec, CTLFLAG_RW, &ubsec_debug, 168 0, "UBSEC driver debugging printfs"); 169#endif 170 171#define READ_REG(sc,r) \ 172 bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r)) 173 174#define WRITE_REG(sc,reg,val) \ 175 bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val) 176 177#define SWAP32(x) (x) = htole32(ntohl((x))) 178#define HTOLE32(x) (x) = htole32(x) 179 180 181struct ubsec_stats ubsecstats; 182SYSCTL_STRUCT(_kern, OID_AUTO, ubsec_stats, CTLFLAG_RD, &ubsecstats, 183 ubsec_stats, "Broadcom driver statistics"); 184/* 185 * ubsec_maxbatch controls the number of crypto ops to voluntarily 186 * collect into one submission to the hardware. This batching happens 187 * when ops are dispatched from the crypto subsystem with a hint that 188 * more are to follow immediately. These ops must also not be marked 189 * with a ``no delay'' flag. 190 */ 191static int ubsec_maxbatch = 1; 192SYSCTL_INT(_kern, OID_AUTO, ubsec_maxbatch, CTLFLAG_RW, &ubsec_maxbatch, 193 0, "Broadcom driver: max ops to batch w/o interrupt"); 194/* 195 * ubsec_maxaggr controls the number of crypto ops to submit to the 196 * hardware as a unit. This aggregation reduces the number of interrupts 197 * to the host at the expense of increased latency (for all but the last 198 * operation). For network traffic setting this to one yields the highest 199 * performance but at the expense of more interrupt processing. 200 */ 201static int ubsec_maxaggr = 1; 202SYSCTL_INT(_kern, OID_AUTO, ubsec_maxaggr, CTLFLAG_RW, &ubsec_maxaggr, 203 0, "Broadcom driver: max ops to aggregate under one interrupt"); 204 205static int 206ubsec_probe(device_t dev) 207{ 208 if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 209 (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 || 210 pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)) 211 return (0); 212 if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 213 (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 || 214 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 || 215 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 216 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822)) 217 return (0); 218 return (ENXIO); 219} 220 221static const char* 222ubsec_partname(struct ubsec_softc *sc) 223{ 224 /* XXX sprintf numbers when not decoded */ 225 switch (pci_get_vendor(sc->sc_dev)) { 226 case PCI_VENDOR_BROADCOM: 227 switch (pci_get_device(sc->sc_dev)) { 228 case PCI_PRODUCT_BROADCOM_5805: return "Broadcom 5805"; 229 case PCI_PRODUCT_BROADCOM_5820: return "Broadcom 5820"; 230 case PCI_PRODUCT_BROADCOM_5821: return "Broadcom 5821"; 231 case PCI_PRODUCT_BROADCOM_5822: return "Broadcom 5822"; 232 } 233 return "Broadcom unknown-part"; 234 case PCI_VENDOR_BLUESTEEL: 235 switch (pci_get_device(sc->sc_dev)) { 236 case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601"; 237 } 238 return "Bluesteel unknown-part"; 239 } 240 return "Unknown-vendor unknown-part"; 241} 242 243static int 244ubsec_attach(device_t dev) 245{ 246 struct ubsec_softc *sc = device_get_softc(dev); 247 struct ubsec_dma *dmap; 248 u_int32_t cmd, i; 249 int rid; 250 251 KASSERT(sc != NULL, ("ubsec_attach: null software carrier!")); 252 bzero(sc, sizeof (*sc)); 253 sc->sc_dev = dev; 254 255 mtx_init(&sc->sc_mtx, device_get_nameunit(dev), "crypto driver", MTX_DEF); 256 257 SIMPLEQ_INIT(&sc->sc_queue); 258 SIMPLEQ_INIT(&sc->sc_qchip); 259 SIMPLEQ_INIT(&sc->sc_queue2); 260 SIMPLEQ_INIT(&sc->sc_qchip2); 261 SIMPLEQ_INIT(&sc->sc_q2free); 262 263 /* XXX handle power management */ 264 265 sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR; 266 267 if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 268 pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601) 269 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 270 271 if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 272 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805) 273 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 274 275 if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 276 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820) 277 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 278 UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 279 280 if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 281 (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 282 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822)) { 283 /* NB: the 5821/5822 defines some additional status bits */ 284 sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY | 285 BS_STAT_MCR2_ALLEMPTY; 286 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 287 UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 288 } 289 /* XXX no PK key support until we sort out the bus_dma stuff */ 290 sc->sc_flags &= ~UBS_FLAGS_KEY; 291 292 cmd = pci_read_config(dev, PCIR_COMMAND, 4); 293 cmd |= PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN; 294 pci_write_config(dev, PCIR_COMMAND, cmd, 4); 295 cmd = pci_read_config(dev, PCIR_COMMAND, 4); 296 297 if (!(cmd & PCIM_CMD_MEMEN)) { 298 device_printf(dev, "failed to enable memory mapping\n"); 299 goto bad; 300 } 301 302 if (!(cmd & PCIM_CMD_BUSMASTEREN)) { 303 device_printf(dev, "failed to enable bus mastering\n"); 304 goto bad; 305 } 306 307 /* 308 * Setup memory-mapping of PCI registers. 309 */ 310 rid = BS_BAR; 311 sc->sc_sr = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid, 312 0, ~0, 1, RF_ACTIVE); 313 if (sc->sc_sr == NULL) { 314 device_printf(dev, "cannot map register space\n"); 315 goto bad; 316 } 317 sc->sc_st = rman_get_bustag(sc->sc_sr); 318 sc->sc_sh = rman_get_bushandle(sc->sc_sr); 319 320 /* 321 * Arrange interrupt line. 322 */ 323 rid = 0; 324 sc->sc_irq = bus_alloc_resource(dev, SYS_RES_IRQ, &rid, 325 0, ~0, 1, RF_SHAREABLE|RF_ACTIVE); 326 if (sc->sc_irq == NULL) { 327 device_printf(dev, "could not map interrupt\n"); 328 goto bad; 329 } 330 /* 331 * NB: Network code assumes we are blocked with splimp() 332 * so make sure the IRQ is mapped appropriately. 333 */ 334 if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET, 335 ubsec_intr, sc, &sc->sc_ih)) { 336 device_printf(dev, "could not establish interrupt\n"); 337 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 338 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 339 goto bad; 340 } 341 342 sc->sc_cid = crypto_get_driverid(0); 343 if (sc->sc_cid < 0) { 344 device_printf(dev, "could not get crypto driver id\n"); 345 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 346 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 347 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 348 goto bad; 349 } 350 351 /* 352 * Setup DMA descriptor area. 353 */ 354 if (bus_dma_tag_create(NULL, /* parent */ 355 1, 0, /* alignment, bounds */ 356 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 357 BUS_SPACE_MAXADDR, /* highaddr */ 358 NULL, NULL, /* filter, filterarg */ 359 0x3ffff, /* maxsize XXX */ 360 UBS_MAX_SCATTER, /* nsegments */ 361 0xffff, /* maxsegsize XXX */ 362 BUS_DMA_ALLOCNOW, /* flags */ 363 &sc->sc_dmat)) { 364 device_printf(dev, "cannot allocate DMA tag\n"); 365 crypto_unregister_all(sc->sc_cid); 366 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 367 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 368 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 369 goto bad; 370 } 371 SIMPLEQ_INIT(&sc->sc_freequeue); 372 dmap = sc->sc_dmaa; 373 for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) { 374 struct ubsec_q *q; 375 376 q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q), 377 M_DEVBUF, M_NOWAIT); 378 if (q == NULL) { 379 device_printf(dev, "cannot allocate queue buffers\n"); 380 break; 381 } 382 383 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk), 384 &dmap->d_alloc, 0)) { 385 device_printf(dev, "cannot allocate dma buffers\n"); 386 free(q, M_DEVBUF); 387 break; 388 } 389 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr; 390 391 q->q_dma = dmap; 392 sc->sc_queuea[i] = q; 393 394 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 395 } 396 397 device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc)); 398 399 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0, 400 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 401 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0, 402 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 403 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0, 404 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 405 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0, 406 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 407 408 /* 409 * Reset Broadcom chip 410 */ 411 ubsec_reset_board(sc); 412 413 /* 414 * Init Broadcom specific PCI settings 415 */ 416 ubsec_init_pciregs(dev); 417 418 /* 419 * Init Broadcom chip 420 */ 421 ubsec_init_board(sc); 422 423#ifndef UBSEC_NO_RNG 424 if (sc->sc_flags & UBS_FLAGS_RNG) { 425 sc->sc_statmask |= BS_STAT_MCR2_DONE; 426 427 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 428 &sc->sc_rng.rng_q.q_mcr, 0)) 429 goto skip_rng; 430 431 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass), 432 &sc->sc_rng.rng_q.q_ctx, 0)) { 433 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 434 goto skip_rng; 435 } 436 437 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) * 438 UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) { 439 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 440 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 441 goto skip_rng; 442 } 443 444 if (hz >= 100) 445 sc->sc_rnghz = hz / 100; 446 else 447 sc->sc_rnghz = 1; 448 /* NB: 1 means the callout runs w/o Giant locked */ 449 callout_init(&sc->sc_rngto, 1); 450 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 451skip_rng: 452 ; 453 } 454#endif /* UBSEC_NO_RNG */ 455 456 if (sc->sc_flags & UBS_FLAGS_KEY) { 457 sc->sc_statmask |= BS_STAT_MCR2_DONE; 458 459 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0, 460 ubsec_kprocess, sc); 461#if 0 462 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0, 463 ubsec_kprocess, sc); 464#endif 465 } 466 return (0); 467bad: 468 mtx_destroy(&sc->sc_mtx); 469 return (ENXIO); 470} 471 472/* 473 * Detach a device that successfully probed. 474 */ 475static int 476ubsec_detach(device_t dev) 477{ 478 struct ubsec_softc *sc = device_get_softc(dev); 479 480 KASSERT(sc != NULL, ("ubsec_detach: null software carrier")); 481 482 UBSEC_LOCK(sc); 483 484 callout_stop(&sc->sc_rngto); 485 486 crypto_unregister_all(sc->sc_cid); 487 488 bus_generic_detach(dev); 489 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 490 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 491 492 bus_dma_tag_destroy(sc->sc_dmat); 493 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 494 495 UBSEC_UNLOCK(sc); 496 497 mtx_destroy(&sc->sc_mtx); 498 499 return (0); 500} 501 502/* 503 * Stop all chip i/o so that the kernel's probe routines don't 504 * get confused by errant DMAs when rebooting. 505 */ 506static void 507ubsec_shutdown(device_t dev) 508{ 509#ifdef notyet 510 ubsec_stop(device_get_softc(dev)); 511#endif 512} 513 514/* 515 * Device suspend routine. 516 */ 517static int 518ubsec_suspend(device_t dev) 519{ 520 struct ubsec_softc *sc = device_get_softc(dev); 521 522 KASSERT(sc != NULL, ("ubsec_suspend: null software carrier")); 523#ifdef notyet 524 /* XXX stop the device and save PCI settings */ 525#endif 526 sc->sc_suspended = 1; 527 528 return (0); 529} 530 531static int 532ubsec_resume(device_t dev) 533{ 534 struct ubsec_softc *sc = device_get_softc(dev); 535 536 KASSERT(sc != NULL, ("ubsec_resume: null software carrier")); 537#ifdef notyet 538 /* XXX retore PCI settings and start the device */ 539#endif 540 sc->sc_suspended = 0; 541 return (0); 542} 543 544/* 545 * UBSEC Interrupt routine 546 */ 547static void 548ubsec_intr(void *arg) 549{ 550 struct ubsec_softc *sc = arg; 551 volatile u_int32_t stat; 552 struct ubsec_q *q; 553 struct ubsec_dma *dmap; 554 int npkts = 0, i; 555 556 UBSEC_LOCK(sc); 557 558 stat = READ_REG(sc, BS_STAT); 559 stat &= sc->sc_statmask; 560 if (stat == 0) { 561 UBSEC_UNLOCK(sc); 562 return; 563 } 564 565 WRITE_REG(sc, BS_STAT, stat); /* IACK */ 566 567 /* 568 * Check to see if we have any packets waiting for us 569 */ 570 if ((stat & BS_STAT_MCR1_DONE)) { 571 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 572 q = SIMPLEQ_FIRST(&sc->sc_qchip); 573 dmap = q->q_dma; 574 575 if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0) 576 break; 577 578 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next); 579 580 npkts = q->q_nstacked_mcrs; 581 sc->sc_nqchip -= 1+npkts; 582 /* 583 * search for further sc_qchip ubsec_q's that share 584 * the same MCR, and complete them too, they must be 585 * at the top. 586 */ 587 for (i = 0; i < npkts; i++) { 588 if(q->q_stacked_mcr[i]) { 589 ubsec_callback(sc, q->q_stacked_mcr[i]); 590 ubsecstats.hst_opackets++; 591 } else { 592 break; 593 } 594 } 595 ubsec_callback(sc, q); 596 ubsecstats.hst_opackets++; 597 } 598 599 /* 600 * Don't send any more packet to chip if there has been 601 * a DMAERR. 602 */ 603 if (!(stat & BS_STAT_DMAERR)) 604 ubsec_feed(sc); 605 } 606 607 /* 608 * Check to see if we have any key setups/rng's waiting for us 609 */ 610 if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) && 611 (stat & BS_STAT_MCR2_DONE)) { 612 struct ubsec_q2 *q2; 613 struct ubsec_mcr *mcr; 614 615 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) { 616 q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 617 618 bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map, 619 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 620 621 mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 622 if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) { 623 bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map, 624 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 625 break; 626 } 627 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q2, q_next); 628 ubsec_callback2(sc, q2); 629 /* 630 * Don't send any more packet to chip if there has been 631 * a DMAERR. 632 */ 633 if (!(stat & BS_STAT_DMAERR)) 634 ubsec_feed2(sc); 635 } 636 } 637 638 /* 639 * Check to see if we got any DMA Error 640 */ 641 if (stat & BS_STAT_DMAERR) { 642#ifdef UBSEC_DEBUG 643 if (ubsec_debug) { 644 volatile u_int32_t a = READ_REG(sc, BS_ERR); 645 646 printf("dmaerr %s@%08x\n", 647 (a & BS_ERR_READ) ? "read" : "write", 648 a & BS_ERR_ADDR); 649 } 650#endif /* UBSEC_DEBUG */ 651 ubsecstats.hst_dmaerr++; 652 ubsec_totalreset(sc); 653 ubsec_feed(sc); 654 } 655 656 if (sc->sc_needwakeup) { /* XXX check high watermark */ 657 int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ); 658#ifdef UBSEC_DEBUG 659 if (ubsec_debug) 660 device_printf(sc->sc_dev, "wakeup crypto (%x)\n", 661 sc->sc_needwakeup); 662#endif /* UBSEC_DEBUG */ 663 sc->sc_needwakeup &= ~wakeup; 664 crypto_unblock(sc->sc_cid, wakeup); 665 } 666 667 UBSEC_UNLOCK(sc); 668} 669 670/* 671 * ubsec_feed() - aggregate and post requests to chip 672 */ 673static int 674ubsec_feed(struct ubsec_softc *sc) 675{ 676 struct ubsec_q *q, *q2; 677 int npkts, i; 678 void *v; 679 u_int32_t stat; 680 681 npkts = sc->sc_nqueue; 682 if (npkts > ubsecstats.hst_maxqueue) 683 ubsecstats.hst_maxqueue = npkts; 684 /* 685 * Decide how many ops to combine in a single MCR. We cannot 686 * aggregate more than UBS_MAX_AGGR because this is the number 687 * of slots defined in the data structure. Otherwise we clamp 688 * based on the tunable parameter ubsec_maxaggr. Note that 689 * aggregation can happen in two ways: either by batching ops 690 * from above or because the h/w backs up and throttles us. 691 * Aggregating ops reduces the number of interrupts to the host 692 * but also (potentially) increases the latency for processing 693 * completed ops as we only get an interrupt when all aggregated 694 * ops have completed. 695 */ 696 if (npkts > UBS_MAX_AGGR) 697 npkts = UBS_MAX_AGGR; 698 if (npkts > ubsec_maxaggr) 699 npkts = ubsec_maxaggr; 700 if (npkts > ubsecstats.hst_maxbatch) 701 ubsecstats.hst_maxbatch = npkts; 702 if (npkts < 2) 703 goto feed1; 704 ubsecstats.hst_totbatch += npkts-1; 705 706 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 707 if (stat & BS_STAT_DMAERR) { 708 ubsec_totalreset(sc); 709 ubsecstats.hst_dmaerr++; 710 } else 711 ubsecstats.hst_mcr1full++; 712 return (0); 713 } 714 715#ifdef UBSEC_DEBUG 716 if (ubsec_debug) 717 printf("merging %d records\n", npkts); 718#endif /* UBSEC_DEBUG */ 719 720 q = SIMPLEQ_FIRST(&sc->sc_queue); 721 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next); 722 --sc->sc_nqueue; 723 724 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE); 725 if (q->q_dst_map != NULL) 726 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD); 727 728 q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */ 729 730 for (i = 0; i < q->q_nstacked_mcrs; i++) { 731 q2 = SIMPLEQ_FIRST(&sc->sc_queue); 732 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map, 733 BUS_DMASYNC_PREWRITE); 734 if (q2->q_dst_map != NULL) 735 bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map, 736 BUS_DMASYNC_PREREAD); 737 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q2, q_next); 738 --sc->sc_nqueue; 739 740 v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) - 741 sizeof(struct ubsec_mcr_add)); 742 bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add)); 743 q->q_stacked_mcr[i] = q2; 744 } 745 q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts); 746 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 747 sc->sc_nqchip += npkts; 748 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 749 ubsecstats.hst_maxqchip = sc->sc_nqchip; 750 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map, 751 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 752 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 753 offsetof(struct ubsec_dmachunk, d_mcr)); 754 return (0); 755 756feed1: 757 while (!SIMPLEQ_EMPTY(&sc->sc_queue)) { 758 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 759 if (stat & BS_STAT_DMAERR) { 760 ubsec_totalreset(sc); 761 ubsecstats.hst_dmaerr++; 762 } else 763 ubsecstats.hst_mcr1full++; 764 break; 765 } 766 767 q = SIMPLEQ_FIRST(&sc->sc_queue); 768 769 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 770 BUS_DMASYNC_PREWRITE); 771 if (q->q_dst_map != NULL) 772 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 773 BUS_DMASYNC_PREREAD); 774 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map, 775 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 776 777 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 778 offsetof(struct ubsec_dmachunk, d_mcr)); 779#ifdef UBSEC_DEBUG 780 if (ubsec_debug) 781 printf("feed: q->chip %p %08x stat %08x\n", 782 q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr), 783 stat); 784#endif /* UBSEC_DEBUG */ 785 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next); 786 --sc->sc_nqueue; 787 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 788 sc->sc_nqchip++; 789 } 790 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 791 ubsecstats.hst_maxqchip = sc->sc_nqchip; 792 return (0); 793} 794 795/* 796 * Allocate a new 'session' and return an encoded session id. 'sidp' 797 * contains our registration id, and should contain an encoded session 798 * id on successful allocation. 799 */ 800static int 801ubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri) 802{ 803 struct cryptoini *c, *encini = NULL, *macini = NULL; 804 struct ubsec_softc *sc = arg; 805 struct ubsec_session *ses = NULL; 806 MD5_CTX md5ctx; 807 SHA1_CTX sha1ctx; 808 int i, sesn; 809 810 KASSERT(sc != NULL, ("ubsec_newsession: null softc")); 811 if (sidp == NULL || cri == NULL || sc == NULL) 812 return (EINVAL); 813 814 for (c = cri; c != NULL; c = c->cri_next) { 815 if (c->cri_alg == CRYPTO_MD5_HMAC || 816 c->cri_alg == CRYPTO_SHA1_HMAC) { 817 if (macini) 818 return (EINVAL); 819 macini = c; 820 } else if (c->cri_alg == CRYPTO_DES_CBC || 821 c->cri_alg == CRYPTO_3DES_CBC) { 822 if (encini) 823 return (EINVAL); 824 encini = c; 825 } else 826 return (EINVAL); 827 } 828 if (encini == NULL && macini == NULL) 829 return (EINVAL); 830 831 if (sc->sc_sessions == NULL) { 832 ses = sc->sc_sessions = (struct ubsec_session *)malloc( 833 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 834 if (ses == NULL) 835 return (ENOMEM); 836 sesn = 0; 837 sc->sc_nsessions = 1; 838 } else { 839 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 840 if (sc->sc_sessions[sesn].ses_used == 0) { 841 ses = &sc->sc_sessions[sesn]; 842 break; 843 } 844 } 845 846 if (ses == NULL) { 847 sesn = sc->sc_nsessions; 848 ses = (struct ubsec_session *)malloc((sesn + 1) * 849 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 850 if (ses == NULL) 851 return (ENOMEM); 852 bcopy(sc->sc_sessions, ses, sesn * 853 sizeof(struct ubsec_session)); 854 bzero(sc->sc_sessions, sesn * 855 sizeof(struct ubsec_session)); 856 free(sc->sc_sessions, M_DEVBUF); 857 sc->sc_sessions = ses; 858 ses = &sc->sc_sessions[sesn]; 859 sc->sc_nsessions++; 860 } 861 } 862 863 bzero(ses, sizeof(struct ubsec_session)); 864 ses->ses_used = 1; 865 if (encini) { 866 /* get an IV, network byte order */ 867 /* XXX may read fewer than requested */ 868 read_random(ses->ses_iv, sizeof(ses->ses_iv)); 869 870 /* Go ahead and compute key in ubsec's byte order */ 871 if (encini->cri_alg == CRYPTO_DES_CBC) { 872 bcopy(encini->cri_key, &ses->ses_deskey[0], 8); 873 bcopy(encini->cri_key, &ses->ses_deskey[2], 8); 874 bcopy(encini->cri_key, &ses->ses_deskey[4], 8); 875 } else 876 bcopy(encini->cri_key, ses->ses_deskey, 24); 877 878 SWAP32(ses->ses_deskey[0]); 879 SWAP32(ses->ses_deskey[1]); 880 SWAP32(ses->ses_deskey[2]); 881 SWAP32(ses->ses_deskey[3]); 882 SWAP32(ses->ses_deskey[4]); 883 SWAP32(ses->ses_deskey[5]); 884 } 885 886 if (macini) { 887 for (i = 0; i < macini->cri_klen / 8; i++) 888 macini->cri_key[i] ^= HMAC_IPAD_VAL; 889 890 if (macini->cri_alg == CRYPTO_MD5_HMAC) { 891 MD5Init(&md5ctx); 892 MD5Update(&md5ctx, macini->cri_key, 893 macini->cri_klen / 8); 894 MD5Update(&md5ctx, hmac_ipad_buffer, 895 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 896 bcopy(md5ctx.state, ses->ses_hminner, 897 sizeof(md5ctx.state)); 898 } else { 899 SHA1Init(&sha1ctx); 900 SHA1Update(&sha1ctx, macini->cri_key, 901 macini->cri_klen / 8); 902 SHA1Update(&sha1ctx, hmac_ipad_buffer, 903 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 904 bcopy(sha1ctx.h.b32, ses->ses_hminner, 905 sizeof(sha1ctx.h.b32)); 906 } 907 908 for (i = 0; i < macini->cri_klen / 8; i++) 909 macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 910 911 if (macini->cri_alg == CRYPTO_MD5_HMAC) { 912 MD5Init(&md5ctx); 913 MD5Update(&md5ctx, macini->cri_key, 914 macini->cri_klen / 8); 915 MD5Update(&md5ctx, hmac_opad_buffer, 916 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 917 bcopy(md5ctx.state, ses->ses_hmouter, 918 sizeof(md5ctx.state)); 919 } else { 920 SHA1Init(&sha1ctx); 921 SHA1Update(&sha1ctx, macini->cri_key, 922 macini->cri_klen / 8); 923 SHA1Update(&sha1ctx, hmac_opad_buffer, 924 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 925 bcopy(sha1ctx.h.b32, ses->ses_hmouter, 926 sizeof(sha1ctx.h.b32)); 927 } 928 929 for (i = 0; i < macini->cri_klen / 8; i++) 930 macini->cri_key[i] ^= HMAC_OPAD_VAL; 931 } 932 933 *sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn); 934 return (0); 935} 936 937/* 938 * Deallocate a session. 939 */ 940static int 941ubsec_freesession(void *arg, u_int64_t tid) 942{ 943 struct ubsec_softc *sc = arg; 944 int session; 945 u_int32_t sid = ((u_int32_t) tid) & 0xffffffff; 946 947 KASSERT(sc != NULL, ("ubsec_freesession: null softc")); 948 if (sc == NULL) 949 return (EINVAL); 950 951 session = UBSEC_SESSION(sid); 952 if (session >= sc->sc_nsessions) 953 return (EINVAL); 954 955 bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session])); 956 return (0); 957} 958 959static void 960ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error) 961{ 962 struct ubsec_operand *op = arg; 963 964 KASSERT(nsegs <= UBS_MAX_SCATTER, 965 ("Too many DMA segments returned when mapping operand")); 966#ifdef UBSEC_DEBUG 967 if (ubsec_debug) 968 printf("ubsec_op_cb: mapsize %u nsegs %d\n", 969 (u_int) mapsize, nsegs); 970#endif 971 op->mapsize = mapsize; 972 op->nsegs = nsegs; 973 bcopy(seg, op->segs, nsegs * sizeof (seg[0])); 974} 975 976static int 977ubsec_process(void *arg, struct cryptop *crp, int hint) 978{ 979 struct ubsec_q *q = NULL; 980 int err = 0, i, j, nicealign; 981 struct ubsec_softc *sc = arg; 982 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 983 int encoffset = 0, macoffset = 0, cpskip, cpoffset; 984 int sskip, dskip, stheend, dtheend; 985 int16_t coffset; 986 struct ubsec_session *ses; 987 struct ubsec_pktctx ctx; 988 struct ubsec_dma *dmap = NULL; 989 990 if (crp == NULL || crp->crp_callback == NULL || sc == NULL) { 991 ubsecstats.hst_invalid++; 992 return (EINVAL); 993 } 994 if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) { 995 ubsecstats.hst_badsession++; 996 return (EINVAL); 997 } 998 999 UBSEC_LOCK(sc); 1000 1001 if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 1002 ubsecstats.hst_queuefull++; 1003 sc->sc_needwakeup |= CRYPTO_SYMQ; 1004 UBSEC_UNLOCK(sc); 1005 return (ERESTART); 1006 } 1007 q = SIMPLEQ_FIRST(&sc->sc_freequeue); 1008 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q, q_next); 1009 UBSEC_UNLOCK(sc); 1010 1011 dmap = q->q_dma; /* Save dma pointer */ 1012 bzero(q, sizeof(struct ubsec_q)); 1013 bzero(&ctx, sizeof(ctx)); 1014 1015 q->q_sesn = UBSEC_SESSION(crp->crp_sid); 1016 q->q_dma = dmap; 1017 ses = &sc->sc_sessions[q->q_sesn]; 1018 1019 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1020 q->q_src_m = (struct mbuf *)crp->crp_buf; 1021 q->q_dst_m = (struct mbuf *)crp->crp_buf; 1022 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1023 q->q_src_io = (struct uio *)crp->crp_buf; 1024 q->q_dst_io = (struct uio *)crp->crp_buf; 1025 } else { 1026 ubsecstats.hst_badflags++; 1027 err = EINVAL; 1028 goto errout; /* XXX we don't handle contiguous blocks! */ 1029 } 1030 1031 bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr)); 1032 1033 dmap->d_dma->d_mcr.mcr_pkts = htole16(1); 1034 dmap->d_dma->d_mcr.mcr_flags = 0; 1035 q->q_crp = crp; 1036 1037 crd1 = crp->crp_desc; 1038 if (crd1 == NULL) { 1039 ubsecstats.hst_nodesc++; 1040 err = EINVAL; 1041 goto errout; 1042 } 1043 crd2 = crd1->crd_next; 1044 1045 if (crd2 == NULL) { 1046 if (crd1->crd_alg == CRYPTO_MD5_HMAC || 1047 crd1->crd_alg == CRYPTO_SHA1_HMAC) { 1048 maccrd = crd1; 1049 enccrd = NULL; 1050 } else if (crd1->crd_alg == CRYPTO_DES_CBC || 1051 crd1->crd_alg == CRYPTO_3DES_CBC) { 1052 maccrd = NULL; 1053 enccrd = crd1; 1054 } else { 1055 ubsecstats.hst_badalg++; 1056 err = EINVAL; 1057 goto errout; 1058 } 1059 } else { 1060 if ((crd1->crd_alg == CRYPTO_MD5_HMAC || 1061 crd1->crd_alg == CRYPTO_SHA1_HMAC) && 1062 (crd2->crd_alg == CRYPTO_DES_CBC || 1063 crd2->crd_alg == CRYPTO_3DES_CBC) && 1064 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 1065 maccrd = crd1; 1066 enccrd = crd2; 1067 } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 1068 crd1->crd_alg == CRYPTO_3DES_CBC) && 1069 (crd2->crd_alg == CRYPTO_MD5_HMAC || 1070 crd2->crd_alg == CRYPTO_SHA1_HMAC) && 1071 (crd1->crd_flags & CRD_F_ENCRYPT)) { 1072 enccrd = crd1; 1073 maccrd = crd2; 1074 } else { 1075 /* 1076 * We cannot order the ubsec as requested 1077 */ 1078 ubsecstats.hst_badalg++; 1079 err = EINVAL; 1080 goto errout; 1081 } 1082 } 1083 1084 if (enccrd) { 1085 encoffset = enccrd->crd_skip; 1086 ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES); 1087 1088 if (enccrd->crd_flags & CRD_F_ENCRYPT) { 1089 q->q_flags |= UBSEC_QFLAGS_COPYOUTIV; 1090 1091 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1092 bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1093 else { 1094 ctx.pc_iv[0] = ses->ses_iv[0]; 1095 ctx.pc_iv[1] = ses->ses_iv[1]; 1096 } 1097 1098 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { 1099 if (crp->crp_flags & CRYPTO_F_IMBUF) 1100 m_copyback(q->q_src_m, 1101 enccrd->crd_inject, 1102 8, (caddr_t)ctx.pc_iv); 1103 else if (crp->crp_flags & CRYPTO_F_IOV) 1104 cuio_copyback(q->q_src_io, 1105 enccrd->crd_inject, 1106 8, (caddr_t)ctx.pc_iv); 1107 } 1108 } else { 1109 ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND); 1110 1111 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1112 bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1113 else if (crp->crp_flags & CRYPTO_F_IMBUF) 1114 m_copydata(q->q_src_m, enccrd->crd_inject, 1115 8, (caddr_t)ctx.pc_iv); 1116 else if (crp->crp_flags & CRYPTO_F_IOV) 1117 cuio_copydata(q->q_src_io, 1118 enccrd->crd_inject, 8, 1119 (caddr_t)ctx.pc_iv); 1120 } 1121 1122 ctx.pc_deskey[0] = ses->ses_deskey[0]; 1123 ctx.pc_deskey[1] = ses->ses_deskey[1]; 1124 ctx.pc_deskey[2] = ses->ses_deskey[2]; 1125 ctx.pc_deskey[3] = ses->ses_deskey[3]; 1126 ctx.pc_deskey[4] = ses->ses_deskey[4]; 1127 ctx.pc_deskey[5] = ses->ses_deskey[5]; 1128 SWAP32(ctx.pc_iv[0]); 1129 SWAP32(ctx.pc_iv[1]); 1130 } 1131 1132 if (maccrd) { 1133 macoffset = maccrd->crd_skip; 1134 1135 if (maccrd->crd_alg == CRYPTO_MD5_HMAC) 1136 ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5); 1137 else 1138 ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1); 1139 1140 for (i = 0; i < 5; i++) { 1141 ctx.pc_hminner[i] = ses->ses_hminner[i]; 1142 ctx.pc_hmouter[i] = ses->ses_hmouter[i]; 1143 1144 HTOLE32(ctx.pc_hminner[i]); 1145 HTOLE32(ctx.pc_hmouter[i]); 1146 } 1147 } 1148 1149 if (enccrd && maccrd) { 1150 /* 1151 * ubsec cannot handle packets where the end of encryption 1152 * and authentication are not the same, or where the 1153 * encrypted part begins before the authenticated part. 1154 */ 1155 if ((encoffset + enccrd->crd_len) != 1156 (macoffset + maccrd->crd_len)) { 1157 ubsecstats.hst_lenmismatch++; 1158 err = EINVAL; 1159 goto errout; 1160 } 1161 if (enccrd->crd_skip < maccrd->crd_skip) { 1162 ubsecstats.hst_skipmismatch++; 1163 err = EINVAL; 1164 goto errout; 1165 } 1166 sskip = maccrd->crd_skip; 1167 cpskip = dskip = enccrd->crd_skip; 1168 stheend = maccrd->crd_len; 1169 dtheend = enccrd->crd_len; 1170 coffset = enccrd->crd_skip - maccrd->crd_skip; 1171 cpoffset = cpskip + dtheend; 1172#ifdef UBSEC_DEBUG 1173 if (ubsec_debug) { 1174 printf("mac: skip %d, len %d, inject %d\n", 1175 maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject); 1176 printf("enc: skip %d, len %d, inject %d\n", 1177 enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject); 1178 printf("src: skip %d, len %d\n", sskip, stheend); 1179 printf("dst: skip %d, len %d\n", dskip, dtheend); 1180 printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n", 1181 coffset, stheend, cpskip, cpoffset); 1182 } 1183#endif 1184 } else { 1185 cpskip = dskip = sskip = macoffset + encoffset; 1186 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len; 1187 cpoffset = cpskip + dtheend; 1188 coffset = 0; 1189 } 1190 ctx.pc_offset = htole16(coffset >> 2); 1191 1192 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) { 1193 ubsecstats.hst_nomap++; 1194 err = ENOMEM; 1195 goto errout; 1196 } 1197 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1198 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map, 1199 q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1200 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1201 q->q_src_map = NULL; 1202 ubsecstats.hst_noload++; 1203 err = ENOMEM; 1204 goto errout; 1205 } 1206 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1207 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map, 1208 q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1209 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1210 q->q_src_map = NULL; 1211 ubsecstats.hst_noload++; 1212 err = ENOMEM; 1213 goto errout; 1214 } 1215 } 1216 nicealign = ubsec_dmamap_aligned(&q->q_src); 1217 1218 dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend); 1219 1220#ifdef UBSEC_DEBUG 1221 if (ubsec_debug) 1222 printf("src skip: %d nicealign: %u\n", sskip, nicealign); 1223#endif 1224 for (i = j = 0; i < q->q_src_nsegs; i++) { 1225 struct ubsec_pktbuf *pb; 1226 bus_size_t packl = q->q_src_segs[i].ds_len; 1227 bus_addr_t packp = q->q_src_segs[i].ds_addr; 1228 1229 if (sskip >= packl) { 1230 sskip -= packl; 1231 continue; 1232 } 1233 1234 packl -= sskip; 1235 packp += sskip; 1236 sskip = 0; 1237 1238 if (packl > 0xfffc) { 1239 err = EIO; 1240 goto errout; 1241 } 1242 1243 if (j == 0) 1244 pb = &dmap->d_dma->d_mcr.mcr_ipktbuf; 1245 else 1246 pb = &dmap->d_dma->d_sbuf[j - 1]; 1247 1248 pb->pb_addr = htole32(packp); 1249 1250 if (stheend) { 1251 if (packl > stheend) { 1252 pb->pb_len = htole32(stheend); 1253 stheend = 0; 1254 } else { 1255 pb->pb_len = htole32(packl); 1256 stheend -= packl; 1257 } 1258 } else 1259 pb->pb_len = htole32(packl); 1260 1261 if ((i + 1) == q->q_src_nsegs) 1262 pb->pb_next = 0; 1263 else 1264 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1265 offsetof(struct ubsec_dmachunk, d_sbuf[j])); 1266 j++; 1267 } 1268 1269 if (enccrd == NULL && maccrd != NULL) { 1270 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0; 1271 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0; 1272 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr + 1273 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1274#ifdef UBSEC_DEBUG 1275 if (ubsec_debug) 1276 printf("opkt: %x %x %x\n", 1277 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr, 1278 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len, 1279 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next); 1280#endif 1281 } else { 1282 if (crp->crp_flags & CRYPTO_F_IOV) { 1283 if (!nicealign) { 1284 ubsecstats.hst_iovmisaligned++; 1285 err = EINVAL; 1286 goto errout; 1287 } 1288 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, 1289 &q->q_dst_map)) { 1290 ubsecstats.hst_nomap++; 1291 err = ENOMEM; 1292 goto errout; 1293 } 1294 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map, 1295 q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) { 1296 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1297 q->q_dst_map = NULL; 1298 ubsecstats.hst_noload++; 1299 err = ENOMEM; 1300 goto errout; 1301 } 1302 } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1303 if (nicealign) { 1304 q->q_dst = q->q_src; 1305 } else { 1306 int totlen, len; 1307 struct mbuf *m, *top, **mp; 1308 1309 ubsecstats.hst_unaligned++; 1310 totlen = q->q_src_mapsize; 1311 if (q->q_src_m->m_flags & M_PKTHDR) { 1312 len = MHLEN; 1313 MGETHDR(m, M_DONTWAIT, MT_DATA); 1314 if (m && !m_dup_pkthdr(m, q->q_src_m, M_DONTWAIT)) { 1315 m_free(m); 1316 m = NULL; 1317 } 1318 } else { 1319 len = MLEN; 1320 MGET(m, M_DONTWAIT, MT_DATA); 1321 } 1322 if (m == NULL) { 1323 ubsecstats.hst_nombuf++; 1324 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1325 goto errout; 1326 } 1327 if (totlen >= MINCLSIZE) { 1328 MCLGET(m, M_DONTWAIT); 1329 if ((m->m_flags & M_EXT) == 0) { 1330 m_free(m); 1331 ubsecstats.hst_nomcl++; 1332 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1333 goto errout; 1334 } 1335 len = MCLBYTES; 1336 } 1337 m->m_len = len; 1338 top = NULL; 1339 mp = ⊤ 1340 1341 while (totlen > 0) { 1342 if (top) { 1343 MGET(m, M_DONTWAIT, MT_DATA); 1344 if (m == NULL) { 1345 m_freem(top); 1346 ubsecstats.hst_nombuf++; 1347 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1348 goto errout; 1349 } 1350 len = MLEN; 1351 } 1352 if (top && totlen >= MINCLSIZE) { 1353 MCLGET(m, M_DONTWAIT); 1354 if ((m->m_flags & M_EXT) == 0) { 1355 *mp = m; 1356 m_freem(top); 1357 ubsecstats.hst_nomcl++; 1358 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1359 goto errout; 1360 } 1361 len = MCLBYTES; 1362 } 1363 m->m_len = len = min(totlen, len); 1364 totlen -= len; 1365 *mp = m; 1366 mp = &m->m_next; 1367 } 1368 q->q_dst_m = top; 1369 ubsec_mcopy(q->q_src_m, q->q_dst_m, 1370 cpskip, cpoffset); 1371 if (bus_dmamap_create(sc->sc_dmat, 1372 BUS_DMA_NOWAIT, &q->q_dst_map) != 0) { 1373 ubsecstats.hst_nomap++; 1374 err = ENOMEM; 1375 goto errout; 1376 } 1377 if (bus_dmamap_load_mbuf(sc->sc_dmat, 1378 q->q_dst_map, q->q_dst_m, 1379 ubsec_op_cb, &q->q_dst, 1380 BUS_DMA_NOWAIT) != 0) { 1381 bus_dmamap_destroy(sc->sc_dmat, 1382 q->q_dst_map); 1383 q->q_dst_map = NULL; 1384 ubsecstats.hst_noload++; 1385 err = ENOMEM; 1386 goto errout; 1387 } 1388 } 1389 } else { 1390 ubsecstats.hst_badflags++; 1391 err = EINVAL; 1392 goto errout; 1393 } 1394 1395#ifdef UBSEC_DEBUG 1396 if (ubsec_debug) 1397 printf("dst skip: %d\n", dskip); 1398#endif 1399 for (i = j = 0; i < q->q_dst_nsegs; i++) { 1400 struct ubsec_pktbuf *pb; 1401 bus_size_t packl = q->q_dst_segs[i].ds_len; 1402 bus_addr_t packp = q->q_dst_segs[i].ds_addr; 1403 1404 if (dskip >= packl) { 1405 dskip -= packl; 1406 continue; 1407 } 1408 1409 packl -= dskip; 1410 packp += dskip; 1411 dskip = 0; 1412 1413 if (packl > 0xfffc) { 1414 err = EIO; 1415 goto errout; 1416 } 1417 1418 if (j == 0) 1419 pb = &dmap->d_dma->d_mcr.mcr_opktbuf; 1420 else 1421 pb = &dmap->d_dma->d_dbuf[j - 1]; 1422 1423 pb->pb_addr = htole32(packp); 1424 1425 if (dtheend) { 1426 if (packl > dtheend) { 1427 pb->pb_len = htole32(dtheend); 1428 dtheend = 0; 1429 } else { 1430 pb->pb_len = htole32(packl); 1431 dtheend -= packl; 1432 } 1433 } else 1434 pb->pb_len = htole32(packl); 1435 1436 if ((i + 1) == q->q_dst_nsegs) { 1437 if (maccrd) 1438 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1439 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1440 else 1441 pb->pb_next = 0; 1442 } else 1443 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1444 offsetof(struct ubsec_dmachunk, d_dbuf[j])); 1445 j++; 1446 } 1447 } 1448 1449 dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr + 1450 offsetof(struct ubsec_dmachunk, d_ctx)); 1451 1452 if (sc->sc_flags & UBS_FLAGS_LONGCTX) { 1453 struct ubsec_pktctx_long *ctxl; 1454 1455 ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr + 1456 offsetof(struct ubsec_dmachunk, d_ctx)); 1457 1458 /* transform small context into long context */ 1459 ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long)); 1460 ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC); 1461 ctxl->pc_flags = ctx.pc_flags; 1462 ctxl->pc_offset = ctx.pc_offset; 1463 for (i = 0; i < 6; i++) 1464 ctxl->pc_deskey[i] = ctx.pc_deskey[i]; 1465 for (i = 0; i < 5; i++) 1466 ctxl->pc_hminner[i] = ctx.pc_hminner[i]; 1467 for (i = 0; i < 5; i++) 1468 ctxl->pc_hmouter[i] = ctx.pc_hmouter[i]; 1469 ctxl->pc_iv[0] = ctx.pc_iv[0]; 1470 ctxl->pc_iv[1] = ctx.pc_iv[1]; 1471 } else 1472 bcopy(&ctx, dmap->d_alloc.dma_vaddr + 1473 offsetof(struct ubsec_dmachunk, d_ctx), 1474 sizeof(struct ubsec_pktctx)); 1475 1476 UBSEC_LOCK(sc); 1477 SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next); 1478 sc->sc_nqueue++; 1479 ubsecstats.hst_ipackets++; 1480 ubsecstats.hst_ibytes += dmap->d_alloc.dma_size; 1481 if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= ubsec_maxbatch) 1482 ubsec_feed(sc); 1483 UBSEC_UNLOCK(sc); 1484 return (0); 1485 1486errout: 1487 if (q != NULL) { 1488 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 1489 m_freem(q->q_dst_m); 1490 1491 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1492 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1493 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1494 } 1495 if (q->q_src_map != NULL) { 1496 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1497 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1498 } 1499 1500 UBSEC_LOCK(sc); 1501 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1502 UBSEC_UNLOCK(sc); 1503 } 1504 if (err != ERESTART) { 1505 crp->crp_etype = err; 1506 crypto_done(crp); 1507 } else { 1508 sc->sc_needwakeup |= CRYPTO_SYMQ; 1509 } 1510 return (err); 1511} 1512 1513static void 1514ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q) 1515{ 1516 struct cryptop *crp = (struct cryptop *)q->q_crp; 1517 struct cryptodesc *crd; 1518 struct ubsec_dma *dmap = q->q_dma; 1519 1520 bus_dmamap_sync(sc->sc_dmat, dmap->d_alloc.dma_map, 1521 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1522 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1523 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 1524 BUS_DMASYNC_POSTREAD); 1525 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1526 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1527 } 1528 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE); 1529 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1530 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1531 1532 if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) { 1533 m_freem(q->q_src_m); 1534 crp->crp_buf = (caddr_t)q->q_dst_m; 1535 } 1536 ubsecstats.hst_obytes += ((struct mbuf *)crp->crp_buf)->m_len; 1537 1538 /* copy out IV for future use */ 1539 if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) { 1540 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1541 if (crd->crd_alg != CRYPTO_DES_CBC && 1542 crd->crd_alg != CRYPTO_3DES_CBC) 1543 continue; 1544 if (crp->crp_flags & CRYPTO_F_IMBUF) 1545 m_copydata((struct mbuf *)crp->crp_buf, 1546 crd->crd_skip + crd->crd_len - 8, 8, 1547 (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv); 1548 else if (crp->crp_flags & CRYPTO_F_IOV) { 1549 cuio_copydata((struct uio *)crp->crp_buf, 1550 crd->crd_skip + crd->crd_len - 8, 8, 1551 (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv); 1552 } 1553 break; 1554 } 1555 } 1556 1557 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1558 if (crd->crd_alg != CRYPTO_MD5_HMAC && 1559 crd->crd_alg != CRYPTO_SHA1_HMAC) 1560 continue; 1561 if (crp->crp_flags & CRYPTO_F_IMBUF) 1562 m_copyback((struct mbuf *)crp->crp_buf, 1563 crd->crd_inject, 12, 1564 (caddr_t)dmap->d_dma->d_macbuf); 1565 else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac) 1566 bcopy((caddr_t)dmap->d_dma->d_macbuf, 1567 crp->crp_mac, 12); 1568 break; 1569 } 1570 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1571 crypto_done(crp); 1572} 1573 1574static void 1575ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset) 1576{ 1577 int i, j, dlen, slen; 1578 caddr_t dptr, sptr; 1579 1580 j = 0; 1581 sptr = srcm->m_data; 1582 slen = srcm->m_len; 1583 dptr = dstm->m_data; 1584 dlen = dstm->m_len; 1585 1586 while (1) { 1587 for (i = 0; i < min(slen, dlen); i++) { 1588 if (j < hoffset || j >= toffset) 1589 *dptr++ = *sptr++; 1590 slen--; 1591 dlen--; 1592 j++; 1593 } 1594 if (slen == 0) { 1595 srcm = srcm->m_next; 1596 if (srcm == NULL) 1597 return; 1598 sptr = srcm->m_data; 1599 slen = srcm->m_len; 1600 } 1601 if (dlen == 0) { 1602 dstm = dstm->m_next; 1603 if (dstm == NULL) 1604 return; 1605 dptr = dstm->m_data; 1606 dlen = dstm->m_len; 1607 } 1608 } 1609} 1610 1611/* 1612 * feed the key generator, must be called at splimp() or higher. 1613 */ 1614static int 1615ubsec_feed2(struct ubsec_softc *sc) 1616{ 1617 struct ubsec_q2 *q; 1618 1619 while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) { 1620 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL) 1621 break; 1622 q = SIMPLEQ_FIRST(&sc->sc_queue2); 1623 1624 bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 1625 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1626 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 1627 BUS_DMASYNC_PREWRITE); 1628 1629 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr); 1630 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q, q_next); 1631 --sc->sc_nqueue2; 1632 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next); 1633 } 1634 return (0); 1635} 1636 1637/* 1638 * Callback for handling random numbers 1639 */ 1640static void 1641ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q) 1642{ 1643 struct cryptkop *krp; 1644 struct ubsec_ctx_keyop *ctx; 1645 1646 ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr; 1647 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, BUS_DMASYNC_POSTWRITE); 1648 1649 switch (q->q_type) { 1650#ifndef UBSEC_NO_RNG 1651 case UBS_CTXOP_RNGBYPASS: { 1652 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q; 1653 1654 bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 1655 BUS_DMASYNC_POSTREAD); 1656 random_harvest(rng->rng_buf.dma_vaddr, 1657 UBSEC_RNG_BUFSIZ*sizeof (u_int32_t), 1658 UBSEC_RNG_BUFSIZ*sizeof (u_int32_t)*NBBY, 0, 1659 RANDOM_PURE); 1660 rng->rng_used = 0; 1661 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1662 break; 1663 } 1664#endif 1665 case UBS_CTXOP_MODEXP: { 1666 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 1667 u_int rlen, clen; 1668 1669 krp = me->me_krp; 1670 rlen = (me->me_modbits + 7) / 8; 1671 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8; 1672 1673 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 1674 BUS_DMASYNC_POSTWRITE); 1675 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 1676 BUS_DMASYNC_POSTWRITE); 1677 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 1678 BUS_DMASYNC_POSTREAD); 1679 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 1680 BUS_DMASYNC_POSTWRITE); 1681 1682 if (clen < rlen) 1683 krp->krp_status = E2BIG; 1684 else { 1685 if (sc->sc_flags & UBS_FLAGS_HWNORM) { 1686 bzero(krp->krp_param[krp->krp_iparams].crp_p, 1687 (krp->krp_param[krp->krp_iparams].crp_nbits 1688 + 7) / 8); 1689 bcopy(me->me_C.dma_vaddr, 1690 krp->krp_param[krp->krp_iparams].crp_p, 1691 (me->me_modbits + 7) / 8); 1692 } else 1693 ubsec_kshift_l(me->me_shiftbits, 1694 me->me_C.dma_vaddr, me->me_normbits, 1695 krp->krp_param[krp->krp_iparams].crp_p, 1696 krp->krp_param[krp->krp_iparams].crp_nbits); 1697 } 1698 1699 crypto_kdone(krp); 1700 1701 /* bzero all potentially sensitive data */ 1702 bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 1703 bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 1704 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 1705 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 1706 1707 /* Can't free here, so put us on the free list. */ 1708 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next); 1709 break; 1710 } 1711 case UBS_CTXOP_RSAPRIV: { 1712 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 1713 u_int len; 1714 1715 krp = rp->rpr_krp; 1716 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 1717 BUS_DMASYNC_POSTWRITE); 1718 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 1719 BUS_DMASYNC_POSTREAD); 1720 1721 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8; 1722 bcopy(rp->rpr_msgout.dma_vaddr, 1723 krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len); 1724 1725 crypto_kdone(krp); 1726 1727 bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 1728 bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 1729 bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size); 1730 1731 /* Can't free here, so put us on the free list. */ 1732 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next); 1733 break; 1734 } 1735 default: 1736 device_printf(sc->sc_dev, "unknown ctx op: %x\n", 1737 letoh16(ctx->ctx_op)); 1738 break; 1739 } 1740} 1741 1742#ifndef UBSEC_NO_RNG 1743static void 1744ubsec_rng(void *vsc) 1745{ 1746 struct ubsec_softc *sc = vsc; 1747 struct ubsec_q2_rng *rng = &sc->sc_rng; 1748 struct ubsec_mcr *mcr; 1749 struct ubsec_ctx_rngbypass *ctx; 1750 1751 UBSEC_LOCK(sc); 1752 if (rng->rng_used) { 1753 UBSEC_UNLOCK(sc); 1754 return; 1755 } 1756 sc->sc_nqueue2++; 1757 if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE) 1758 goto out; 1759 1760 mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr; 1761 ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr; 1762 1763 mcr->mcr_pkts = htole16(1); 1764 mcr->mcr_flags = 0; 1765 mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr); 1766 mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0; 1767 mcr->mcr_ipktbuf.pb_len = 0; 1768 mcr->mcr_reserved = mcr->mcr_pktlen = 0; 1769 mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr); 1770 mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) & 1771 UBS_PKTBUF_LEN); 1772 mcr->mcr_opktbuf.pb_next = 0; 1773 1774 ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass)); 1775 ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS); 1776 rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS; 1777 1778 bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, BUS_DMASYNC_PREREAD); 1779 1780 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next); 1781 rng->rng_used = 1; 1782 ubsec_feed2(sc); 1783 ubsecstats.hst_rng++; 1784 UBSEC_UNLOCK(sc); 1785 1786 return; 1787 1788out: 1789 /* 1790 * Something weird happened, generate our own call back. 1791 */ 1792 sc->sc_nqueue2--; 1793 UBSEC_UNLOCK(sc); 1794 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1795} 1796#endif /* UBSEC_NO_RNG */ 1797 1798static void 1799ubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 1800{ 1801 bus_addr_t *paddr = (bus_addr_t*) arg; 1802 *paddr = segs->ds_addr; 1803} 1804 1805static int 1806ubsec_dma_malloc( 1807 struct ubsec_softc *sc, 1808 bus_size_t size, 1809 struct ubsec_dma_alloc *dma, 1810 int mapflags 1811) 1812{ 1813 int r; 1814 1815 r = bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &dma->dma_map); 1816 if (r != 0) 1817 goto fail_0; 1818 1819 r = bus_dmamem_alloc(sc->sc_dmat, (void**) &dma->dma_vaddr, 1820 BUS_DMA_NOWAIT, &dma->dma_map); 1821 if (r != 0) 1822 goto fail_1; 1823 1824 r = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr, 1825 size, 1826 ubsec_dmamap_cb, 1827 &dma->dma_paddr, 1828 mapflags | BUS_DMA_NOWAIT); 1829 if (r != 0) 1830 goto fail_2; 1831 1832 dma->dma_size = size; 1833 return (0); 1834 1835fail_2: 1836 bus_dmamap_unload(sc->sc_dmat, dma->dma_map); 1837fail_1: 1838 bus_dmamem_free(sc->sc_dmat, dma->dma_vaddr, dma->dma_map); 1839fail_0: 1840 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map); 1841 dma->dma_map = NULL; 1842 return (r); 1843} 1844 1845static void 1846ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma) 1847{ 1848 bus_dmamap_unload(sc->sc_dmat, dma->dma_map); 1849 bus_dmamem_free(sc->sc_dmat, dma->dma_vaddr, dma->dma_map); 1850 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map); 1851} 1852 1853/* 1854 * Resets the board. Values in the regesters are left as is 1855 * from the reset (i.e. initial values are assigned elsewhere). 1856 */ 1857static void 1858ubsec_reset_board(struct ubsec_softc *sc) 1859{ 1860 volatile u_int32_t ctrl; 1861 1862 ctrl = READ_REG(sc, BS_CTRL); 1863 ctrl |= BS_CTRL_RESET; 1864 WRITE_REG(sc, BS_CTRL, ctrl); 1865 1866 /* 1867 * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us 1868 */ 1869 DELAY(10); 1870} 1871 1872/* 1873 * Init Broadcom registers 1874 */ 1875static void 1876ubsec_init_board(struct ubsec_softc *sc) 1877{ 1878 u_int32_t ctrl; 1879 1880 ctrl = READ_REG(sc, BS_CTRL); 1881 ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64); 1882 ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT; 1883 1884 if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) 1885 ctrl |= BS_CTRL_MCR2INT; 1886 else 1887 ctrl &= ~BS_CTRL_MCR2INT; 1888 1889 if (sc->sc_flags & UBS_FLAGS_HWNORM) 1890 ctrl &= ~BS_CTRL_SWNORM; 1891 1892 WRITE_REG(sc, BS_CTRL, ctrl); 1893} 1894 1895/* 1896 * Init Broadcom PCI registers 1897 */ 1898static void 1899ubsec_init_pciregs(device_t dev) 1900{ 1901#if 0 1902 u_int32_t misc; 1903 1904 misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT); 1905 misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT)) 1906 | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT); 1907 misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT)) 1908 | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT); 1909 pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc); 1910#endif 1911 1912 /* 1913 * This will set the cache line size to 1, this will 1914 * force the BCM58xx chip just to do burst read/writes. 1915 * Cache line read/writes are to slow 1916 */ 1917 pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1); 1918} 1919 1920/* 1921 * Clean up after a chip crash. 1922 * It is assumed that the caller in splimp() 1923 */ 1924static void 1925ubsec_cleanchip(struct ubsec_softc *sc) 1926{ 1927 struct ubsec_q *q; 1928 1929 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 1930 q = SIMPLEQ_FIRST(&sc->sc_qchip); 1931 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next); 1932 ubsec_free_q(sc, q); 1933 } 1934 sc->sc_nqchip = 0; 1935} 1936 1937/* 1938 * Free an ubsec_q. 1939 * It is assumed that the caller is within spimp(). 1940 */ 1941static int 1942ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q) 1943{ 1944 struct ubsec_q *q2; 1945 struct cryptop *crp; 1946 int npkts; 1947 int i; 1948 1949 npkts = q->q_nstacked_mcrs; 1950 1951 for (i = 0; i < npkts; i++) { 1952 if(q->q_stacked_mcr[i]) { 1953 q2 = q->q_stacked_mcr[i]; 1954 1955 if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m)) 1956 m_freem(q2->q_dst_m); 1957 1958 crp = (struct cryptop *)q2->q_crp; 1959 1960 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next); 1961 1962 crp->crp_etype = EFAULT; 1963 crypto_done(crp); 1964 } else { 1965 break; 1966 } 1967 } 1968 1969 /* 1970 * Free header MCR 1971 */ 1972 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 1973 m_freem(q->q_dst_m); 1974 1975 crp = (struct cryptop *)q->q_crp; 1976 1977 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1978 1979 crp->crp_etype = EFAULT; 1980 crypto_done(crp); 1981 return(0); 1982} 1983 1984/* 1985 * Routine to reset the chip and clean up. 1986 * It is assumed that the caller is in splimp() 1987 */ 1988static void 1989ubsec_totalreset(struct ubsec_softc *sc) 1990{ 1991 ubsec_reset_board(sc); 1992 ubsec_init_board(sc); 1993 ubsec_cleanchip(sc); 1994} 1995 1996static int 1997ubsec_dmamap_aligned(struct ubsec_operand *op) 1998{ 1999 int i; 2000 2001 for (i = 0; i < op->nsegs; i++) { 2002 if (op->segs[i].ds_addr & 3) 2003 return (0); 2004 if ((i != (op->nsegs - 1)) && 2005 (op->segs[i].ds_len & 3)) 2006 return (0); 2007 } 2008 return (1); 2009} 2010 2011static void 2012ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q) 2013{ 2014 switch (q->q_type) { 2015 case UBS_CTXOP_MODEXP: { 2016 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 2017 2018 ubsec_dma_free(sc, &me->me_q.q_mcr); 2019 ubsec_dma_free(sc, &me->me_q.q_ctx); 2020 ubsec_dma_free(sc, &me->me_M); 2021 ubsec_dma_free(sc, &me->me_E); 2022 ubsec_dma_free(sc, &me->me_C); 2023 ubsec_dma_free(sc, &me->me_epb); 2024 free(me, M_DEVBUF); 2025 break; 2026 } 2027 case UBS_CTXOP_RSAPRIV: { 2028 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 2029 2030 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2031 ubsec_dma_free(sc, &rp->rpr_q.q_ctx); 2032 ubsec_dma_free(sc, &rp->rpr_msgin); 2033 ubsec_dma_free(sc, &rp->rpr_msgout); 2034 free(rp, M_DEVBUF); 2035 break; 2036 } 2037 default: 2038 device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type); 2039 break; 2040 } 2041} 2042 2043static int 2044ubsec_kprocess(void *arg, struct cryptkop *krp, int hint) 2045{ 2046 struct ubsec_softc *sc = arg; 2047 int r; 2048 2049 if (krp == NULL || krp->krp_callback == NULL) 2050 return (EINVAL); 2051 2052 while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) { 2053 struct ubsec_q2 *q; 2054 2055 q = SIMPLEQ_FIRST(&sc->sc_q2free); 2056 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q, q_next); 2057 ubsec_kfree(sc, q); 2058 } 2059 2060 switch (krp->krp_op) { 2061 case CRK_MOD_EXP: 2062 if (sc->sc_flags & UBS_FLAGS_HWNORM) 2063 r = ubsec_kprocess_modexp_hw(sc, krp, hint); 2064 else 2065 r = ubsec_kprocess_modexp_sw(sc, krp, hint); 2066 break; 2067 case CRK_MOD_EXP_CRT: 2068 return (ubsec_kprocess_rsapriv(sc, krp, hint)); 2069 default: 2070 device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n", 2071 krp->krp_op); 2072 krp->krp_status = EOPNOTSUPP; 2073 crypto_kdone(krp); 2074 return (0); 2075 } 2076 return (0); /* silence compiler */ 2077} 2078 2079/* 2080 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization) 2081 */ 2082static int 2083ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2084{ 2085 struct ubsec_q2_modexp *me; 2086 struct ubsec_mcr *mcr; 2087 struct ubsec_ctx_modexp *ctx; 2088 struct ubsec_pktbuf *epb; 2089 int err = 0; 2090 u_int nbits, normbits, mbits, shiftbits, ebits; 2091 2092 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2093 if (me == NULL) { 2094 err = ENOMEM; 2095 goto errout; 2096 } 2097 bzero(me, sizeof *me); 2098 me->me_krp = krp; 2099 me->me_q.q_type = UBS_CTXOP_MODEXP; 2100 2101 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2102 if (nbits <= 512) 2103 normbits = 512; 2104 else if (nbits <= 768) 2105 normbits = 768; 2106 else if (nbits <= 1024) 2107 normbits = 1024; 2108 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2109 normbits = 1536; 2110 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2111 normbits = 2048; 2112 else { 2113 err = E2BIG; 2114 goto errout; 2115 } 2116 2117 shiftbits = normbits - nbits; 2118 2119 me->me_modbits = nbits; 2120 me->me_shiftbits = shiftbits; 2121 me->me_normbits = normbits; 2122 2123 /* Sanity check: result bits must be >= true modulus bits. */ 2124 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2125 err = ERANGE; 2126 goto errout; 2127 } 2128 2129 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2130 &me->me_q.q_mcr, 0)) { 2131 err = ENOMEM; 2132 goto errout; 2133 } 2134 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2135 2136 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2137 &me->me_q.q_ctx, 0)) { 2138 err = ENOMEM; 2139 goto errout; 2140 } 2141 2142 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2143 if (mbits > nbits) { 2144 err = E2BIG; 2145 goto errout; 2146 } 2147 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2148 err = ENOMEM; 2149 goto errout; 2150 } 2151 ubsec_kshift_r(shiftbits, 2152 krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits, 2153 me->me_M.dma_vaddr, normbits); 2154 2155 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2156 err = ENOMEM; 2157 goto errout; 2158 } 2159 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2160 2161 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2162 if (ebits > nbits) { 2163 err = E2BIG; 2164 goto errout; 2165 } 2166 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2167 err = ENOMEM; 2168 goto errout; 2169 } 2170 ubsec_kshift_r(shiftbits, 2171 krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits, 2172 me->me_E.dma_vaddr, normbits); 2173 2174 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2175 &me->me_epb, 0)) { 2176 err = ENOMEM; 2177 goto errout; 2178 } 2179 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2180 epb->pb_addr = htole32(me->me_E.dma_paddr); 2181 epb->pb_next = 0; 2182 epb->pb_len = htole32(normbits / 8); 2183 2184#ifdef UBSEC_DEBUG 2185 if (ubsec_debug) { 2186 printf("Epb "); 2187 ubsec_dump_pb(epb); 2188 } 2189#endif 2190 2191 mcr->mcr_pkts = htole16(1); 2192 mcr->mcr_flags = 0; 2193 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2194 mcr->mcr_reserved = 0; 2195 mcr->mcr_pktlen = 0; 2196 2197 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2198 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2199 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2200 2201 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2202 mcr->mcr_opktbuf.pb_next = 0; 2203 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2204 2205#ifdef DIAGNOSTIC 2206 /* Misaligned output buffer will hang the chip. */ 2207 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2208 panic("%s: modexp invalid addr 0x%x\n", 2209 device_get_nameunit(sc->sc_dev), 2210 letoh32(mcr->mcr_opktbuf.pb_addr)); 2211 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2212 panic("%s: modexp invalid len 0x%x\n", 2213 device_get_nameunit(sc->sc_dev), 2214 letoh32(mcr->mcr_opktbuf.pb_len)); 2215#endif 2216 2217 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2218 bzero(ctx, sizeof(*ctx)); 2219 ubsec_kshift_r(shiftbits, 2220 krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits, 2221 ctx->me_N, normbits); 2222 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2223 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2224 ctx->me_E_len = htole16(nbits); 2225 ctx->me_N_len = htole16(nbits); 2226 2227#ifdef UBSEC_DEBUG 2228 if (ubsec_debug) { 2229 ubsec_dump_mcr(mcr); 2230 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2231 } 2232#endif 2233 2234 /* 2235 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2236 * everything else. 2237 */ 2238 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, BUS_DMASYNC_PREWRITE); 2239 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, BUS_DMASYNC_PREWRITE); 2240 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, BUS_DMASYNC_PREREAD); 2241 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, BUS_DMASYNC_PREWRITE); 2242 2243 /* Enqueue and we're done... */ 2244 UBSEC_LOCK(sc); 2245 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2246 ubsec_feed2(sc); 2247 ubsecstats.hst_modexp++; 2248 UBSEC_UNLOCK(sc); 2249 2250 return (0); 2251 2252errout: 2253 if (me != NULL) { 2254 if (me->me_q.q_mcr.dma_map != NULL) 2255 ubsec_dma_free(sc, &me->me_q.q_mcr); 2256 if (me->me_q.q_ctx.dma_map != NULL) { 2257 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2258 ubsec_dma_free(sc, &me->me_q.q_ctx); 2259 } 2260 if (me->me_M.dma_map != NULL) { 2261 bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2262 ubsec_dma_free(sc, &me->me_M); 2263 } 2264 if (me->me_E.dma_map != NULL) { 2265 bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2266 ubsec_dma_free(sc, &me->me_E); 2267 } 2268 if (me->me_C.dma_map != NULL) { 2269 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2270 ubsec_dma_free(sc, &me->me_C); 2271 } 2272 if (me->me_epb.dma_map != NULL) 2273 ubsec_dma_free(sc, &me->me_epb); 2274 free(me, M_DEVBUF); 2275 } 2276 krp->krp_status = err; 2277 crypto_kdone(krp); 2278 return (0); 2279} 2280 2281/* 2282 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization) 2283 */ 2284static int 2285ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2286{ 2287 struct ubsec_q2_modexp *me; 2288 struct ubsec_mcr *mcr; 2289 struct ubsec_ctx_modexp *ctx; 2290 struct ubsec_pktbuf *epb; 2291 int err = 0; 2292 u_int nbits, normbits, mbits, shiftbits, ebits; 2293 2294 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2295 if (me == NULL) { 2296 err = ENOMEM; 2297 goto errout; 2298 } 2299 bzero(me, sizeof *me); 2300 me->me_krp = krp; 2301 me->me_q.q_type = UBS_CTXOP_MODEXP; 2302 2303 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2304 if (nbits <= 512) 2305 normbits = 512; 2306 else if (nbits <= 768) 2307 normbits = 768; 2308 else if (nbits <= 1024) 2309 normbits = 1024; 2310 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2311 normbits = 1536; 2312 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2313 normbits = 2048; 2314 else { 2315 err = E2BIG; 2316 goto errout; 2317 } 2318 2319 shiftbits = normbits - nbits; 2320 2321 /* XXX ??? */ 2322 me->me_modbits = nbits; 2323 me->me_shiftbits = shiftbits; 2324 me->me_normbits = normbits; 2325 2326 /* Sanity check: result bits must be >= true modulus bits. */ 2327 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2328 err = ERANGE; 2329 goto errout; 2330 } 2331 2332 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2333 &me->me_q.q_mcr, 0)) { 2334 err = ENOMEM; 2335 goto errout; 2336 } 2337 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2338 2339 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2340 &me->me_q.q_ctx, 0)) { 2341 err = ENOMEM; 2342 goto errout; 2343 } 2344 2345 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2346 if (mbits > nbits) { 2347 err = E2BIG; 2348 goto errout; 2349 } 2350 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2351 err = ENOMEM; 2352 goto errout; 2353 } 2354 bzero(me->me_M.dma_vaddr, normbits / 8); 2355 bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p, 2356 me->me_M.dma_vaddr, (mbits + 7) / 8); 2357 2358 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2359 err = ENOMEM; 2360 goto errout; 2361 } 2362 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2363 2364 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2365 if (ebits > nbits) { 2366 err = E2BIG; 2367 goto errout; 2368 } 2369 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2370 err = ENOMEM; 2371 goto errout; 2372 } 2373 bzero(me->me_E.dma_vaddr, normbits / 8); 2374 bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p, 2375 me->me_E.dma_vaddr, (ebits + 7) / 8); 2376 2377 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2378 &me->me_epb, 0)) { 2379 err = ENOMEM; 2380 goto errout; 2381 } 2382 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2383 epb->pb_addr = htole32(me->me_E.dma_paddr); 2384 epb->pb_next = 0; 2385 epb->pb_len = htole32((ebits + 7) / 8); 2386 2387#ifdef UBSEC_DEBUG 2388 printf("Epb "); 2389 ubsec_dump_pb(epb); 2390#endif 2391 2392 mcr->mcr_pkts = htole16(1); 2393 mcr->mcr_flags = 0; 2394 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2395 mcr->mcr_reserved = 0; 2396 mcr->mcr_pktlen = 0; 2397 2398 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2399 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2400 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2401 2402 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2403 mcr->mcr_opktbuf.pb_next = 0; 2404 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2405 2406#ifdef DIAGNOSTIC 2407 /* Misaligned output buffer will hang the chip. */ 2408 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2409 panic("%s: modexp invalid addr 0x%x\n", 2410 device_get_nameunit(sc->sc_dev), 2411 letoh32(mcr->mcr_opktbuf.pb_addr)); 2412 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2413 panic("%s: modexp invalid len 0x%x\n", 2414 device_get_nameunit(sc->sc_dev), 2415 letoh32(mcr->mcr_opktbuf.pb_len)); 2416#endif 2417 2418 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2419 bzero(ctx, sizeof(*ctx)); 2420 bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N, 2421 (nbits + 7) / 8); 2422 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2423 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2424 ctx->me_E_len = htole16(ebits); 2425 ctx->me_N_len = htole16(nbits); 2426 2427#ifdef UBSEC_DEBUG 2428 if (ubsec_debug) { 2429 ubsec_dump_mcr(mcr); 2430 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2431 } 2432#endif 2433 2434 /* 2435 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2436 * everything else. 2437 */ 2438 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, BUS_DMASYNC_PREWRITE); 2439 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, BUS_DMASYNC_PREWRITE); 2440 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, BUS_DMASYNC_PREREAD); 2441 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, BUS_DMASYNC_PREWRITE); 2442 2443 /* Enqueue and we're done... */ 2444 UBSEC_LOCK(sc); 2445 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2446 ubsec_feed2(sc); 2447 UBSEC_UNLOCK(sc); 2448 2449 return (0); 2450 2451errout: 2452 if (me != NULL) { 2453 if (me->me_q.q_mcr.dma_map != NULL) 2454 ubsec_dma_free(sc, &me->me_q.q_mcr); 2455 if (me->me_q.q_ctx.dma_map != NULL) { 2456 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2457 ubsec_dma_free(sc, &me->me_q.q_ctx); 2458 } 2459 if (me->me_M.dma_map != NULL) { 2460 bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2461 ubsec_dma_free(sc, &me->me_M); 2462 } 2463 if (me->me_E.dma_map != NULL) { 2464 bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2465 ubsec_dma_free(sc, &me->me_E); 2466 } 2467 if (me->me_C.dma_map != NULL) { 2468 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2469 ubsec_dma_free(sc, &me->me_C); 2470 } 2471 if (me->me_epb.dma_map != NULL) 2472 ubsec_dma_free(sc, &me->me_epb); 2473 free(me, M_DEVBUF); 2474 } 2475 krp->krp_status = err; 2476 crypto_kdone(krp); 2477 return (0); 2478} 2479 2480static int 2481ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2482{ 2483 struct ubsec_q2_rsapriv *rp = NULL; 2484 struct ubsec_mcr *mcr; 2485 struct ubsec_ctx_rsapriv *ctx; 2486 int err = 0; 2487 u_int padlen, msglen; 2488 2489 msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]); 2490 padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]); 2491 if (msglen > padlen) 2492 padlen = msglen; 2493 2494 if (padlen <= 256) 2495 padlen = 256; 2496 else if (padlen <= 384) 2497 padlen = 384; 2498 else if (padlen <= 512) 2499 padlen = 512; 2500 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768) 2501 padlen = 768; 2502 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024) 2503 padlen = 1024; 2504 else { 2505 err = E2BIG; 2506 goto errout; 2507 } 2508 2509 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) { 2510 err = E2BIG; 2511 goto errout; 2512 } 2513 2514 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) { 2515 err = E2BIG; 2516 goto errout; 2517 } 2518 2519 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) { 2520 err = E2BIG; 2521 goto errout; 2522 } 2523 2524 rp = (struct ubsec_q2_rsapriv *)malloc(sizeof *rp, M_DEVBUF, M_NOWAIT); 2525 if (rp == NULL) 2526 return (ENOMEM); 2527 bzero(rp, sizeof *rp); 2528 rp->rpr_krp = krp; 2529 rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV; 2530 2531 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2532 &rp->rpr_q.q_mcr, 0)) { 2533 err = ENOMEM; 2534 goto errout; 2535 } 2536 mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr; 2537 2538 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv), 2539 &rp->rpr_q.q_ctx, 0)) { 2540 err = ENOMEM; 2541 goto errout; 2542 } 2543 ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr; 2544 bzero(ctx, sizeof *ctx); 2545 2546 /* Copy in p */ 2547 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p, 2548 &ctx->rpr_buf[0 * (padlen / 8)], 2549 (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8); 2550 2551 /* Copy in q */ 2552 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p, 2553 &ctx->rpr_buf[1 * (padlen / 8)], 2554 (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8); 2555 2556 /* Copy in dp */ 2557 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p, 2558 &ctx->rpr_buf[2 * (padlen / 8)], 2559 (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8); 2560 2561 /* Copy in dq */ 2562 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p, 2563 &ctx->rpr_buf[3 * (padlen / 8)], 2564 (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8); 2565 2566 /* Copy in pinv */ 2567 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p, 2568 &ctx->rpr_buf[4 * (padlen / 8)], 2569 (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8); 2570 2571 msglen = padlen * 2; 2572 2573 /* Copy in input message (aligned buffer/length). */ 2574 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) { 2575 /* Is this likely? */ 2576 err = E2BIG; 2577 goto errout; 2578 } 2579 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) { 2580 err = ENOMEM; 2581 goto errout; 2582 } 2583 bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8); 2584 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p, 2585 rp->rpr_msgin.dma_vaddr, 2586 (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8); 2587 2588 /* Prepare space for output message (aligned buffer/length). */ 2589 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) { 2590 /* Is this likely? */ 2591 err = E2BIG; 2592 goto errout; 2593 } 2594 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) { 2595 err = ENOMEM; 2596 goto errout; 2597 } 2598 bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8); 2599 2600 mcr->mcr_pkts = htole16(1); 2601 mcr->mcr_flags = 0; 2602 mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr); 2603 mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr); 2604 mcr->mcr_ipktbuf.pb_next = 0; 2605 mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size); 2606 mcr->mcr_reserved = 0; 2607 mcr->mcr_pktlen = htole16(msglen); 2608 mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr); 2609 mcr->mcr_opktbuf.pb_next = 0; 2610 mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size); 2611 2612#ifdef DIAGNOSTIC 2613 if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) { 2614 panic("%s: rsapriv: invalid msgin %x(0x%jx)", 2615 device_get_nameunit(sc->sc_dev), 2616 rp->rpr_msgin.dma_paddr, (uintmax_t)rp->rpr_msgin.dma_size); 2617 } 2618 if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) { 2619 panic("%s: rsapriv: invalid msgout %x(0x%jx)", 2620 device_get_nameunit(sc->sc_dev), 2621 rp->rpr_msgout.dma_paddr, (uintmax_t)rp->rpr_msgout.dma_size); 2622 } 2623#endif 2624 2625 ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8)); 2626 ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV); 2627 ctx->rpr_q_len = htole16(padlen); 2628 ctx->rpr_p_len = htole16(padlen); 2629 2630 /* 2631 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2632 * everything else. 2633 */ 2634 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 2635 BUS_DMASYNC_PREWRITE); 2636 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 2637 BUS_DMASYNC_PREREAD); 2638 2639 /* Enqueue and we're done... */ 2640 UBSEC_LOCK(sc); 2641 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next); 2642 ubsec_feed2(sc); 2643 ubsecstats.hst_modexpcrt++; 2644 UBSEC_UNLOCK(sc); 2645 return (0); 2646 2647errout: 2648 if (rp != NULL) { 2649 if (rp->rpr_q.q_mcr.dma_map != NULL) 2650 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2651 if (rp->rpr_msgin.dma_map != NULL) { 2652 bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 2653 ubsec_dma_free(sc, &rp->rpr_msgin); 2654 } 2655 if (rp->rpr_msgout.dma_map != NULL) { 2656 bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 2657 ubsec_dma_free(sc, &rp->rpr_msgout); 2658 } 2659 free(rp, M_DEVBUF); 2660 } 2661 krp->krp_status = err; 2662 crypto_kdone(krp); 2663 return (0); 2664} 2665 2666#ifdef UBSEC_DEBUG 2667static void 2668ubsec_dump_pb(volatile struct ubsec_pktbuf *pb) 2669{ 2670 printf("addr 0x%x (0x%x) next 0x%x\n", 2671 pb->pb_addr, pb->pb_len, pb->pb_next); 2672} 2673 2674static void 2675ubsec_dump_ctx2(struct ubsec_ctx_keyop *c) 2676{ 2677 printf("CTX (0x%x):\n", c->ctx_len); 2678 switch (letoh16(c->ctx_op)) { 2679 case UBS_CTXOP_RNGBYPASS: 2680 case UBS_CTXOP_RNGSHA1: 2681 break; 2682 case UBS_CTXOP_MODEXP: 2683 { 2684 struct ubsec_ctx_modexp *cx = (void *)c; 2685 int i, len; 2686 2687 printf(" Elen %u, Nlen %u\n", 2688 letoh16(cx->me_E_len), letoh16(cx->me_N_len)); 2689 len = (cx->me_N_len + 7)/8; 2690 for (i = 0; i < len; i++) 2691 printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]); 2692 printf("\n"); 2693 break; 2694 } 2695 default: 2696 printf("unknown context: %x\n", c->ctx_op); 2697 } 2698 printf("END CTX\n"); 2699} 2700 2701static void 2702ubsec_dump_mcr(struct ubsec_mcr *mcr) 2703{ 2704 volatile struct ubsec_mcr_add *ma; 2705 int i; 2706 2707 printf("MCR:\n"); 2708 printf(" pkts: %u, flags 0x%x\n", 2709 letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags)); 2710 ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp; 2711 for (i = 0; i < letoh16(mcr->mcr_pkts); i++) { 2712 printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i, 2713 letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen), 2714 letoh16(ma->mcr_reserved)); 2715 printf(" %d: ipkt ", i); 2716 ubsec_dump_pb(&ma->mcr_ipktbuf); 2717 printf(" %d: opkt ", i); 2718 ubsec_dump_pb(&ma->mcr_opktbuf); 2719 ma++; 2720 } 2721 printf("END MCR\n"); 2722} 2723#endif /* UBSEC_DEBUG */ 2724 2725/* 2726 * Return the number of significant bits of a big number. 2727 */ 2728static int 2729ubsec_ksigbits(struct crparam *cr) 2730{ 2731 u_int plen = (cr->crp_nbits + 7) / 8; 2732 int i, sig = plen * 8; 2733 u_int8_t c, *p = cr->crp_p; 2734 2735 for (i = plen - 1; i >= 0; i--) { 2736 c = p[i]; 2737 if (c != 0) { 2738 while ((c & 0x80) == 0) { 2739 sig--; 2740 c <<= 1; 2741 } 2742 break; 2743 } 2744 sig -= 8; 2745 } 2746 return (sig); 2747} 2748 2749static void 2750ubsec_kshift_r( 2751 u_int shiftbits, 2752 u_int8_t *src, u_int srcbits, 2753 u_int8_t *dst, u_int dstbits) 2754{ 2755 u_int slen, dlen; 2756 int i, si, di, n; 2757 2758 slen = (srcbits + 7) / 8; 2759 dlen = (dstbits + 7) / 8; 2760 2761 for (i = 0; i < slen; i++) 2762 dst[i] = src[i]; 2763 for (i = 0; i < dlen - slen; i++) 2764 dst[slen + i] = 0; 2765 2766 n = shiftbits / 8; 2767 if (n != 0) { 2768 si = dlen - n - 1; 2769 di = dlen - 1; 2770 while (si >= 0) 2771 dst[di--] = dst[si--]; 2772 while (di >= 0) 2773 dst[di--] = 0; 2774 } 2775 2776 n = shiftbits % 8; 2777 if (n != 0) { 2778 for (i = dlen - 1; i > 0; i--) 2779 dst[i] = (dst[i] << n) | 2780 (dst[i - 1] >> (8 - n)); 2781 dst[0] = dst[0] << n; 2782 } 2783} 2784 2785static void 2786ubsec_kshift_l( 2787 u_int shiftbits, 2788 u_int8_t *src, u_int srcbits, 2789 u_int8_t *dst, u_int dstbits) 2790{ 2791 int slen, dlen, i, n; 2792 2793 slen = (srcbits + 7) / 8; 2794 dlen = (dstbits + 7) / 8; 2795 2796 n = shiftbits / 8; 2797 for (i = 0; i < slen; i++) 2798 dst[i] = src[i + n]; 2799 for (i = 0; i < dlen - slen; i++) 2800 dst[slen + i] = 0; 2801 2802 n = shiftbits % 8; 2803 if (n != 0) { 2804 for (i = 0; i < (dlen - 1); i++) 2805 dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n)); 2806 dst[dlen - 1] = dst[dlen - 1] >> n; 2807 } 2808} 2809