ubsec.c revision 110522
1/* $FreeBSD: head/sys/dev/ubsec/ubsec.c 110522 2003-02-07 23:02:02Z sam $ */ 2/* $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $ */ 3 4/* 5 * Copyright (c) 2000 Jason L. Wright (jason@thought.net) 6 * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org) 7 * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com) 8 * 9 * All rights reserved. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by Jason L. Wright 22 * 4. The name of the author may not be used to endorse or promote products 23 * derived from this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 26 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 27 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 28 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 29 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 30 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 31 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 33 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 34 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 * 37 * Effort sponsored in part by the Defense Advanced Research Projects 38 * Agency (DARPA) and Air Force Research Laboratory, Air Force 39 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 40 * 41 */ 42 43#define UBSEC_DEBUG 44 45/* 46 * uBsec 5[56]01, 58xx hardware crypto accelerator 47 */ 48 49#include <sys/param.h> 50#include <sys/systm.h> 51#include <sys/proc.h> 52#include <sys/errno.h> 53#include <sys/malloc.h> 54#include <sys/kernel.h> 55#include <sys/mbuf.h> 56#include <sys/lock.h> 57#include <sys/mutex.h> 58#include <sys/stdint.h> 59#include <sys/sysctl.h> 60#include <sys/endian.h> 61 62#include <vm/vm.h> 63#include <vm/pmap.h> 64 65#include <machine/clock.h> 66#include <machine/bus.h> 67#include <machine/resource.h> 68#include <sys/bus.h> 69#include <sys/rman.h> 70 71#include <crypto/sha1.h> 72#include <opencrypto/cryptodev.h> 73#include <opencrypto/cryptosoft.h> 74#include <sys/md5.h> 75#include <sys/random.h> 76 77#include <pci/pcivar.h> 78#include <pci/pcireg.h> 79 80/* grr, #defines for gratuitous incompatibility in queue.h */ 81#define SIMPLEQ_HEAD STAILQ_HEAD 82#define SIMPLEQ_ENTRY STAILQ_ENTRY 83#define SIMPLEQ_INIT STAILQ_INIT 84#define SIMPLEQ_INSERT_TAIL STAILQ_INSERT_TAIL 85#define SIMPLEQ_EMPTY STAILQ_EMPTY 86#define SIMPLEQ_FIRST STAILQ_FIRST 87#define SIMPLEQ_REMOVE_HEAD STAILQ_REMOVE_HEAD_UNTIL 88/* ditto for endian.h */ 89#define letoh16(x) le16toh(x) 90#define letoh32(x) le32toh(x) 91 92#include <dev/ubsec/ubsecreg.h> 93#include <dev/ubsec/ubsecvar.h> 94 95/* 96 * Prototypes and count for the pci_device structure 97 */ 98static int ubsec_probe(device_t); 99static int ubsec_attach(device_t); 100static int ubsec_detach(device_t); 101static int ubsec_suspend(device_t); 102static int ubsec_resume(device_t); 103static void ubsec_shutdown(device_t); 104 105static device_method_t ubsec_methods[] = { 106 /* Device interface */ 107 DEVMETHOD(device_probe, ubsec_probe), 108 DEVMETHOD(device_attach, ubsec_attach), 109 DEVMETHOD(device_detach, ubsec_detach), 110 DEVMETHOD(device_suspend, ubsec_suspend), 111 DEVMETHOD(device_resume, ubsec_resume), 112 DEVMETHOD(device_shutdown, ubsec_shutdown), 113 114 /* bus interface */ 115 DEVMETHOD(bus_print_child, bus_generic_print_child), 116 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 117 118 { 0, 0 } 119}; 120static driver_t ubsec_driver = { 121 "ubsec", 122 ubsec_methods, 123 sizeof (struct ubsec_softc) 124}; 125static devclass_t ubsec_devclass; 126 127DRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0); 128MODULE_DEPEND(ubsec, crypto, 1, 1, 1); 129 130static void ubsec_intr(void *); 131static int ubsec_newsession(void *, u_int32_t *, struct cryptoini *); 132static int ubsec_freesession(void *, u_int64_t); 133static int ubsec_process(void *, struct cryptop *, int); 134static void ubsec_callback(struct ubsec_softc *, struct ubsec_q *); 135static int ubsec_feed(struct ubsec_softc *); 136static void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int); 137static void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *); 138static int ubsec_feed2(struct ubsec_softc *); 139static void ubsec_rng(void *); 140static int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t, 141 struct ubsec_dma_alloc *, int); 142#define ubsec_dma_sync(_dma, _flags) \ 143 bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags)) 144static void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *); 145static int ubsec_dmamap_aligned(struct ubsec_operand *op); 146 147static void ubsec_reset_board(struct ubsec_softc *sc); 148static void ubsec_init_board(struct ubsec_softc *sc); 149static void ubsec_init_pciregs(device_t dev); 150static void ubsec_totalreset(struct ubsec_softc *sc); 151 152static int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q); 153 154static int ubsec_kprocess(void*, struct cryptkop *, int); 155static int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int); 156static int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int); 157static int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int); 158static void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *); 159static int ubsec_ksigbits(struct crparam *); 160static void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 161static void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 162 163SYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0, "Broadcom driver parameters"); 164 165#ifdef UBSEC_DEBUG 166static void ubsec_dump_pb(volatile struct ubsec_pktbuf *); 167static void ubsec_dump_mcr(struct ubsec_mcr *); 168static void ubsec_dump_ctx2(struct ubsec_ctx_keyop *); 169 170static int ubsec_debug = 0; 171SYSCTL_INT(_hw_ubsec, OID_AUTO, debug, CTLFLAG_RW, &ubsec_debug, 172 0, "control debugging msgs"); 173#endif 174 175#define READ_REG(sc,r) \ 176 bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r)) 177 178#define WRITE_REG(sc,reg,val) \ 179 bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val) 180 181#define SWAP32(x) (x) = htole32(ntohl((x))) 182#define HTOLE32(x) (x) = htole32(x) 183 184struct ubsec_stats ubsecstats; 185SYSCTL_STRUCT(_hw_ubsec, OID_AUTO, stats, CTLFLAG_RD, &ubsecstats, 186 ubsec_stats, "driver statistics"); 187/* 188 * ubsec_maxbatch controls the number of crypto ops to voluntarily 189 * collect into one submission to the hardware. This batching happens 190 * when ops are dispatched from the crypto subsystem with a hint that 191 * more are to follow immediately. These ops must also not be marked 192 * with a ``no delay'' flag. 193 */ 194static int ubsec_maxbatch = 1; 195SYSCTL_INT(_hw_ubsec, OID_AUTO, maxbatch, CTLFLAG_RW, &ubsec_maxbatch, 196 0, "max ops to batch w/o interrupt"); 197/* 198 * ubsec_maxaggr controls the number of crypto ops to submit to the 199 * hardware as a unit. This aggregation reduces the number of interrupts 200 * to the host at the expense of increased latency (for all but the last 201 * operation). For network traffic setting this to one yields the highest 202 * performance but at the expense of more interrupt processing. 203 */ 204static int ubsec_maxaggr = 1; 205SYSCTL_INT(_hw_ubsec, OID_AUTO, maxaggr, CTLFLAG_RW, &ubsec_maxaggr, 206 0, "max ops to aggregate under one interrupt"); 207 208static int 209ubsec_probe(device_t dev) 210{ 211 if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 212 (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 || 213 pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)) 214 return (0); 215 if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 216 (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 || 217 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 || 218 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 219 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 || 220 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 221 )) 222 return (0); 223 return (ENXIO); 224} 225 226static const char* 227ubsec_partname(struct ubsec_softc *sc) 228{ 229 /* XXX sprintf numbers when not decoded */ 230 switch (pci_get_vendor(sc->sc_dev)) { 231 case PCI_VENDOR_BROADCOM: 232 switch (pci_get_device(sc->sc_dev)) { 233 case PCI_PRODUCT_BROADCOM_5805: return "Broadcom 5805"; 234 case PCI_PRODUCT_BROADCOM_5820: return "Broadcom 5820"; 235 case PCI_PRODUCT_BROADCOM_5821: return "Broadcom 5821"; 236 case PCI_PRODUCT_BROADCOM_5822: return "Broadcom 5822"; 237 case PCI_PRODUCT_BROADCOM_5823: return "Broadcom 5823"; 238 } 239 return "Broadcom unknown-part"; 240 case PCI_VENDOR_BLUESTEEL: 241 switch (pci_get_device(sc->sc_dev)) { 242 case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601"; 243 } 244 return "Bluesteel unknown-part"; 245 } 246 return "Unknown-vendor unknown-part"; 247} 248 249static int 250ubsec_attach(device_t dev) 251{ 252 struct ubsec_softc *sc = device_get_softc(dev); 253 struct ubsec_dma *dmap; 254 u_int32_t cmd, i; 255 int rid; 256 257 KASSERT(sc != NULL, ("ubsec_attach: null software carrier!")); 258 bzero(sc, sizeof (*sc)); 259 sc->sc_dev = dev; 260 261 mtx_init(&sc->sc_mtx, device_get_nameunit(dev), "crypto driver", MTX_DEF); 262 263 SIMPLEQ_INIT(&sc->sc_queue); 264 SIMPLEQ_INIT(&sc->sc_qchip); 265 SIMPLEQ_INIT(&sc->sc_queue2); 266 SIMPLEQ_INIT(&sc->sc_qchip2); 267 SIMPLEQ_INIT(&sc->sc_q2free); 268 269 /* XXX handle power management */ 270 271 sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR; 272 273 if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 274 pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601) 275 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 276 277 if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 278 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805) 279 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 280 281 if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 282 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820) 283 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 284 UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 285 286 if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 287 (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 288 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 || 289 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 )) { 290 /* NB: the 5821/5822 defines some additional status bits */ 291 sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY | 292 BS_STAT_MCR2_ALLEMPTY; 293 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 294 UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 295 } 296 297 cmd = pci_read_config(dev, PCIR_COMMAND, 4); 298 cmd |= PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN; 299 pci_write_config(dev, PCIR_COMMAND, cmd, 4); 300 cmd = pci_read_config(dev, PCIR_COMMAND, 4); 301 302 if (!(cmd & PCIM_CMD_MEMEN)) { 303 device_printf(dev, "failed to enable memory mapping\n"); 304 goto bad; 305 } 306 307 if (!(cmd & PCIM_CMD_BUSMASTEREN)) { 308 device_printf(dev, "failed to enable bus mastering\n"); 309 goto bad; 310 } 311 312 /* 313 * Setup memory-mapping of PCI registers. 314 */ 315 rid = BS_BAR; 316 sc->sc_sr = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid, 317 0, ~0, 1, RF_ACTIVE); 318 if (sc->sc_sr == NULL) { 319 device_printf(dev, "cannot map register space\n"); 320 goto bad; 321 } 322 sc->sc_st = rman_get_bustag(sc->sc_sr); 323 sc->sc_sh = rman_get_bushandle(sc->sc_sr); 324 325 /* 326 * Arrange interrupt line. 327 */ 328 rid = 0; 329 sc->sc_irq = bus_alloc_resource(dev, SYS_RES_IRQ, &rid, 330 0, ~0, 1, RF_SHAREABLE|RF_ACTIVE); 331 if (sc->sc_irq == NULL) { 332 device_printf(dev, "could not map interrupt\n"); 333 goto bad1; 334 } 335 /* 336 * NB: Network code assumes we are blocked with splimp() 337 * so make sure the IRQ is mapped appropriately. 338 */ 339 if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET, 340 ubsec_intr, sc, &sc->sc_ih)) { 341 device_printf(dev, "could not establish interrupt\n"); 342 goto bad2; 343 } 344 345 sc->sc_cid = crypto_get_driverid(0); 346 if (sc->sc_cid < 0) { 347 device_printf(dev, "could not get crypto driver id\n"); 348 goto bad3; 349 } 350 351 /* 352 * Setup DMA descriptor area. 353 */ 354 if (bus_dma_tag_create(NULL, /* parent */ 355 1, 0, /* alignment, bounds */ 356 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 357 BUS_SPACE_MAXADDR, /* highaddr */ 358 NULL, NULL, /* filter, filterarg */ 359 0x3ffff, /* maxsize */ 360 UBS_MAX_SCATTER, /* nsegments */ 361 0xffff, /* maxsegsize */ 362 BUS_DMA_ALLOCNOW, /* flags */ 363 &sc->sc_dmat)) { 364 device_printf(dev, "cannot allocate DMA tag\n"); 365 goto bad4; 366 } 367 SIMPLEQ_INIT(&sc->sc_freequeue); 368 dmap = sc->sc_dmaa; 369 for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) { 370 struct ubsec_q *q; 371 372 q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q), 373 M_DEVBUF, M_NOWAIT); 374 if (q == NULL) { 375 device_printf(dev, "cannot allocate queue buffers\n"); 376 break; 377 } 378 379 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk), 380 &dmap->d_alloc, 0)) { 381 device_printf(dev, "cannot allocate dma buffers\n"); 382 free(q, M_DEVBUF); 383 break; 384 } 385 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr; 386 387 q->q_dma = dmap; 388 sc->sc_queuea[i] = q; 389 390 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 391 } 392 393 device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc)); 394 395 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0, 396 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 397 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0, 398 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 399 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0, 400 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 401 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0, 402 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 403 404 /* 405 * Reset Broadcom chip 406 */ 407 ubsec_reset_board(sc); 408 409 /* 410 * Init Broadcom specific PCI settings 411 */ 412 ubsec_init_pciregs(dev); 413 414 /* 415 * Init Broadcom chip 416 */ 417 ubsec_init_board(sc); 418 419#ifndef UBSEC_NO_RNG 420 if (sc->sc_flags & UBS_FLAGS_RNG) { 421 sc->sc_statmask |= BS_STAT_MCR2_DONE; 422 423 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 424 &sc->sc_rng.rng_q.q_mcr, 0)) 425 goto skip_rng; 426 427 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass), 428 &sc->sc_rng.rng_q.q_ctx, 0)) { 429 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 430 goto skip_rng; 431 } 432 433 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) * 434 UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) { 435 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 436 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 437 goto skip_rng; 438 } 439 440 if (hz >= 100) 441 sc->sc_rnghz = hz / 100; 442 else 443 sc->sc_rnghz = 1; 444 /* NB: 1 means the callout runs w/o Giant locked */ 445 callout_init(&sc->sc_rngto, 1); 446 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 447skip_rng: 448 ; 449 } 450#endif /* UBSEC_NO_RNG */ 451 452 if (sc->sc_flags & UBS_FLAGS_KEY) { 453 sc->sc_statmask |= BS_STAT_MCR2_DONE; 454 455 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0, 456 ubsec_kprocess, sc); 457#if 0 458 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0, 459 ubsec_kprocess, sc); 460#endif 461 } 462 return (0); 463bad4: 464 crypto_unregister_all(sc->sc_cid); 465bad3: 466 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 467bad2: 468 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 469bad1: 470 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 471bad: 472 mtx_destroy(&sc->sc_mtx); 473 return (ENXIO); 474} 475 476/* 477 * Detach a device that successfully probed. 478 */ 479static int 480ubsec_detach(device_t dev) 481{ 482 struct ubsec_softc *sc = device_get_softc(dev); 483 484 KASSERT(sc != NULL, ("ubsec_detach: null software carrier")); 485 486 /* XXX wait/abort active ops */ 487 488 UBSEC_LOCK(sc); 489 490 callout_stop(&sc->sc_rngto); 491 492 crypto_unregister_all(sc->sc_cid); 493 494 while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 495 struct ubsec_q *q; 496 497 q = SIMPLEQ_FIRST(&sc->sc_freequeue); 498 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q, q_next); 499 ubsec_dma_free(sc, &q->q_dma->d_alloc); 500 free(q, M_DEVBUF); 501 } 502#ifndef UBSEC_NO_RNG 503 if (sc->sc_flags & UBS_FLAGS_RNG) { 504 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 505 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 506 ubsec_dma_free(sc, &sc->sc_rng.rng_buf); 507 } 508#endif /* UBSEC_NO_RNG */ 509 510 bus_generic_detach(dev); 511 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 512 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 513 514 bus_dma_tag_destroy(sc->sc_dmat); 515 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 516 517 UBSEC_UNLOCK(sc); 518 519 mtx_destroy(&sc->sc_mtx); 520 521 return (0); 522} 523 524/* 525 * Stop all chip i/o so that the kernel's probe routines don't 526 * get confused by errant DMAs when rebooting. 527 */ 528static void 529ubsec_shutdown(device_t dev) 530{ 531#ifdef notyet 532 ubsec_stop(device_get_softc(dev)); 533#endif 534} 535 536/* 537 * Device suspend routine. 538 */ 539static int 540ubsec_suspend(device_t dev) 541{ 542 struct ubsec_softc *sc = device_get_softc(dev); 543 544 KASSERT(sc != NULL, ("ubsec_suspend: null software carrier")); 545#ifdef notyet 546 /* XXX stop the device and save PCI settings */ 547#endif 548 sc->sc_suspended = 1; 549 550 return (0); 551} 552 553static int 554ubsec_resume(device_t dev) 555{ 556 struct ubsec_softc *sc = device_get_softc(dev); 557 558 KASSERT(sc != NULL, ("ubsec_resume: null software carrier")); 559#ifdef notyet 560 /* XXX retore PCI settings and start the device */ 561#endif 562 sc->sc_suspended = 0; 563 return (0); 564} 565 566/* 567 * UBSEC Interrupt routine 568 */ 569static void 570ubsec_intr(void *arg) 571{ 572 struct ubsec_softc *sc = arg; 573 volatile u_int32_t stat; 574 struct ubsec_q *q; 575 struct ubsec_dma *dmap; 576 int npkts = 0, i; 577 578 UBSEC_LOCK(sc); 579 580 stat = READ_REG(sc, BS_STAT); 581 stat &= sc->sc_statmask; 582 if (stat == 0) { 583 UBSEC_UNLOCK(sc); 584 return; 585 } 586 587 WRITE_REG(sc, BS_STAT, stat); /* IACK */ 588 589 /* 590 * Check to see if we have any packets waiting for us 591 */ 592 if ((stat & BS_STAT_MCR1_DONE)) { 593 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 594 q = SIMPLEQ_FIRST(&sc->sc_qchip); 595 dmap = q->q_dma; 596 597 if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0) 598 break; 599 600 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next); 601 602 npkts = q->q_nstacked_mcrs; 603 sc->sc_nqchip -= 1+npkts; 604 /* 605 * search for further sc_qchip ubsec_q's that share 606 * the same MCR, and complete them too, they must be 607 * at the top. 608 */ 609 for (i = 0; i < npkts; i++) { 610 if(q->q_stacked_mcr[i]) { 611 ubsec_callback(sc, q->q_stacked_mcr[i]); 612 ubsecstats.hst_opackets++; 613 } else { 614 break; 615 } 616 } 617 ubsec_callback(sc, q); 618 ubsecstats.hst_opackets++; 619 } 620 621 /* 622 * Don't send any more packet to chip if there has been 623 * a DMAERR. 624 */ 625 if (!(stat & BS_STAT_DMAERR)) 626 ubsec_feed(sc); 627 } 628 629 /* 630 * Check to see if we have any key setups/rng's waiting for us 631 */ 632 if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) && 633 (stat & BS_STAT_MCR2_DONE)) { 634 struct ubsec_q2 *q2; 635 struct ubsec_mcr *mcr; 636 637 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) { 638 q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 639 640 ubsec_dma_sync(&q2->q_mcr, 641 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 642 643 mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 644 if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) { 645 ubsec_dma_sync(&q2->q_mcr, 646 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 647 break; 648 } 649 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q2, q_next); 650 ubsec_callback2(sc, q2); 651 /* 652 * Don't send any more packet to chip if there has been 653 * a DMAERR. 654 */ 655 if (!(stat & BS_STAT_DMAERR)) 656 ubsec_feed2(sc); 657 } 658 } 659 660 /* 661 * Check to see if we got any DMA Error 662 */ 663 if (stat & BS_STAT_DMAERR) { 664#ifdef UBSEC_DEBUG 665 if (ubsec_debug) { 666 volatile u_int32_t a = READ_REG(sc, BS_ERR); 667 668 printf("dmaerr %s@%08x\n", 669 (a & BS_ERR_READ) ? "read" : "write", 670 a & BS_ERR_ADDR); 671 } 672#endif /* UBSEC_DEBUG */ 673 ubsecstats.hst_dmaerr++; 674 ubsec_totalreset(sc); 675 ubsec_feed(sc); 676 } 677 678 if (sc->sc_needwakeup) { /* XXX check high watermark */ 679 int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ); 680#ifdef UBSEC_DEBUG 681 if (ubsec_debug) 682 device_printf(sc->sc_dev, "wakeup crypto (%x)\n", 683 sc->sc_needwakeup); 684#endif /* UBSEC_DEBUG */ 685 sc->sc_needwakeup &= ~wakeup; 686 crypto_unblock(sc->sc_cid, wakeup); 687 } 688 689 UBSEC_UNLOCK(sc); 690} 691 692/* 693 * ubsec_feed() - aggregate and post requests to chip 694 */ 695static int 696ubsec_feed(struct ubsec_softc *sc) 697{ 698 struct ubsec_q *q, *q2; 699 int npkts, i; 700 void *v; 701 u_int32_t stat; 702 703 npkts = sc->sc_nqueue; 704 if (npkts > ubsecstats.hst_maxqueue) 705 ubsecstats.hst_maxqueue = npkts; 706 /* 707 * Decide how many ops to combine in a single MCR. We cannot 708 * aggregate more than UBS_MAX_AGGR because this is the number 709 * of slots defined in the data structure. Otherwise we clamp 710 * based on the tunable parameter ubsec_maxaggr. Note that 711 * aggregation can happen in two ways: either by batching ops 712 * from above or because the h/w backs up and throttles us. 713 * Aggregating ops reduces the number of interrupts to the host 714 * but also (potentially) increases the latency for processing 715 * completed ops as we only get an interrupt when all aggregated 716 * ops have completed. 717 */ 718 if (npkts > UBS_MAX_AGGR) 719 npkts = UBS_MAX_AGGR; 720 if (npkts > ubsec_maxaggr) 721 npkts = ubsec_maxaggr; 722 if (npkts > ubsecstats.hst_maxbatch) 723 ubsecstats.hst_maxbatch = npkts; 724 if (npkts < 2) 725 goto feed1; 726 ubsecstats.hst_totbatch += npkts-1; 727 728 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 729 if (stat & BS_STAT_DMAERR) { 730 ubsec_totalreset(sc); 731 ubsecstats.hst_dmaerr++; 732 } else 733 ubsecstats.hst_mcr1full++; 734 return (0); 735 } 736 737#ifdef UBSEC_DEBUG 738 if (ubsec_debug) 739 printf("merging %d records\n", npkts); 740#endif /* UBSEC_DEBUG */ 741 742 q = SIMPLEQ_FIRST(&sc->sc_queue); 743 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next); 744 --sc->sc_nqueue; 745 746 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE); 747 if (q->q_dst_map != NULL) 748 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD); 749 750 q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */ 751 752 for (i = 0; i < q->q_nstacked_mcrs; i++) { 753 q2 = SIMPLEQ_FIRST(&sc->sc_queue); 754 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map, 755 BUS_DMASYNC_PREWRITE); 756 if (q2->q_dst_map != NULL) 757 bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map, 758 BUS_DMASYNC_PREREAD); 759 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q2, q_next); 760 --sc->sc_nqueue; 761 762 v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) - 763 sizeof(struct ubsec_mcr_add)); 764 bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add)); 765 q->q_stacked_mcr[i] = q2; 766 } 767 q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts); 768 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 769 sc->sc_nqchip += npkts; 770 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 771 ubsecstats.hst_maxqchip = sc->sc_nqchip; 772 ubsec_dma_sync(&q->q_dma->d_alloc, 773 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 774 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 775 offsetof(struct ubsec_dmachunk, d_mcr)); 776 return (0); 777 778feed1: 779 while (!SIMPLEQ_EMPTY(&sc->sc_queue)) { 780 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 781 if (stat & BS_STAT_DMAERR) { 782 ubsec_totalreset(sc); 783 ubsecstats.hst_dmaerr++; 784 } else 785 ubsecstats.hst_mcr1full++; 786 break; 787 } 788 789 q = SIMPLEQ_FIRST(&sc->sc_queue); 790 791 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 792 BUS_DMASYNC_PREWRITE); 793 if (q->q_dst_map != NULL) 794 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 795 BUS_DMASYNC_PREREAD); 796 ubsec_dma_sync(&q->q_dma->d_alloc, 797 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 798 799 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 800 offsetof(struct ubsec_dmachunk, d_mcr)); 801#ifdef UBSEC_DEBUG 802 if (ubsec_debug) 803 printf("feed: q->chip %p %08x stat %08x\n", 804 q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr), 805 stat); 806#endif /* UBSEC_DEBUG */ 807 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next); 808 --sc->sc_nqueue; 809 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 810 sc->sc_nqchip++; 811 } 812 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 813 ubsecstats.hst_maxqchip = sc->sc_nqchip; 814 return (0); 815} 816 817/* 818 * Allocate a new 'session' and return an encoded session id. 'sidp' 819 * contains our registration id, and should contain an encoded session 820 * id on successful allocation. 821 */ 822static int 823ubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri) 824{ 825 struct cryptoini *c, *encini = NULL, *macini = NULL; 826 struct ubsec_softc *sc = arg; 827 struct ubsec_session *ses = NULL; 828 MD5_CTX md5ctx; 829 SHA1_CTX sha1ctx; 830 int i, sesn; 831 832 KASSERT(sc != NULL, ("ubsec_newsession: null softc")); 833 if (sidp == NULL || cri == NULL || sc == NULL) 834 return (EINVAL); 835 836 for (c = cri; c != NULL; c = c->cri_next) { 837 if (c->cri_alg == CRYPTO_MD5_HMAC || 838 c->cri_alg == CRYPTO_SHA1_HMAC) { 839 if (macini) 840 return (EINVAL); 841 macini = c; 842 } else if (c->cri_alg == CRYPTO_DES_CBC || 843 c->cri_alg == CRYPTO_3DES_CBC) { 844 if (encini) 845 return (EINVAL); 846 encini = c; 847 } else 848 return (EINVAL); 849 } 850 if (encini == NULL && macini == NULL) 851 return (EINVAL); 852 853 if (sc->sc_sessions == NULL) { 854 ses = sc->sc_sessions = (struct ubsec_session *)malloc( 855 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 856 if (ses == NULL) 857 return (ENOMEM); 858 sesn = 0; 859 sc->sc_nsessions = 1; 860 } else { 861 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 862 if (sc->sc_sessions[sesn].ses_used == 0) { 863 ses = &sc->sc_sessions[sesn]; 864 break; 865 } 866 } 867 868 if (ses == NULL) { 869 sesn = sc->sc_nsessions; 870 ses = (struct ubsec_session *)malloc((sesn + 1) * 871 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 872 if (ses == NULL) 873 return (ENOMEM); 874 bcopy(sc->sc_sessions, ses, sesn * 875 sizeof(struct ubsec_session)); 876 bzero(sc->sc_sessions, sesn * 877 sizeof(struct ubsec_session)); 878 free(sc->sc_sessions, M_DEVBUF); 879 sc->sc_sessions = ses; 880 ses = &sc->sc_sessions[sesn]; 881 sc->sc_nsessions++; 882 } 883 } 884 885 bzero(ses, sizeof(struct ubsec_session)); 886 ses->ses_used = 1; 887 if (encini) { 888 /* get an IV, network byte order */ 889 /* XXX may read fewer than requested */ 890 read_random(ses->ses_iv, sizeof(ses->ses_iv)); 891 892 /* Go ahead and compute key in ubsec's byte order */ 893 if (encini->cri_alg == CRYPTO_DES_CBC) { 894 bcopy(encini->cri_key, &ses->ses_deskey[0], 8); 895 bcopy(encini->cri_key, &ses->ses_deskey[2], 8); 896 bcopy(encini->cri_key, &ses->ses_deskey[4], 8); 897 } else 898 bcopy(encini->cri_key, ses->ses_deskey, 24); 899 900 SWAP32(ses->ses_deskey[0]); 901 SWAP32(ses->ses_deskey[1]); 902 SWAP32(ses->ses_deskey[2]); 903 SWAP32(ses->ses_deskey[3]); 904 SWAP32(ses->ses_deskey[4]); 905 SWAP32(ses->ses_deskey[5]); 906 } 907 908 if (macini) { 909 for (i = 0; i < macini->cri_klen / 8; i++) 910 macini->cri_key[i] ^= HMAC_IPAD_VAL; 911 912 if (macini->cri_alg == CRYPTO_MD5_HMAC) { 913 MD5Init(&md5ctx); 914 MD5Update(&md5ctx, macini->cri_key, 915 macini->cri_klen / 8); 916 MD5Update(&md5ctx, hmac_ipad_buffer, 917 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 918 bcopy(md5ctx.state, ses->ses_hminner, 919 sizeof(md5ctx.state)); 920 } else { 921 SHA1Init(&sha1ctx); 922 SHA1Update(&sha1ctx, macini->cri_key, 923 macini->cri_klen / 8); 924 SHA1Update(&sha1ctx, hmac_ipad_buffer, 925 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 926 bcopy(sha1ctx.h.b32, ses->ses_hminner, 927 sizeof(sha1ctx.h.b32)); 928 } 929 930 for (i = 0; i < macini->cri_klen / 8; i++) 931 macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 932 933 if (macini->cri_alg == CRYPTO_MD5_HMAC) { 934 MD5Init(&md5ctx); 935 MD5Update(&md5ctx, macini->cri_key, 936 macini->cri_klen / 8); 937 MD5Update(&md5ctx, hmac_opad_buffer, 938 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 939 bcopy(md5ctx.state, ses->ses_hmouter, 940 sizeof(md5ctx.state)); 941 } else { 942 SHA1Init(&sha1ctx); 943 SHA1Update(&sha1ctx, macini->cri_key, 944 macini->cri_klen / 8); 945 SHA1Update(&sha1ctx, hmac_opad_buffer, 946 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 947 bcopy(sha1ctx.h.b32, ses->ses_hmouter, 948 sizeof(sha1ctx.h.b32)); 949 } 950 951 for (i = 0; i < macini->cri_klen / 8; i++) 952 macini->cri_key[i] ^= HMAC_OPAD_VAL; 953 } 954 955 *sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn); 956 return (0); 957} 958 959/* 960 * Deallocate a session. 961 */ 962static int 963ubsec_freesession(void *arg, u_int64_t tid) 964{ 965 struct ubsec_softc *sc = arg; 966 int session; 967 u_int32_t sid = ((u_int32_t) tid) & 0xffffffff; 968 969 KASSERT(sc != NULL, ("ubsec_freesession: null softc")); 970 if (sc == NULL) 971 return (EINVAL); 972 973 session = UBSEC_SESSION(sid); 974 if (session >= sc->sc_nsessions) 975 return (EINVAL); 976 977 bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session])); 978 return (0); 979} 980 981static void 982ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error) 983{ 984 struct ubsec_operand *op = arg; 985 986 KASSERT(nsegs <= UBS_MAX_SCATTER, 987 ("Too many DMA segments returned when mapping operand")); 988#ifdef UBSEC_DEBUG 989 if (ubsec_debug) 990 printf("ubsec_op_cb: mapsize %u nsegs %d\n", 991 (u_int) mapsize, nsegs); 992#endif 993 op->mapsize = mapsize; 994 op->nsegs = nsegs; 995 bcopy(seg, op->segs, nsegs * sizeof (seg[0])); 996} 997 998static int 999ubsec_process(void *arg, struct cryptop *crp, int hint) 1000{ 1001 struct ubsec_q *q = NULL; 1002 int err = 0, i, j, nicealign; 1003 struct ubsec_softc *sc = arg; 1004 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 1005 int encoffset = 0, macoffset = 0, cpskip, cpoffset; 1006 int sskip, dskip, stheend, dtheend; 1007 int16_t coffset; 1008 struct ubsec_session *ses; 1009 struct ubsec_pktctx ctx; 1010 struct ubsec_dma *dmap = NULL; 1011 1012 if (crp == NULL || crp->crp_callback == NULL || sc == NULL) { 1013 ubsecstats.hst_invalid++; 1014 return (EINVAL); 1015 } 1016 if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) { 1017 ubsecstats.hst_badsession++; 1018 return (EINVAL); 1019 } 1020 1021 UBSEC_LOCK(sc); 1022 1023 if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 1024 ubsecstats.hst_queuefull++; 1025 sc->sc_needwakeup |= CRYPTO_SYMQ; 1026 UBSEC_UNLOCK(sc); 1027 return (ERESTART); 1028 } 1029 q = SIMPLEQ_FIRST(&sc->sc_freequeue); 1030 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q, q_next); 1031 UBSEC_UNLOCK(sc); 1032 1033 dmap = q->q_dma; /* Save dma pointer */ 1034 bzero(q, sizeof(struct ubsec_q)); 1035 bzero(&ctx, sizeof(ctx)); 1036 1037 q->q_sesn = UBSEC_SESSION(crp->crp_sid); 1038 q->q_dma = dmap; 1039 ses = &sc->sc_sessions[q->q_sesn]; 1040 1041 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1042 q->q_src_m = (struct mbuf *)crp->crp_buf; 1043 q->q_dst_m = (struct mbuf *)crp->crp_buf; 1044 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1045 q->q_src_io = (struct uio *)crp->crp_buf; 1046 q->q_dst_io = (struct uio *)crp->crp_buf; 1047 } else { 1048 ubsecstats.hst_badflags++; 1049 err = EINVAL; 1050 goto errout; /* XXX we don't handle contiguous blocks! */ 1051 } 1052 1053 bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr)); 1054 1055 dmap->d_dma->d_mcr.mcr_pkts = htole16(1); 1056 dmap->d_dma->d_mcr.mcr_flags = 0; 1057 q->q_crp = crp; 1058 1059 crd1 = crp->crp_desc; 1060 if (crd1 == NULL) { 1061 ubsecstats.hst_nodesc++; 1062 err = EINVAL; 1063 goto errout; 1064 } 1065 crd2 = crd1->crd_next; 1066 1067 if (crd2 == NULL) { 1068 if (crd1->crd_alg == CRYPTO_MD5_HMAC || 1069 crd1->crd_alg == CRYPTO_SHA1_HMAC) { 1070 maccrd = crd1; 1071 enccrd = NULL; 1072 } else if (crd1->crd_alg == CRYPTO_DES_CBC || 1073 crd1->crd_alg == CRYPTO_3DES_CBC) { 1074 maccrd = NULL; 1075 enccrd = crd1; 1076 } else { 1077 ubsecstats.hst_badalg++; 1078 err = EINVAL; 1079 goto errout; 1080 } 1081 } else { 1082 if ((crd1->crd_alg == CRYPTO_MD5_HMAC || 1083 crd1->crd_alg == CRYPTO_SHA1_HMAC) && 1084 (crd2->crd_alg == CRYPTO_DES_CBC || 1085 crd2->crd_alg == CRYPTO_3DES_CBC) && 1086 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 1087 maccrd = crd1; 1088 enccrd = crd2; 1089 } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 1090 crd1->crd_alg == CRYPTO_3DES_CBC) && 1091 (crd2->crd_alg == CRYPTO_MD5_HMAC || 1092 crd2->crd_alg == CRYPTO_SHA1_HMAC) && 1093 (crd1->crd_flags & CRD_F_ENCRYPT)) { 1094 enccrd = crd1; 1095 maccrd = crd2; 1096 } else { 1097 /* 1098 * We cannot order the ubsec as requested 1099 */ 1100 ubsecstats.hst_badalg++; 1101 err = EINVAL; 1102 goto errout; 1103 } 1104 } 1105 1106 if (enccrd) { 1107 encoffset = enccrd->crd_skip; 1108 ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES); 1109 1110 if (enccrd->crd_flags & CRD_F_ENCRYPT) { 1111 q->q_flags |= UBSEC_QFLAGS_COPYOUTIV; 1112 1113 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1114 bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1115 else { 1116 ctx.pc_iv[0] = ses->ses_iv[0]; 1117 ctx.pc_iv[1] = ses->ses_iv[1]; 1118 } 1119 1120 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { 1121 if (crp->crp_flags & CRYPTO_F_IMBUF) 1122 m_copyback(q->q_src_m, 1123 enccrd->crd_inject, 1124 8, (caddr_t)ctx.pc_iv); 1125 else if (crp->crp_flags & CRYPTO_F_IOV) 1126 cuio_copyback(q->q_src_io, 1127 enccrd->crd_inject, 1128 8, (caddr_t)ctx.pc_iv); 1129 } 1130 } else { 1131 ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND); 1132 1133 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1134 bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1135 else if (crp->crp_flags & CRYPTO_F_IMBUF) 1136 m_copydata(q->q_src_m, enccrd->crd_inject, 1137 8, (caddr_t)ctx.pc_iv); 1138 else if (crp->crp_flags & CRYPTO_F_IOV) 1139 cuio_copydata(q->q_src_io, 1140 enccrd->crd_inject, 8, 1141 (caddr_t)ctx.pc_iv); 1142 } 1143 1144 ctx.pc_deskey[0] = ses->ses_deskey[0]; 1145 ctx.pc_deskey[1] = ses->ses_deskey[1]; 1146 ctx.pc_deskey[2] = ses->ses_deskey[2]; 1147 ctx.pc_deskey[3] = ses->ses_deskey[3]; 1148 ctx.pc_deskey[4] = ses->ses_deskey[4]; 1149 ctx.pc_deskey[5] = ses->ses_deskey[5]; 1150 SWAP32(ctx.pc_iv[0]); 1151 SWAP32(ctx.pc_iv[1]); 1152 } 1153 1154 if (maccrd) { 1155 macoffset = maccrd->crd_skip; 1156 1157 if (maccrd->crd_alg == CRYPTO_MD5_HMAC) 1158 ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5); 1159 else 1160 ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1); 1161 1162 for (i = 0; i < 5; i++) { 1163 ctx.pc_hminner[i] = ses->ses_hminner[i]; 1164 ctx.pc_hmouter[i] = ses->ses_hmouter[i]; 1165 1166 HTOLE32(ctx.pc_hminner[i]); 1167 HTOLE32(ctx.pc_hmouter[i]); 1168 } 1169 } 1170 1171 if (enccrd && maccrd) { 1172 /* 1173 * ubsec cannot handle packets where the end of encryption 1174 * and authentication are not the same, or where the 1175 * encrypted part begins before the authenticated part. 1176 */ 1177 if ((encoffset + enccrd->crd_len) != 1178 (macoffset + maccrd->crd_len)) { 1179 ubsecstats.hst_lenmismatch++; 1180 err = EINVAL; 1181 goto errout; 1182 } 1183 if (enccrd->crd_skip < maccrd->crd_skip) { 1184 ubsecstats.hst_skipmismatch++; 1185 err = EINVAL; 1186 goto errout; 1187 } 1188 sskip = maccrd->crd_skip; 1189 cpskip = dskip = enccrd->crd_skip; 1190 stheend = maccrd->crd_len; 1191 dtheend = enccrd->crd_len; 1192 coffset = enccrd->crd_skip - maccrd->crd_skip; 1193 cpoffset = cpskip + dtheend; 1194#ifdef UBSEC_DEBUG 1195 if (ubsec_debug) { 1196 printf("mac: skip %d, len %d, inject %d\n", 1197 maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject); 1198 printf("enc: skip %d, len %d, inject %d\n", 1199 enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject); 1200 printf("src: skip %d, len %d\n", sskip, stheend); 1201 printf("dst: skip %d, len %d\n", dskip, dtheend); 1202 printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n", 1203 coffset, stheend, cpskip, cpoffset); 1204 } 1205#endif 1206 } else { 1207 cpskip = dskip = sskip = macoffset + encoffset; 1208 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len; 1209 cpoffset = cpskip + dtheend; 1210 coffset = 0; 1211 } 1212 ctx.pc_offset = htole16(coffset >> 2); 1213 1214 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) { 1215 ubsecstats.hst_nomap++; 1216 err = ENOMEM; 1217 goto errout; 1218 } 1219 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1220 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map, 1221 q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1222 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1223 q->q_src_map = NULL; 1224 ubsecstats.hst_noload++; 1225 err = ENOMEM; 1226 goto errout; 1227 } 1228 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1229 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map, 1230 q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1231 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1232 q->q_src_map = NULL; 1233 ubsecstats.hst_noload++; 1234 err = ENOMEM; 1235 goto errout; 1236 } 1237 } 1238 nicealign = ubsec_dmamap_aligned(&q->q_src); 1239 1240 dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend); 1241 1242#ifdef UBSEC_DEBUG 1243 if (ubsec_debug) 1244 printf("src skip: %d nicealign: %u\n", sskip, nicealign); 1245#endif 1246 for (i = j = 0; i < q->q_src_nsegs; i++) { 1247 struct ubsec_pktbuf *pb; 1248 bus_size_t packl = q->q_src_segs[i].ds_len; 1249 bus_addr_t packp = q->q_src_segs[i].ds_addr; 1250 1251 if (sskip >= packl) { 1252 sskip -= packl; 1253 continue; 1254 } 1255 1256 packl -= sskip; 1257 packp += sskip; 1258 sskip = 0; 1259 1260 if (packl > 0xfffc) { 1261 err = EIO; 1262 goto errout; 1263 } 1264 1265 if (j == 0) 1266 pb = &dmap->d_dma->d_mcr.mcr_ipktbuf; 1267 else 1268 pb = &dmap->d_dma->d_sbuf[j - 1]; 1269 1270 pb->pb_addr = htole32(packp); 1271 1272 if (stheend) { 1273 if (packl > stheend) { 1274 pb->pb_len = htole32(stheend); 1275 stheend = 0; 1276 } else { 1277 pb->pb_len = htole32(packl); 1278 stheend -= packl; 1279 } 1280 } else 1281 pb->pb_len = htole32(packl); 1282 1283 if ((i + 1) == q->q_src_nsegs) 1284 pb->pb_next = 0; 1285 else 1286 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1287 offsetof(struct ubsec_dmachunk, d_sbuf[j])); 1288 j++; 1289 } 1290 1291 if (enccrd == NULL && maccrd != NULL) { 1292 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0; 1293 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0; 1294 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr + 1295 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1296#ifdef UBSEC_DEBUG 1297 if (ubsec_debug) 1298 printf("opkt: %x %x %x\n", 1299 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr, 1300 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len, 1301 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next); 1302#endif 1303 } else { 1304 if (crp->crp_flags & CRYPTO_F_IOV) { 1305 if (!nicealign) { 1306 ubsecstats.hst_iovmisaligned++; 1307 err = EINVAL; 1308 goto errout; 1309 } 1310 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, 1311 &q->q_dst_map)) { 1312 ubsecstats.hst_nomap++; 1313 err = ENOMEM; 1314 goto errout; 1315 } 1316 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map, 1317 q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) { 1318 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1319 q->q_dst_map = NULL; 1320 ubsecstats.hst_noload++; 1321 err = ENOMEM; 1322 goto errout; 1323 } 1324 } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1325 if (nicealign) { 1326 q->q_dst = q->q_src; 1327 } else { 1328 int totlen, len; 1329 struct mbuf *m, *top, **mp; 1330 1331 ubsecstats.hst_unaligned++; 1332 totlen = q->q_src_mapsize; 1333 if (q->q_src_m->m_flags & M_PKTHDR) { 1334 len = MHLEN; 1335 MGETHDR(m, M_NOWAIT, MT_DATA); 1336 if (m && !m_dup_pkthdr(m, q->q_src_m, M_NOWAIT)) { 1337 m_free(m); 1338 m = NULL; 1339 } 1340 } else { 1341 len = MLEN; 1342 MGET(m, M_NOWAIT, MT_DATA); 1343 } 1344 if (m == NULL) { 1345 ubsecstats.hst_nombuf++; 1346 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1347 goto errout; 1348 } 1349 if (totlen >= MINCLSIZE) { 1350 MCLGET(m, M_NOWAIT); 1351 if ((m->m_flags & M_EXT) == 0) { 1352 m_free(m); 1353 ubsecstats.hst_nomcl++; 1354 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1355 goto errout; 1356 } 1357 len = MCLBYTES; 1358 } 1359 m->m_len = len; 1360 top = NULL; 1361 mp = ⊤ 1362 1363 while (totlen > 0) { 1364 if (top) { 1365 MGET(m, M_NOWAIT, MT_DATA); 1366 if (m == NULL) { 1367 m_freem(top); 1368 ubsecstats.hst_nombuf++; 1369 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1370 goto errout; 1371 } 1372 len = MLEN; 1373 } 1374 if (top && totlen >= MINCLSIZE) { 1375 MCLGET(m, M_NOWAIT); 1376 if ((m->m_flags & M_EXT) == 0) { 1377 *mp = m; 1378 m_freem(top); 1379 ubsecstats.hst_nomcl++; 1380 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1381 goto errout; 1382 } 1383 len = MCLBYTES; 1384 } 1385 m->m_len = len = min(totlen, len); 1386 totlen -= len; 1387 *mp = m; 1388 mp = &m->m_next; 1389 } 1390 q->q_dst_m = top; 1391 ubsec_mcopy(q->q_src_m, q->q_dst_m, 1392 cpskip, cpoffset); 1393 if (bus_dmamap_create(sc->sc_dmat, 1394 BUS_DMA_NOWAIT, &q->q_dst_map) != 0) { 1395 ubsecstats.hst_nomap++; 1396 err = ENOMEM; 1397 goto errout; 1398 } 1399 if (bus_dmamap_load_mbuf(sc->sc_dmat, 1400 q->q_dst_map, q->q_dst_m, 1401 ubsec_op_cb, &q->q_dst, 1402 BUS_DMA_NOWAIT) != 0) { 1403 bus_dmamap_destroy(sc->sc_dmat, 1404 q->q_dst_map); 1405 q->q_dst_map = NULL; 1406 ubsecstats.hst_noload++; 1407 err = ENOMEM; 1408 goto errout; 1409 } 1410 } 1411 } else { 1412 ubsecstats.hst_badflags++; 1413 err = EINVAL; 1414 goto errout; 1415 } 1416 1417#ifdef UBSEC_DEBUG 1418 if (ubsec_debug) 1419 printf("dst skip: %d\n", dskip); 1420#endif 1421 for (i = j = 0; i < q->q_dst_nsegs; i++) { 1422 struct ubsec_pktbuf *pb; 1423 bus_size_t packl = q->q_dst_segs[i].ds_len; 1424 bus_addr_t packp = q->q_dst_segs[i].ds_addr; 1425 1426 if (dskip >= packl) { 1427 dskip -= packl; 1428 continue; 1429 } 1430 1431 packl -= dskip; 1432 packp += dskip; 1433 dskip = 0; 1434 1435 if (packl > 0xfffc) { 1436 err = EIO; 1437 goto errout; 1438 } 1439 1440 if (j == 0) 1441 pb = &dmap->d_dma->d_mcr.mcr_opktbuf; 1442 else 1443 pb = &dmap->d_dma->d_dbuf[j - 1]; 1444 1445 pb->pb_addr = htole32(packp); 1446 1447 if (dtheend) { 1448 if (packl > dtheend) { 1449 pb->pb_len = htole32(dtheend); 1450 dtheend = 0; 1451 } else { 1452 pb->pb_len = htole32(packl); 1453 dtheend -= packl; 1454 } 1455 } else 1456 pb->pb_len = htole32(packl); 1457 1458 if ((i + 1) == q->q_dst_nsegs) { 1459 if (maccrd) 1460 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1461 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1462 else 1463 pb->pb_next = 0; 1464 } else 1465 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1466 offsetof(struct ubsec_dmachunk, d_dbuf[j])); 1467 j++; 1468 } 1469 } 1470 1471 dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr + 1472 offsetof(struct ubsec_dmachunk, d_ctx)); 1473 1474 if (sc->sc_flags & UBS_FLAGS_LONGCTX) { 1475 struct ubsec_pktctx_long *ctxl; 1476 1477 ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr + 1478 offsetof(struct ubsec_dmachunk, d_ctx)); 1479 1480 /* transform small context into long context */ 1481 ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long)); 1482 ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC); 1483 ctxl->pc_flags = ctx.pc_flags; 1484 ctxl->pc_offset = ctx.pc_offset; 1485 for (i = 0; i < 6; i++) 1486 ctxl->pc_deskey[i] = ctx.pc_deskey[i]; 1487 for (i = 0; i < 5; i++) 1488 ctxl->pc_hminner[i] = ctx.pc_hminner[i]; 1489 for (i = 0; i < 5; i++) 1490 ctxl->pc_hmouter[i] = ctx.pc_hmouter[i]; 1491 ctxl->pc_iv[0] = ctx.pc_iv[0]; 1492 ctxl->pc_iv[1] = ctx.pc_iv[1]; 1493 } else 1494 bcopy(&ctx, dmap->d_alloc.dma_vaddr + 1495 offsetof(struct ubsec_dmachunk, d_ctx), 1496 sizeof(struct ubsec_pktctx)); 1497 1498 UBSEC_LOCK(sc); 1499 SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next); 1500 sc->sc_nqueue++; 1501 ubsecstats.hst_ipackets++; 1502 ubsecstats.hst_ibytes += dmap->d_alloc.dma_size; 1503 if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= ubsec_maxbatch) 1504 ubsec_feed(sc); 1505 UBSEC_UNLOCK(sc); 1506 return (0); 1507 1508errout: 1509 if (q != NULL) { 1510 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 1511 m_freem(q->q_dst_m); 1512 1513 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1514 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1515 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1516 } 1517 if (q->q_src_map != NULL) { 1518 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1519 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1520 } 1521 1522 UBSEC_LOCK(sc); 1523 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1524 UBSEC_UNLOCK(sc); 1525 } 1526 if (err != ERESTART) { 1527 crp->crp_etype = err; 1528 crypto_done(crp); 1529 } else { 1530 sc->sc_needwakeup |= CRYPTO_SYMQ; 1531 } 1532 return (err); 1533} 1534 1535static void 1536ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q) 1537{ 1538 struct cryptop *crp = (struct cryptop *)q->q_crp; 1539 struct cryptodesc *crd; 1540 struct ubsec_dma *dmap = q->q_dma; 1541 1542 ubsec_dma_sync(&dmap->d_alloc, 1543 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1544 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1545 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 1546 BUS_DMASYNC_POSTREAD); 1547 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1548 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1549 } 1550 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE); 1551 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1552 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1553 1554 if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) { 1555 m_freem(q->q_src_m); 1556 crp->crp_buf = (caddr_t)q->q_dst_m; 1557 } 1558 ubsecstats.hst_obytes += ((struct mbuf *)crp->crp_buf)->m_len; 1559 1560 /* copy out IV for future use */ 1561 if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) { 1562 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1563 if (crd->crd_alg != CRYPTO_DES_CBC && 1564 crd->crd_alg != CRYPTO_3DES_CBC) 1565 continue; 1566 if (crp->crp_flags & CRYPTO_F_IMBUF) 1567 m_copydata((struct mbuf *)crp->crp_buf, 1568 crd->crd_skip + crd->crd_len - 8, 8, 1569 (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv); 1570 else if (crp->crp_flags & CRYPTO_F_IOV) { 1571 cuio_copydata((struct uio *)crp->crp_buf, 1572 crd->crd_skip + crd->crd_len - 8, 8, 1573 (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv); 1574 } 1575 break; 1576 } 1577 } 1578 1579 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1580 if (crd->crd_alg != CRYPTO_MD5_HMAC && 1581 crd->crd_alg != CRYPTO_SHA1_HMAC) 1582 continue; 1583 if (crp->crp_flags & CRYPTO_F_IMBUF) 1584 m_copyback((struct mbuf *)crp->crp_buf, 1585 crd->crd_inject, 12, 1586 (caddr_t)dmap->d_dma->d_macbuf); 1587 else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac) 1588 bcopy((caddr_t)dmap->d_dma->d_macbuf, 1589 crp->crp_mac, 12); 1590 break; 1591 } 1592 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1593 crypto_done(crp); 1594} 1595 1596static void 1597ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset) 1598{ 1599 int i, j, dlen, slen; 1600 caddr_t dptr, sptr; 1601 1602 j = 0; 1603 sptr = srcm->m_data; 1604 slen = srcm->m_len; 1605 dptr = dstm->m_data; 1606 dlen = dstm->m_len; 1607 1608 while (1) { 1609 for (i = 0; i < min(slen, dlen); i++) { 1610 if (j < hoffset || j >= toffset) 1611 *dptr++ = *sptr++; 1612 slen--; 1613 dlen--; 1614 j++; 1615 } 1616 if (slen == 0) { 1617 srcm = srcm->m_next; 1618 if (srcm == NULL) 1619 return; 1620 sptr = srcm->m_data; 1621 slen = srcm->m_len; 1622 } 1623 if (dlen == 0) { 1624 dstm = dstm->m_next; 1625 if (dstm == NULL) 1626 return; 1627 dptr = dstm->m_data; 1628 dlen = dstm->m_len; 1629 } 1630 } 1631} 1632 1633/* 1634 * feed the key generator, must be called at splimp() or higher. 1635 */ 1636static int 1637ubsec_feed2(struct ubsec_softc *sc) 1638{ 1639 struct ubsec_q2 *q; 1640 1641 while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) { 1642 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL) 1643 break; 1644 q = SIMPLEQ_FIRST(&sc->sc_queue2); 1645 1646 ubsec_dma_sync(&q->q_mcr, 1647 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1648 ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_PREWRITE); 1649 1650 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr); 1651 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q, q_next); 1652 --sc->sc_nqueue2; 1653 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next); 1654 } 1655 return (0); 1656} 1657 1658/* 1659 * Callback for handling random numbers 1660 */ 1661static void 1662ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q) 1663{ 1664 struct cryptkop *krp; 1665 struct ubsec_ctx_keyop *ctx; 1666 1667 ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr; 1668 ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_POSTWRITE); 1669 1670 switch (q->q_type) { 1671#ifndef UBSEC_NO_RNG 1672 case UBS_CTXOP_RNGBYPASS: { 1673 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q; 1674 1675 ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_POSTREAD); 1676 random_harvest(rng->rng_buf.dma_vaddr, 1677 UBSEC_RNG_BUFSIZ*sizeof (u_int32_t), 1678 UBSEC_RNG_BUFSIZ*sizeof (u_int32_t)*NBBY, 0, 1679 RANDOM_PURE); 1680 rng->rng_used = 0; 1681 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1682 break; 1683 } 1684#endif 1685 case UBS_CTXOP_MODEXP: { 1686 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 1687 u_int rlen, clen; 1688 1689 krp = me->me_krp; 1690 rlen = (me->me_modbits + 7) / 8; 1691 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8; 1692 1693 ubsec_dma_sync(&me->me_M, BUS_DMASYNC_POSTWRITE); 1694 ubsec_dma_sync(&me->me_E, BUS_DMASYNC_POSTWRITE); 1695 ubsec_dma_sync(&me->me_C, BUS_DMASYNC_POSTREAD); 1696 ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_POSTWRITE); 1697 1698 if (clen < rlen) 1699 krp->krp_status = E2BIG; 1700 else { 1701 if (sc->sc_flags & UBS_FLAGS_HWNORM) { 1702 bzero(krp->krp_param[krp->krp_iparams].crp_p, 1703 (krp->krp_param[krp->krp_iparams].crp_nbits 1704 + 7) / 8); 1705 bcopy(me->me_C.dma_vaddr, 1706 krp->krp_param[krp->krp_iparams].crp_p, 1707 (me->me_modbits + 7) / 8); 1708 } else 1709 ubsec_kshift_l(me->me_shiftbits, 1710 me->me_C.dma_vaddr, me->me_normbits, 1711 krp->krp_param[krp->krp_iparams].crp_p, 1712 krp->krp_param[krp->krp_iparams].crp_nbits); 1713 } 1714 1715 crypto_kdone(krp); 1716 1717 /* bzero all potentially sensitive data */ 1718 bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 1719 bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 1720 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 1721 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 1722 1723 /* Can't free here, so put us on the free list. */ 1724 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next); 1725 break; 1726 } 1727 case UBS_CTXOP_RSAPRIV: { 1728 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 1729 u_int len; 1730 1731 krp = rp->rpr_krp; 1732 ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_POSTWRITE); 1733 ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_POSTREAD); 1734 1735 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8; 1736 bcopy(rp->rpr_msgout.dma_vaddr, 1737 krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len); 1738 1739 crypto_kdone(krp); 1740 1741 bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 1742 bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 1743 bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size); 1744 1745 /* Can't free here, so put us on the free list. */ 1746 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next); 1747 break; 1748 } 1749 default: 1750 device_printf(sc->sc_dev, "unknown ctx op: %x\n", 1751 letoh16(ctx->ctx_op)); 1752 break; 1753 } 1754} 1755 1756#ifndef UBSEC_NO_RNG 1757static void 1758ubsec_rng(void *vsc) 1759{ 1760 struct ubsec_softc *sc = vsc; 1761 struct ubsec_q2_rng *rng = &sc->sc_rng; 1762 struct ubsec_mcr *mcr; 1763 struct ubsec_ctx_rngbypass *ctx; 1764 1765 UBSEC_LOCK(sc); 1766 if (rng->rng_used) { 1767 UBSEC_UNLOCK(sc); 1768 return; 1769 } 1770 sc->sc_nqueue2++; 1771 if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE) 1772 goto out; 1773 1774 mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr; 1775 ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr; 1776 1777 mcr->mcr_pkts = htole16(1); 1778 mcr->mcr_flags = 0; 1779 mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr); 1780 mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0; 1781 mcr->mcr_ipktbuf.pb_len = 0; 1782 mcr->mcr_reserved = mcr->mcr_pktlen = 0; 1783 mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr); 1784 mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) & 1785 UBS_PKTBUF_LEN); 1786 mcr->mcr_opktbuf.pb_next = 0; 1787 1788 ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass)); 1789 ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS); 1790 rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS; 1791 1792 ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_PREREAD); 1793 1794 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next); 1795 rng->rng_used = 1; 1796 ubsec_feed2(sc); 1797 ubsecstats.hst_rng++; 1798 UBSEC_UNLOCK(sc); 1799 1800 return; 1801 1802out: 1803 /* 1804 * Something weird happened, generate our own call back. 1805 */ 1806 sc->sc_nqueue2--; 1807 UBSEC_UNLOCK(sc); 1808 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1809} 1810#endif /* UBSEC_NO_RNG */ 1811 1812static void 1813ubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 1814{ 1815 bus_addr_t *paddr = (bus_addr_t*) arg; 1816 *paddr = segs->ds_addr; 1817} 1818 1819static int 1820ubsec_dma_malloc( 1821 struct ubsec_softc *sc, 1822 bus_size_t size, 1823 struct ubsec_dma_alloc *dma, 1824 int mapflags 1825) 1826{ 1827 int r; 1828 1829 /* XXX could specify sc_dmat as parent but that just adds overhead */ 1830 r = bus_dma_tag_create(NULL, /* parent */ 1831 1, 0, /* alignment, bounds */ 1832 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 1833 BUS_SPACE_MAXADDR, /* highaddr */ 1834 NULL, NULL, /* filter, filterarg */ 1835 size, /* maxsize */ 1836 1, /* nsegments */ 1837 size, /* maxsegsize */ 1838 BUS_DMA_ALLOCNOW, /* flags */ 1839 &dma->dma_tag); 1840 if (r != 0) { 1841 device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1842 "bus_dma_tag_create failed; error %u\n", r); 1843 goto fail_0; 1844 } 1845 1846 r = bus_dmamap_create(dma->dma_tag, BUS_DMA_NOWAIT, &dma->dma_map); 1847 if (r != 0) { 1848 device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1849 "bus_dmamap_create failed; error %u\n", r); 1850 goto fail_1; 1851 } 1852 1853 r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 1854 BUS_DMA_NOWAIT, &dma->dma_map); 1855 if (r != 0) { 1856 device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1857 "bus_dmammem_alloc failed; size %zu, error %u\n", 1858 size, r); 1859 goto fail_2; 1860 } 1861 1862 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 1863 size, 1864 ubsec_dmamap_cb, 1865 &dma->dma_paddr, 1866 mapflags | BUS_DMA_NOWAIT); 1867 if (r != 0) { 1868 device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1869 "bus_dmamap_load failed; error %u\n", r); 1870 goto fail_3; 1871 } 1872 1873 dma->dma_size = size; 1874 return (0); 1875 1876fail_3: 1877 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1878fail_2: 1879 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1880fail_1: 1881 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1882 bus_dma_tag_destroy(dma->dma_tag); 1883fail_0: 1884 dma->dma_map = NULL; 1885 dma->dma_tag = NULL; 1886 return (r); 1887} 1888 1889static void 1890ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma) 1891{ 1892 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1893 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1894 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1895 bus_dma_tag_destroy(dma->dma_tag); 1896} 1897 1898/* 1899 * Resets the board. Values in the regesters are left as is 1900 * from the reset (i.e. initial values are assigned elsewhere). 1901 */ 1902static void 1903ubsec_reset_board(struct ubsec_softc *sc) 1904{ 1905 volatile u_int32_t ctrl; 1906 1907 ctrl = READ_REG(sc, BS_CTRL); 1908 ctrl |= BS_CTRL_RESET; 1909 WRITE_REG(sc, BS_CTRL, ctrl); 1910 1911 /* 1912 * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us 1913 */ 1914 DELAY(10); 1915} 1916 1917/* 1918 * Init Broadcom registers 1919 */ 1920static void 1921ubsec_init_board(struct ubsec_softc *sc) 1922{ 1923 u_int32_t ctrl; 1924 1925 ctrl = READ_REG(sc, BS_CTRL); 1926 ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64); 1927 ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT; 1928 1929 if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) 1930 ctrl |= BS_CTRL_MCR2INT; 1931 else 1932 ctrl &= ~BS_CTRL_MCR2INT; 1933 1934 if (sc->sc_flags & UBS_FLAGS_HWNORM) 1935 ctrl &= ~BS_CTRL_SWNORM; 1936 1937 WRITE_REG(sc, BS_CTRL, ctrl); 1938} 1939 1940/* 1941 * Init Broadcom PCI registers 1942 */ 1943static void 1944ubsec_init_pciregs(device_t dev) 1945{ 1946#if 0 1947 u_int32_t misc; 1948 1949 misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT); 1950 misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT)) 1951 | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT); 1952 misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT)) 1953 | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT); 1954 pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc); 1955#endif 1956 1957 /* 1958 * This will set the cache line size to 1, this will 1959 * force the BCM58xx chip just to do burst read/writes. 1960 * Cache line read/writes are to slow 1961 */ 1962 pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1); 1963} 1964 1965/* 1966 * Clean up after a chip crash. 1967 * It is assumed that the caller in splimp() 1968 */ 1969static void 1970ubsec_cleanchip(struct ubsec_softc *sc) 1971{ 1972 struct ubsec_q *q; 1973 1974 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 1975 q = SIMPLEQ_FIRST(&sc->sc_qchip); 1976 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next); 1977 ubsec_free_q(sc, q); 1978 } 1979 sc->sc_nqchip = 0; 1980} 1981 1982/* 1983 * free a ubsec_q 1984 * It is assumed that the caller is within splimp(). 1985 */ 1986static int 1987ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q) 1988{ 1989 struct ubsec_q *q2; 1990 struct cryptop *crp; 1991 int npkts; 1992 int i; 1993 1994 npkts = q->q_nstacked_mcrs; 1995 1996 for (i = 0; i < npkts; i++) { 1997 if(q->q_stacked_mcr[i]) { 1998 q2 = q->q_stacked_mcr[i]; 1999 2000 if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m)) 2001 m_freem(q2->q_dst_m); 2002 2003 crp = (struct cryptop *)q2->q_crp; 2004 2005 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next); 2006 2007 crp->crp_etype = EFAULT; 2008 crypto_done(crp); 2009 } else { 2010 break; 2011 } 2012 } 2013 2014 /* 2015 * Free header MCR 2016 */ 2017 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 2018 m_freem(q->q_dst_m); 2019 2020 crp = (struct cryptop *)q->q_crp; 2021 2022 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 2023 2024 crp->crp_etype = EFAULT; 2025 crypto_done(crp); 2026 return(0); 2027} 2028 2029/* 2030 * Routine to reset the chip and clean up. 2031 * It is assumed that the caller is in splimp() 2032 */ 2033static void 2034ubsec_totalreset(struct ubsec_softc *sc) 2035{ 2036 ubsec_reset_board(sc); 2037 ubsec_init_board(sc); 2038 ubsec_cleanchip(sc); 2039} 2040 2041static int 2042ubsec_dmamap_aligned(struct ubsec_operand *op) 2043{ 2044 int i; 2045 2046 for (i = 0; i < op->nsegs; i++) { 2047 if (op->segs[i].ds_addr & 3) 2048 return (0); 2049 if ((i != (op->nsegs - 1)) && 2050 (op->segs[i].ds_len & 3)) 2051 return (0); 2052 } 2053 return (1); 2054} 2055 2056static void 2057ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q) 2058{ 2059 switch (q->q_type) { 2060 case UBS_CTXOP_MODEXP: { 2061 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 2062 2063 ubsec_dma_free(sc, &me->me_q.q_mcr); 2064 ubsec_dma_free(sc, &me->me_q.q_ctx); 2065 ubsec_dma_free(sc, &me->me_M); 2066 ubsec_dma_free(sc, &me->me_E); 2067 ubsec_dma_free(sc, &me->me_C); 2068 ubsec_dma_free(sc, &me->me_epb); 2069 free(me, M_DEVBUF); 2070 break; 2071 } 2072 case UBS_CTXOP_RSAPRIV: { 2073 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 2074 2075 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2076 ubsec_dma_free(sc, &rp->rpr_q.q_ctx); 2077 ubsec_dma_free(sc, &rp->rpr_msgin); 2078 ubsec_dma_free(sc, &rp->rpr_msgout); 2079 free(rp, M_DEVBUF); 2080 break; 2081 } 2082 default: 2083 device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type); 2084 break; 2085 } 2086} 2087 2088static int 2089ubsec_kprocess(void *arg, struct cryptkop *krp, int hint) 2090{ 2091 struct ubsec_softc *sc = arg; 2092 int r; 2093 2094 if (krp == NULL || krp->krp_callback == NULL) 2095 return (EINVAL); 2096 2097 while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) { 2098 struct ubsec_q2 *q; 2099 2100 q = SIMPLEQ_FIRST(&sc->sc_q2free); 2101 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q, q_next); 2102 ubsec_kfree(sc, q); 2103 } 2104 2105 switch (krp->krp_op) { 2106 case CRK_MOD_EXP: 2107 if (sc->sc_flags & UBS_FLAGS_HWNORM) 2108 r = ubsec_kprocess_modexp_hw(sc, krp, hint); 2109 else 2110 r = ubsec_kprocess_modexp_sw(sc, krp, hint); 2111 break; 2112 case CRK_MOD_EXP_CRT: 2113 return (ubsec_kprocess_rsapriv(sc, krp, hint)); 2114 default: 2115 device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n", 2116 krp->krp_op); 2117 krp->krp_status = EOPNOTSUPP; 2118 crypto_kdone(krp); 2119 return (0); 2120 } 2121 return (0); /* silence compiler */ 2122} 2123 2124/* 2125 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization) 2126 */ 2127static int 2128ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2129{ 2130 struct ubsec_q2_modexp *me; 2131 struct ubsec_mcr *mcr; 2132 struct ubsec_ctx_modexp *ctx; 2133 struct ubsec_pktbuf *epb; 2134 int err = 0; 2135 u_int nbits, normbits, mbits, shiftbits, ebits; 2136 2137 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2138 if (me == NULL) { 2139 err = ENOMEM; 2140 goto errout; 2141 } 2142 bzero(me, sizeof *me); 2143 me->me_krp = krp; 2144 me->me_q.q_type = UBS_CTXOP_MODEXP; 2145 2146 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2147 if (nbits <= 512) 2148 normbits = 512; 2149 else if (nbits <= 768) 2150 normbits = 768; 2151 else if (nbits <= 1024) 2152 normbits = 1024; 2153 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2154 normbits = 1536; 2155 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2156 normbits = 2048; 2157 else { 2158 err = E2BIG; 2159 goto errout; 2160 } 2161 2162 shiftbits = normbits - nbits; 2163 2164 me->me_modbits = nbits; 2165 me->me_shiftbits = shiftbits; 2166 me->me_normbits = normbits; 2167 2168 /* Sanity check: result bits must be >= true modulus bits. */ 2169 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2170 err = ERANGE; 2171 goto errout; 2172 } 2173 2174 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2175 &me->me_q.q_mcr, 0)) { 2176 err = ENOMEM; 2177 goto errout; 2178 } 2179 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2180 2181 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2182 &me->me_q.q_ctx, 0)) { 2183 err = ENOMEM; 2184 goto errout; 2185 } 2186 2187 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2188 if (mbits > nbits) { 2189 err = E2BIG; 2190 goto errout; 2191 } 2192 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2193 err = ENOMEM; 2194 goto errout; 2195 } 2196 ubsec_kshift_r(shiftbits, 2197 krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits, 2198 me->me_M.dma_vaddr, normbits); 2199 2200 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2201 err = ENOMEM; 2202 goto errout; 2203 } 2204 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2205 2206 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2207 if (ebits > nbits) { 2208 err = E2BIG; 2209 goto errout; 2210 } 2211 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2212 err = ENOMEM; 2213 goto errout; 2214 } 2215 ubsec_kshift_r(shiftbits, 2216 krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits, 2217 me->me_E.dma_vaddr, normbits); 2218 2219 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2220 &me->me_epb, 0)) { 2221 err = ENOMEM; 2222 goto errout; 2223 } 2224 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2225 epb->pb_addr = htole32(me->me_E.dma_paddr); 2226 epb->pb_next = 0; 2227 epb->pb_len = htole32(normbits / 8); 2228 2229#ifdef UBSEC_DEBUG 2230 if (ubsec_debug) { 2231 printf("Epb "); 2232 ubsec_dump_pb(epb); 2233 } 2234#endif 2235 2236 mcr->mcr_pkts = htole16(1); 2237 mcr->mcr_flags = 0; 2238 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2239 mcr->mcr_reserved = 0; 2240 mcr->mcr_pktlen = 0; 2241 2242 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2243 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2244 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2245 2246 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2247 mcr->mcr_opktbuf.pb_next = 0; 2248 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2249 2250#ifdef DIAGNOSTIC 2251 /* Misaligned output buffer will hang the chip. */ 2252 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2253 panic("%s: modexp invalid addr 0x%x\n", 2254 device_get_nameunit(sc->sc_dev), 2255 letoh32(mcr->mcr_opktbuf.pb_addr)); 2256 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2257 panic("%s: modexp invalid len 0x%x\n", 2258 device_get_nameunit(sc->sc_dev), 2259 letoh32(mcr->mcr_opktbuf.pb_len)); 2260#endif 2261 2262 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2263 bzero(ctx, sizeof(*ctx)); 2264 ubsec_kshift_r(shiftbits, 2265 krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits, 2266 ctx->me_N, normbits); 2267 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2268 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2269 ctx->me_E_len = htole16(nbits); 2270 ctx->me_N_len = htole16(nbits); 2271 2272#ifdef UBSEC_DEBUG 2273 if (ubsec_debug) { 2274 ubsec_dump_mcr(mcr); 2275 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2276 } 2277#endif 2278 2279 /* 2280 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2281 * everything else. 2282 */ 2283 ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE); 2284 ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE); 2285 ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD); 2286 ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE); 2287 2288 /* Enqueue and we're done... */ 2289 UBSEC_LOCK(sc); 2290 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2291 ubsec_feed2(sc); 2292 ubsecstats.hst_modexp++; 2293 UBSEC_UNLOCK(sc); 2294 2295 return (0); 2296 2297errout: 2298 if (me != NULL) { 2299 if (me->me_q.q_mcr.dma_map != NULL) 2300 ubsec_dma_free(sc, &me->me_q.q_mcr); 2301 if (me->me_q.q_ctx.dma_map != NULL) { 2302 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2303 ubsec_dma_free(sc, &me->me_q.q_ctx); 2304 } 2305 if (me->me_M.dma_map != NULL) { 2306 bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2307 ubsec_dma_free(sc, &me->me_M); 2308 } 2309 if (me->me_E.dma_map != NULL) { 2310 bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2311 ubsec_dma_free(sc, &me->me_E); 2312 } 2313 if (me->me_C.dma_map != NULL) { 2314 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2315 ubsec_dma_free(sc, &me->me_C); 2316 } 2317 if (me->me_epb.dma_map != NULL) 2318 ubsec_dma_free(sc, &me->me_epb); 2319 free(me, M_DEVBUF); 2320 } 2321 krp->krp_status = err; 2322 crypto_kdone(krp); 2323 return (0); 2324} 2325 2326/* 2327 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization) 2328 */ 2329static int 2330ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2331{ 2332 struct ubsec_q2_modexp *me; 2333 struct ubsec_mcr *mcr; 2334 struct ubsec_ctx_modexp *ctx; 2335 struct ubsec_pktbuf *epb; 2336 int err = 0; 2337 u_int nbits, normbits, mbits, shiftbits, ebits; 2338 2339 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2340 if (me == NULL) { 2341 err = ENOMEM; 2342 goto errout; 2343 } 2344 bzero(me, sizeof *me); 2345 me->me_krp = krp; 2346 me->me_q.q_type = UBS_CTXOP_MODEXP; 2347 2348 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2349 if (nbits <= 512) 2350 normbits = 512; 2351 else if (nbits <= 768) 2352 normbits = 768; 2353 else if (nbits <= 1024) 2354 normbits = 1024; 2355 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2356 normbits = 1536; 2357 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2358 normbits = 2048; 2359 else { 2360 err = E2BIG; 2361 goto errout; 2362 } 2363 2364 shiftbits = normbits - nbits; 2365 2366 /* XXX ??? */ 2367 me->me_modbits = nbits; 2368 me->me_shiftbits = shiftbits; 2369 me->me_normbits = normbits; 2370 2371 /* Sanity check: result bits must be >= true modulus bits. */ 2372 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2373 err = ERANGE; 2374 goto errout; 2375 } 2376 2377 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2378 &me->me_q.q_mcr, 0)) { 2379 err = ENOMEM; 2380 goto errout; 2381 } 2382 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2383 2384 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2385 &me->me_q.q_ctx, 0)) { 2386 err = ENOMEM; 2387 goto errout; 2388 } 2389 2390 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2391 if (mbits > nbits) { 2392 err = E2BIG; 2393 goto errout; 2394 } 2395 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2396 err = ENOMEM; 2397 goto errout; 2398 } 2399 bzero(me->me_M.dma_vaddr, normbits / 8); 2400 bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p, 2401 me->me_M.dma_vaddr, (mbits + 7) / 8); 2402 2403 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2404 err = ENOMEM; 2405 goto errout; 2406 } 2407 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2408 2409 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2410 if (ebits > nbits) { 2411 err = E2BIG; 2412 goto errout; 2413 } 2414 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2415 err = ENOMEM; 2416 goto errout; 2417 } 2418 bzero(me->me_E.dma_vaddr, normbits / 8); 2419 bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p, 2420 me->me_E.dma_vaddr, (ebits + 7) / 8); 2421 2422 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2423 &me->me_epb, 0)) { 2424 err = ENOMEM; 2425 goto errout; 2426 } 2427 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2428 epb->pb_addr = htole32(me->me_E.dma_paddr); 2429 epb->pb_next = 0; 2430 epb->pb_len = htole32((ebits + 7) / 8); 2431 2432#ifdef UBSEC_DEBUG 2433 if (ubsec_debug) { 2434 printf("Epb "); 2435 ubsec_dump_pb(epb); 2436 } 2437#endif 2438 2439 mcr->mcr_pkts = htole16(1); 2440 mcr->mcr_flags = 0; 2441 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2442 mcr->mcr_reserved = 0; 2443 mcr->mcr_pktlen = 0; 2444 2445 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2446 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2447 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2448 2449 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2450 mcr->mcr_opktbuf.pb_next = 0; 2451 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2452 2453#ifdef DIAGNOSTIC 2454 /* Misaligned output buffer will hang the chip. */ 2455 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2456 panic("%s: modexp invalid addr 0x%x\n", 2457 device_get_nameunit(sc->sc_dev), 2458 letoh32(mcr->mcr_opktbuf.pb_addr)); 2459 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2460 panic("%s: modexp invalid len 0x%x\n", 2461 device_get_nameunit(sc->sc_dev), 2462 letoh32(mcr->mcr_opktbuf.pb_len)); 2463#endif 2464 2465 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2466 bzero(ctx, sizeof(*ctx)); 2467 bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N, 2468 (nbits + 7) / 8); 2469 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2470 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2471 ctx->me_E_len = htole16(ebits); 2472 ctx->me_N_len = htole16(nbits); 2473 2474#ifdef UBSEC_DEBUG 2475 if (ubsec_debug) { 2476 ubsec_dump_mcr(mcr); 2477 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2478 } 2479#endif 2480 2481 /* 2482 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2483 * everything else. 2484 */ 2485 ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE); 2486 ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE); 2487 ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD); 2488 ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE); 2489 2490 /* Enqueue and we're done... */ 2491 UBSEC_LOCK(sc); 2492 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2493 ubsec_feed2(sc); 2494 UBSEC_UNLOCK(sc); 2495 2496 return (0); 2497 2498errout: 2499 if (me != NULL) { 2500 if (me->me_q.q_mcr.dma_map != NULL) 2501 ubsec_dma_free(sc, &me->me_q.q_mcr); 2502 if (me->me_q.q_ctx.dma_map != NULL) { 2503 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2504 ubsec_dma_free(sc, &me->me_q.q_ctx); 2505 } 2506 if (me->me_M.dma_map != NULL) { 2507 bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2508 ubsec_dma_free(sc, &me->me_M); 2509 } 2510 if (me->me_E.dma_map != NULL) { 2511 bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2512 ubsec_dma_free(sc, &me->me_E); 2513 } 2514 if (me->me_C.dma_map != NULL) { 2515 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2516 ubsec_dma_free(sc, &me->me_C); 2517 } 2518 if (me->me_epb.dma_map != NULL) 2519 ubsec_dma_free(sc, &me->me_epb); 2520 free(me, M_DEVBUF); 2521 } 2522 krp->krp_status = err; 2523 crypto_kdone(krp); 2524 return (0); 2525} 2526 2527static int 2528ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2529{ 2530 struct ubsec_q2_rsapriv *rp = NULL; 2531 struct ubsec_mcr *mcr; 2532 struct ubsec_ctx_rsapriv *ctx; 2533 int err = 0; 2534 u_int padlen, msglen; 2535 2536 msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]); 2537 padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]); 2538 if (msglen > padlen) 2539 padlen = msglen; 2540 2541 if (padlen <= 256) 2542 padlen = 256; 2543 else if (padlen <= 384) 2544 padlen = 384; 2545 else if (padlen <= 512) 2546 padlen = 512; 2547 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768) 2548 padlen = 768; 2549 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024) 2550 padlen = 1024; 2551 else { 2552 err = E2BIG; 2553 goto errout; 2554 } 2555 2556 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) { 2557 err = E2BIG; 2558 goto errout; 2559 } 2560 2561 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) { 2562 err = E2BIG; 2563 goto errout; 2564 } 2565 2566 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) { 2567 err = E2BIG; 2568 goto errout; 2569 } 2570 2571 rp = (struct ubsec_q2_rsapriv *)malloc(sizeof *rp, M_DEVBUF, M_NOWAIT); 2572 if (rp == NULL) 2573 return (ENOMEM); 2574 bzero(rp, sizeof *rp); 2575 rp->rpr_krp = krp; 2576 rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV; 2577 2578 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2579 &rp->rpr_q.q_mcr, 0)) { 2580 err = ENOMEM; 2581 goto errout; 2582 } 2583 mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr; 2584 2585 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv), 2586 &rp->rpr_q.q_ctx, 0)) { 2587 err = ENOMEM; 2588 goto errout; 2589 } 2590 ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr; 2591 bzero(ctx, sizeof *ctx); 2592 2593 /* Copy in p */ 2594 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p, 2595 &ctx->rpr_buf[0 * (padlen / 8)], 2596 (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8); 2597 2598 /* Copy in q */ 2599 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p, 2600 &ctx->rpr_buf[1 * (padlen / 8)], 2601 (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8); 2602 2603 /* Copy in dp */ 2604 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p, 2605 &ctx->rpr_buf[2 * (padlen / 8)], 2606 (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8); 2607 2608 /* Copy in dq */ 2609 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p, 2610 &ctx->rpr_buf[3 * (padlen / 8)], 2611 (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8); 2612 2613 /* Copy in pinv */ 2614 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p, 2615 &ctx->rpr_buf[4 * (padlen / 8)], 2616 (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8); 2617 2618 msglen = padlen * 2; 2619 2620 /* Copy in input message (aligned buffer/length). */ 2621 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) { 2622 /* Is this likely? */ 2623 err = E2BIG; 2624 goto errout; 2625 } 2626 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) { 2627 err = ENOMEM; 2628 goto errout; 2629 } 2630 bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8); 2631 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p, 2632 rp->rpr_msgin.dma_vaddr, 2633 (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8); 2634 2635 /* Prepare space for output message (aligned buffer/length). */ 2636 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) { 2637 /* Is this likely? */ 2638 err = E2BIG; 2639 goto errout; 2640 } 2641 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) { 2642 err = ENOMEM; 2643 goto errout; 2644 } 2645 bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8); 2646 2647 mcr->mcr_pkts = htole16(1); 2648 mcr->mcr_flags = 0; 2649 mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr); 2650 mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr); 2651 mcr->mcr_ipktbuf.pb_next = 0; 2652 mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size); 2653 mcr->mcr_reserved = 0; 2654 mcr->mcr_pktlen = htole16(msglen); 2655 mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr); 2656 mcr->mcr_opktbuf.pb_next = 0; 2657 mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size); 2658 2659#ifdef DIAGNOSTIC 2660 if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) { 2661 panic("%s: rsapriv: invalid msgin %x(0x%jx)", 2662 device_get_nameunit(sc->sc_dev), 2663 rp->rpr_msgin.dma_paddr, (uintmax_t)rp->rpr_msgin.dma_size); 2664 } 2665 if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) { 2666 panic("%s: rsapriv: invalid msgout %x(0x%jx)", 2667 device_get_nameunit(sc->sc_dev), 2668 rp->rpr_msgout.dma_paddr, (uintmax_t)rp->rpr_msgout.dma_size); 2669 } 2670#endif 2671 2672 ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8)); 2673 ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV); 2674 ctx->rpr_q_len = htole16(padlen); 2675 ctx->rpr_p_len = htole16(padlen); 2676 2677 /* 2678 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2679 * everything else. 2680 */ 2681 ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_PREWRITE); 2682 ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_PREREAD); 2683 2684 /* Enqueue and we're done... */ 2685 UBSEC_LOCK(sc); 2686 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next); 2687 ubsec_feed2(sc); 2688 ubsecstats.hst_modexpcrt++; 2689 UBSEC_UNLOCK(sc); 2690 return (0); 2691 2692errout: 2693 if (rp != NULL) { 2694 if (rp->rpr_q.q_mcr.dma_map != NULL) 2695 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2696 if (rp->rpr_msgin.dma_map != NULL) { 2697 bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 2698 ubsec_dma_free(sc, &rp->rpr_msgin); 2699 } 2700 if (rp->rpr_msgout.dma_map != NULL) { 2701 bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 2702 ubsec_dma_free(sc, &rp->rpr_msgout); 2703 } 2704 free(rp, M_DEVBUF); 2705 } 2706 krp->krp_status = err; 2707 crypto_kdone(krp); 2708 return (0); 2709} 2710 2711#ifdef UBSEC_DEBUG 2712static void 2713ubsec_dump_pb(volatile struct ubsec_pktbuf *pb) 2714{ 2715 printf("addr 0x%x (0x%x) next 0x%x\n", 2716 pb->pb_addr, pb->pb_len, pb->pb_next); 2717} 2718 2719static void 2720ubsec_dump_ctx2(struct ubsec_ctx_keyop *c) 2721{ 2722 printf("CTX (0x%x):\n", c->ctx_len); 2723 switch (letoh16(c->ctx_op)) { 2724 case UBS_CTXOP_RNGBYPASS: 2725 case UBS_CTXOP_RNGSHA1: 2726 break; 2727 case UBS_CTXOP_MODEXP: 2728 { 2729 struct ubsec_ctx_modexp *cx = (void *)c; 2730 int i, len; 2731 2732 printf(" Elen %u, Nlen %u\n", 2733 letoh16(cx->me_E_len), letoh16(cx->me_N_len)); 2734 len = (cx->me_N_len + 7)/8; 2735 for (i = 0; i < len; i++) 2736 printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]); 2737 printf("\n"); 2738 break; 2739 } 2740 default: 2741 printf("unknown context: %x\n", c->ctx_op); 2742 } 2743 printf("END CTX\n"); 2744} 2745 2746static void 2747ubsec_dump_mcr(struct ubsec_mcr *mcr) 2748{ 2749 volatile struct ubsec_mcr_add *ma; 2750 int i; 2751 2752 printf("MCR:\n"); 2753 printf(" pkts: %u, flags 0x%x\n", 2754 letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags)); 2755 ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp; 2756 for (i = 0; i < letoh16(mcr->mcr_pkts); i++) { 2757 printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i, 2758 letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen), 2759 letoh16(ma->mcr_reserved)); 2760 printf(" %d: ipkt ", i); 2761 ubsec_dump_pb(&ma->mcr_ipktbuf); 2762 printf(" %d: opkt ", i); 2763 ubsec_dump_pb(&ma->mcr_opktbuf); 2764 ma++; 2765 } 2766 printf("END MCR\n"); 2767} 2768#endif /* UBSEC_DEBUG */ 2769 2770/* 2771 * Return the number of significant bits of a big number. 2772 */ 2773static int 2774ubsec_ksigbits(struct crparam *cr) 2775{ 2776 u_int plen = (cr->crp_nbits + 7) / 8; 2777 int i, sig = plen * 8; 2778 u_int8_t c, *p = cr->crp_p; 2779 2780 for (i = plen - 1; i >= 0; i--) { 2781 c = p[i]; 2782 if (c != 0) { 2783 while ((c & 0x80) == 0) { 2784 sig--; 2785 c <<= 1; 2786 } 2787 break; 2788 } 2789 sig -= 8; 2790 } 2791 return (sig); 2792} 2793 2794static void 2795ubsec_kshift_r( 2796 u_int shiftbits, 2797 u_int8_t *src, u_int srcbits, 2798 u_int8_t *dst, u_int dstbits) 2799{ 2800 u_int slen, dlen; 2801 int i, si, di, n; 2802 2803 slen = (srcbits + 7) / 8; 2804 dlen = (dstbits + 7) / 8; 2805 2806 for (i = 0; i < slen; i++) 2807 dst[i] = src[i]; 2808 for (i = 0; i < dlen - slen; i++) 2809 dst[slen + i] = 0; 2810 2811 n = shiftbits / 8; 2812 if (n != 0) { 2813 si = dlen - n - 1; 2814 di = dlen - 1; 2815 while (si >= 0) 2816 dst[di--] = dst[si--]; 2817 while (di >= 0) 2818 dst[di--] = 0; 2819 } 2820 2821 n = shiftbits % 8; 2822 if (n != 0) { 2823 for (i = dlen - 1; i > 0; i--) 2824 dst[i] = (dst[i] << n) | 2825 (dst[i - 1] >> (8 - n)); 2826 dst[0] = dst[0] << n; 2827 } 2828} 2829 2830static void 2831ubsec_kshift_l( 2832 u_int shiftbits, 2833 u_int8_t *src, u_int srcbits, 2834 u_int8_t *dst, u_int dstbits) 2835{ 2836 int slen, dlen, i, n; 2837 2838 slen = (srcbits + 7) / 8; 2839 dlen = (dstbits + 7) / 8; 2840 2841 n = shiftbits / 8; 2842 for (i = 0; i < slen; i++) 2843 dst[i] = src[i + n]; 2844 for (i = 0; i < dlen - slen; i++) 2845 dst[slen + i] = 0; 2846 2847 n = shiftbits % 8; 2848 if (n != 0) { 2849 for (i = 0; i < (dlen - 1); i++) 2850 dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n)); 2851 dst[dlen - 1] = dst[dlen - 1] >> n; 2852 } 2853} 2854