ubsec.c revision 104918
1166255Sdelphij/* $FreeBSD: head/sys/dev/ubsec/ubsec.c 104918 2002-10-11 17:34:00Z sam $ */ 2166255Sdelphij/* $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $ */ 3166255Sdelphij 4166255Sdelphij/* 5166255Sdelphij * Copyright (c) 2000 Jason L. Wright (jason@thought.net) 6166255Sdelphij * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org) 7166255Sdelphij * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com) 8166255Sdelphij * 9166255Sdelphij * All rights reserved. 10166255Sdelphij * 11166255Sdelphij * Redistribution and use in source and binary forms, with or without 12166255Sdelphij * modification, are permitted provided that the following conditions 13166255Sdelphij * are met: 14166255Sdelphij * 1. Redistributions of source code must retain the above copyright 15166255Sdelphij * notice, this list of conditions and the following disclaimer. 16166255Sdelphij * 2. Redistributions in binary form must reproduce the above copyright 17166255Sdelphij * notice, this list of conditions and the following disclaimer in the 18166255Sdelphij * documentation and/or other materials provided with the distribution. 19166255Sdelphij * 3. All advertising materials mentioning features or use of this software 20166255Sdelphij * must display the following acknowledgement: 21166255Sdelphij * This product includes software developed by Jason L. Wright 22166255Sdelphij * 4. The name of the author may not be used to endorse or promote products 23166255Sdelphij * derived from this software without specific prior written permission. 24166255Sdelphij * 25166255Sdelphij * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 26166255Sdelphij * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 27166255Sdelphij * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 28166255Sdelphij * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 29166255Sdelphij * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 30166255Sdelphij * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 31166255Sdelphij * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32166255Sdelphij * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 33166255Sdelphij * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 34166255Sdelphij * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35166255Sdelphij * POSSIBILITY OF SUCH DAMAGE. 36166255Sdelphij * 37166255Sdelphij * Effort sponsored in part by the Defense Advanced Research Projects 38166255Sdelphij * Agency (DARPA) and Air Force Research Laboratory, Air Force 39166255Sdelphij * Materiel Command, USAF, under agreement number F30602-01-2-0537. 40166255Sdelphij * 41166255Sdelphij */ 42166255Sdelphij 43166255Sdelphij#define UBSEC_DEBUG 44166255Sdelphij 45166255Sdelphij/* 46166255Sdelphij * uBsec 5[56]01, 58xx hardware crypto accelerator 47166255Sdelphij */ 48166255Sdelphij 49166255Sdelphij#include <sys/param.h> 50166255Sdelphij#include <sys/systm.h> 51166255Sdelphij#include <sys/proc.h> 52166255Sdelphij#include <sys/errno.h> 53166255Sdelphij#include <sys/malloc.h> 54166255Sdelphij#include <sys/kernel.h> 55166255Sdelphij#include <sys/mbuf.h> 56#include <sys/lock.h> 57#include <sys/mutex.h> 58#include <sys/sysctl.h> 59#include <sys/endian.h> 60 61#include <vm/vm.h> 62#include <vm/pmap.h> 63 64#include <machine/clock.h> 65#include <machine/bus.h> 66#include <machine/resource.h> 67#include <sys/bus.h> 68#include <sys/rman.h> 69 70#include <crypto/sha1.h> 71#include <opencrypto/cryptodev.h> 72#include <opencrypto/cryptosoft.h> 73#include <sys/md5.h> 74#include <sys/random.h> 75 76#include <pci/pcivar.h> 77#include <pci/pcireg.h> 78 79/* grr, #defines for gratuitous incompatibility in queue.h */ 80#define SIMPLEQ_HEAD STAILQ_HEAD 81#define SIMPLEQ_ENTRY STAILQ_ENTRY 82#define SIMPLEQ_INIT STAILQ_INIT 83#define SIMPLEQ_INSERT_TAIL STAILQ_INSERT_TAIL 84#define SIMPLEQ_EMPTY STAILQ_EMPTY 85#define SIMPLEQ_FIRST STAILQ_FIRST 86#define SIMPLEQ_REMOVE_HEAD STAILQ_REMOVE_HEAD_UNTIL 87/* ditto for endian.h */ 88#define letoh16(x) le16toh(x) 89#define letoh32(x) le32toh(x) 90 91#include <dev/ubsec/ubsecreg.h> 92#include <dev/ubsec/ubsecvar.h> 93 94/* 95 * Prototypes and count for the pci_device structure 96 */ 97static int ubsec_probe(device_t); 98static int ubsec_attach(device_t); 99static int ubsec_detach(device_t); 100static int ubsec_suspend(device_t); 101static int ubsec_resume(device_t); 102static void ubsec_shutdown(device_t); 103 104static device_method_t ubsec_methods[] = { 105 /* Device interface */ 106 DEVMETHOD(device_probe, ubsec_probe), 107 DEVMETHOD(device_attach, ubsec_attach), 108 DEVMETHOD(device_detach, ubsec_detach), 109 DEVMETHOD(device_suspend, ubsec_suspend), 110 DEVMETHOD(device_resume, ubsec_resume), 111 DEVMETHOD(device_shutdown, ubsec_shutdown), 112 113 /* bus interface */ 114 DEVMETHOD(bus_print_child, bus_generic_print_child), 115 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 116 117 { 0, 0 } 118}; 119static driver_t ubsec_driver = { 120 "ubsec", 121 ubsec_methods, 122 sizeof (struct ubsec_softc) 123}; 124static devclass_t ubsec_devclass; 125 126DRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0); 127 128static void ubsec_intr(void *); 129static int ubsec_newsession(void *, u_int32_t *, struct cryptoini *); 130static int ubsec_freesession(void *, u_int64_t); 131static int ubsec_process(void *, struct cryptop *, int); 132static void ubsec_callback(struct ubsec_softc *, struct ubsec_q *); 133static int ubsec_feed(struct ubsec_softc *); 134static void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int); 135static void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *); 136static int ubsec_feed2(struct ubsec_softc *); 137static void ubsec_rng(void *); 138static int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t, 139 struct ubsec_dma_alloc *, int); 140static void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *); 141static int ubsec_dmamap_aligned(struct ubsec_operand *op); 142 143static void ubsec_reset_board(struct ubsec_softc *sc); 144static void ubsec_init_board(struct ubsec_softc *sc); 145static void ubsec_init_pciregs(device_t dev); 146static void ubsec_totalreset(struct ubsec_softc *sc); 147 148static int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q); 149 150static int ubsec_kprocess(void*, struct cryptkop *, int); 151static int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int); 152static int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int); 153static int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int); 154static void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *); 155static int ubsec_ksigbits(struct crparam *); 156static void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 157static void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 158 159#ifdef UBSEC_DEBUG 160static void ubsec_dump_pb(volatile struct ubsec_pktbuf *); 161static void ubsec_dump_mcr(struct ubsec_mcr *); 162static void ubsec_dump_ctx2(struct ubsec_ctx_keyop *); 163 164static int ubsec_debug = 0; 165SYSCTL_INT(_debug, OID_AUTO, ubsec, CTLFLAG_RW, &ubsec_debug, 166 0, "UBSEC driver debugging printfs"); 167#endif 168 169#define READ_REG(sc,r) \ 170 bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r)) 171 172#define WRITE_REG(sc,reg,val) \ 173 bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val) 174 175#define SWAP32(x) (x) = htole32(ntohl((x))) 176#define HTOLE32(x) (x) = htole32(x) 177 178 179struct ubsec_stats ubsecstats; 180SYSCTL_STRUCT(_kern, OID_AUTO, ubsec_stats, CTLFLAG_RD, &ubsecstats, 181 ubsec_stats, "Broadcom driver statistics"); 182static int ubsec_maxbatch = 2; /* XXX tune based on part+sys speed */ 183SYSCTL_INT(_kern, OID_AUTO, ubsec_maxbatch, CTLFLAG_RW, &ubsec_maxbatch, 184 0, "Broadcom driver: max ops to batch w/o interrupt"); 185 186int 187ubsec_probe(device_t dev) 188{ 189 if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 190 (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 || 191 pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)) 192 return (0); 193 if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 194 (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 || 195 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 || 196 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 197 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822)) 198 return (0); 199 return (ENXIO); 200} 201 202static const char* 203ubsec_partname(struct ubsec_softc *sc) 204{ 205 /* XXX sprintf numbers when not decoded */ 206 switch (pci_get_vendor(sc->sc_dev)) { 207 case PCI_VENDOR_BROADCOM: 208 switch (pci_get_device(sc->sc_dev)) { 209 case PCI_PRODUCT_BROADCOM_5805: return "Broadcom 5805"; 210 case PCI_PRODUCT_BROADCOM_5820: return "Broadcom 5820"; 211 case PCI_PRODUCT_BROADCOM_5821: return "Broadcom 5821"; 212 case PCI_PRODUCT_BROADCOM_5822: return "Broadcom 5822"; 213 } 214 return "Broadcom unknown-part"; 215 case PCI_VENDOR_BLUESTEEL: 216 switch (pci_get_device(sc->sc_dev)) { 217 case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601"; 218 } 219 return "Bluesteel unknown-part"; 220 } 221 return "Unknown-vendor unknown-part"; 222} 223 224static int 225ubsec_attach(device_t dev) 226{ 227 struct ubsec_softc *sc = device_get_softc(dev); 228 struct ubsec_dma *dmap; 229 u_int32_t cmd, i; 230 int rid; 231 232 KASSERT(sc != NULL, ("ubsec_attach: null software carrier!")); 233 bzero(sc, sizeof (*sc)); 234 sc->sc_dev = dev; 235 236 mtx_init(&sc->sc_mtx, device_get_nameunit(dev), "crypto driver", MTX_DEF); 237 238 SIMPLEQ_INIT(&sc->sc_queue); 239 SIMPLEQ_INIT(&sc->sc_qchip); 240 SIMPLEQ_INIT(&sc->sc_queue2); 241 SIMPLEQ_INIT(&sc->sc_qchip2); 242 SIMPLEQ_INIT(&sc->sc_q2free); 243 244 /* XXX handle power management */ 245 246 sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR; 247 248 if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 249 pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601) 250 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 251 252 if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 253 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805) 254 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 255 256 if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 257 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820) 258 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 259 UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 260 261 if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 262 (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 263 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822)) { 264 /* NB: the 5821/5822 defines some additional status bits */ 265 sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY | 266 BS_STAT_MCR2_ALLEMPTY; 267 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 268 UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 269 } 270 /* XXX no PK key support until we sort out the bus_dma stuff */ 271 sc->sc_flags &= ~UBS_FLAGS_KEY; 272 273 cmd = pci_read_config(dev, PCIR_COMMAND, 4); 274 cmd |= PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN; 275 pci_write_config(dev, PCIR_COMMAND, cmd, 4); 276 cmd = pci_read_config(dev, PCIR_COMMAND, 4); 277 278 if (!(cmd & PCIM_CMD_MEMEN)) { 279 device_printf(dev, "failed to enable memory mapping\n"); 280 goto bad; 281 } 282 283 if (!(cmd & PCIM_CMD_BUSMASTEREN)) { 284 device_printf(dev, "failed to enable bus mastering\n"); 285 goto bad; 286 } 287 288 /* 289 * Setup memory-mapping of PCI registers. 290 */ 291 rid = BS_BAR; 292 sc->sc_sr = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid, 293 0, ~0, 1, RF_ACTIVE); 294 if (sc->sc_sr == NULL) { 295 device_printf(dev, "cannot map register space\n"); 296 goto bad; 297 } 298 sc->sc_st = rman_get_bustag(sc->sc_sr); 299 sc->sc_sh = rman_get_bushandle(sc->sc_sr); 300 301 /* 302 * Arrange interrupt line. 303 */ 304 rid = 0; 305 sc->sc_irq = bus_alloc_resource(dev, SYS_RES_IRQ, &rid, 306 0, ~0, 1, RF_SHAREABLE|RF_ACTIVE); 307 if (sc->sc_irq == NULL) { 308 device_printf(dev, "could not map interrupt\n"); 309 goto bad; 310 } 311 /* 312 * NB: Network code assumes we are blocked with splimp() 313 * so make sure the IRQ is mapped appropriately. 314 */ 315 if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET, 316 ubsec_intr, sc, &sc->sc_ih)) { 317 device_printf(dev, "could not establish interrupt\n"); 318 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 319 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 320 goto bad; 321 } 322 323 sc->sc_cid = crypto_get_driverid(0); 324 if (sc->sc_cid < 0) { 325 device_printf(dev, "could not get crypto driver id\n"); 326 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 327 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 328 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 329 goto bad; 330 } 331 332 /* 333 * Setup DMA descriptor area. 334 */ 335 if (bus_dma_tag_create(NULL, /* parent */ 336 1, 0, /* alignment, bounds */ 337 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 338 BUS_SPACE_MAXADDR, /* highaddr */ 339 NULL, NULL, /* filter, filterarg */ 340 0x3ffff, /* maxsize XXX */ 341 UBS_MAX_SCATTER, /* nsegments */ 342 0xffff, /* maxsegsize XXX */ 343 BUS_DMA_ALLOCNOW, /* flags */ 344 &sc->sc_dmat)) { 345 device_printf(dev, "cannot allocate DMA tag\n"); 346 crypto_unregister_all(sc->sc_cid); 347 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 348 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 349 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 350 goto bad; 351 } 352 SIMPLEQ_INIT(&sc->sc_freequeue); 353 dmap = sc->sc_dmaa; 354 for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) { 355 struct ubsec_q *q; 356 357 q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q), 358 M_DEVBUF, M_NOWAIT); 359 if (q == NULL) { 360 device_printf(dev, "cannot allocate queue buffers\n"); 361 break; 362 } 363 364 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk), 365 &dmap->d_alloc, 0)) { 366 device_printf(dev, "cannot allocate dma buffers\n"); 367 free(q, M_DEVBUF); 368 break; 369 } 370 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr; 371 372 q->q_dma = dmap; 373 sc->sc_queuea[i] = q; 374 375 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 376 } 377 378 device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc)); 379 380 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0, 381 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 382 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0, 383 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 384 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0, 385 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 386 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0, 387 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 388 389 /* 390 * Reset Broadcom chip 391 */ 392 ubsec_reset_board(sc); 393 394 /* 395 * Init Broadcom specific PCI settings 396 */ 397 ubsec_init_pciregs(dev); 398 399 /* 400 * Init Broadcom chip 401 */ 402 ubsec_init_board(sc); 403 404#ifndef UBSEC_NO_RNG 405 if (sc->sc_flags & UBS_FLAGS_RNG) { 406 sc->sc_statmask |= BS_STAT_MCR2_DONE; 407 408 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 409 &sc->sc_rng.rng_q.q_mcr, 0)) 410 goto skip_rng; 411 412 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass), 413 &sc->sc_rng.rng_q.q_ctx, 0)) { 414 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 415 goto skip_rng; 416 } 417 418 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) * 419 UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) { 420 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 421 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 422 goto skip_rng; 423 } 424 425 if (hz >= 100) 426 sc->sc_rnghz = hz / 100; 427 else 428 sc->sc_rnghz = 1; 429 /* NB: 1 means the callout runs w/o Giant locked */ 430 callout_init(&sc->sc_rngto, 1); 431 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 432skip_rng: 433 ; 434 } 435#endif /* UBSEC_NO_RNG */ 436 437 if (sc->sc_flags & UBS_FLAGS_KEY) { 438 sc->sc_statmask |= BS_STAT_MCR2_DONE; 439 440 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0, 441 ubsec_kprocess, sc); 442#if 0 443 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0, 444 ubsec_kprocess, sc); 445#endif 446 } 447 return (0); 448bad: 449 mtx_destroy(&sc->sc_mtx); 450 return (ENXIO); 451} 452 453/* 454 * Detach a device that successfully probed. 455 */ 456static int 457ubsec_detach(device_t dev) 458{ 459 struct ubsec_softc *sc = device_get_softc(dev); 460 461 KASSERT(sc != NULL, ("ubsec_detach: null software carrier")); 462 463 UBSEC_LOCK(sc); 464 465 callout_stop(&sc->sc_rngto); 466 467 crypto_unregister_all(sc->sc_cid); 468 469 bus_generic_detach(dev); 470 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 471 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 472 473 bus_dma_tag_destroy(sc->sc_dmat); 474 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 475 476 UBSEC_UNLOCK(sc); 477 478 mtx_destroy(&sc->sc_mtx); 479 480 return (0); 481} 482 483/* 484 * Stop all chip i/o so that the kernel's probe routines don't 485 * get confused by errant DMAs when rebooting. 486 */ 487static void 488ubsec_shutdown(device_t dev) 489{ 490#ifdef notyet 491 ubsec_stop(device_get_softc(dev)); 492#endif 493} 494 495/* 496 * Device suspend routine. 497 */ 498static int 499ubsec_suspend(device_t dev) 500{ 501 struct ubsec_softc *sc = device_get_softc(dev); 502 503 KASSERT(sc != NULL, ("ubsec_suspend: null software carrier")); 504#ifdef notyet 505 /* XXX stop the device and save PCI settings */ 506#endif 507 sc->sc_suspended = 1; 508 509 return (0); 510} 511 512static int 513ubsec_resume(device_t dev) 514{ 515 struct ubsec_softc *sc = device_get_softc(dev); 516 517 KASSERT(sc != NULL, ("ubsec_resume: null software carrier")); 518#ifdef notyet 519 /* XXX retore PCI settings and start the device */ 520#endif 521 sc->sc_suspended = 0; 522 return (0); 523} 524 525/* 526 * UBSEC Interrupt routine 527 */ 528static void 529ubsec_intr(void *arg) 530{ 531 struct ubsec_softc *sc = arg; 532 volatile u_int32_t stat; 533 struct ubsec_q *q; 534 struct ubsec_dma *dmap; 535 int npkts = 0, i; 536 537 UBSEC_LOCK(sc); 538 539 stat = READ_REG(sc, BS_STAT); 540 stat &= sc->sc_statmask; 541 if (stat == 0) { 542 UBSEC_UNLOCK(sc); 543 return; 544 } 545 546 WRITE_REG(sc, BS_STAT, stat); /* IACK */ 547 548 /* 549 * Check to see if we have any packets waiting for us 550 */ 551 if ((stat & BS_STAT_MCR1_DONE)) { 552 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 553 q = SIMPLEQ_FIRST(&sc->sc_qchip); 554 dmap = q->q_dma; 555 556 if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0) 557 break; 558 559 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next); 560 561 npkts = q->q_nstacked_mcrs; 562 /* 563 * search for further sc_qchip ubsec_q's that share 564 * the same MCR, and complete them too, they must be 565 * at the top. 566 */ 567 for (i = 0; i < npkts; i++) { 568 if(q->q_stacked_mcr[i]) { 569 ubsec_callback(sc, q->q_stacked_mcr[i]); 570 ubsecstats.hst_opackets++; 571 } else { 572 break; 573 } 574 } 575 ubsec_callback(sc, q); 576 ubsecstats.hst_opackets++; 577 } 578 579 /* 580 * Don't send any more packet to chip if there has been 581 * a DMAERR. 582 */ 583 if (!(stat & BS_STAT_DMAERR)) 584 ubsec_feed(sc); 585 } 586 587 /* 588 * Check to see if we have any key setups/rng's waiting for us 589 */ 590 if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) && 591 (stat & BS_STAT_MCR2_DONE)) { 592 struct ubsec_q2 *q2; 593 struct ubsec_mcr *mcr; 594 595 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) { 596 q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 597 598 bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map, 599 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 600 601 mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 602 if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) { 603 bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map, 604 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 605 break; 606 } 607 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q2, q_next); 608 ubsec_callback2(sc, q2); 609 /* 610 * Don't send any more packet to chip if there has been 611 * a DMAERR. 612 */ 613 if (!(stat & BS_STAT_DMAERR)) 614 ubsec_feed2(sc); 615 } 616 } 617 618 /* 619 * Check to see if we got any DMA Error 620 */ 621 if (stat & BS_STAT_DMAERR) { 622#ifdef UBSEC_DEBUG 623 if (ubsec_debug) { 624 volatile u_int32_t a = READ_REG(sc, BS_ERR); 625 626 printf("dmaerr %s@%08x\n", 627 (a & BS_ERR_READ) ? "read" : "write", 628 a & BS_ERR_ADDR); 629 } 630#endif /* UBSEC_DEBUG */ 631 ubsecstats.hst_dmaerr++; 632 ubsec_totalreset(sc); 633 ubsec_feed(sc); 634 } 635 636 if (sc->sc_needwakeup) { /* XXX check high watermark */ 637 int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ); 638#ifdef UBSEC_DEBUG 639 if (ubsec_debug) 640 device_printf(sc->sc_dev, "wakeup crypto (%x)\n", 641 sc->sc_needwakeup); 642#endif /* UBSEC_DEBUG */ 643 sc->sc_needwakeup &= ~wakeup; 644 crypto_unblock(sc->sc_cid, wakeup); 645 } 646 647 UBSEC_UNLOCK(sc); 648} 649 650/* 651 * ubsec_feed() - aggregate and post requests to chip 652 */ 653static int 654ubsec_feed(struct ubsec_softc *sc) 655{ 656 struct ubsec_q *q, *q2; 657 int npkts, i; 658 void *v; 659 u_int32_t stat; 660 661 npkts = sc->sc_nqueue; 662 if (npkts > UBS_MAX_AGGR) 663 npkts = UBS_MAX_AGGR; 664 if (npkts > ubsecstats.hst_maxbatch) 665 ubsecstats.hst_maxbatch = npkts; 666 if (npkts < 2) 667 goto feed1; 668 ubsecstats.hst_totbatch += npkts-1; 669 670 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 671 if(stat & BS_STAT_DMAERR) { 672 ubsec_totalreset(sc); 673 ubsecstats.hst_dmaerr++; 674 } 675 return (0); 676 } 677 678#ifdef UBSEC_DEBUG 679 if (ubsec_debug) 680 printf("merging %d records\n", npkts); 681#endif /* UBSEC_DEBUG */ 682 683 q = SIMPLEQ_FIRST(&sc->sc_queue); 684 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next); 685 --sc->sc_nqueue; 686 687 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE); 688 if (q->q_dst_map != NULL) 689 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD); 690 691 q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */ 692 693 for (i = 0; i < q->q_nstacked_mcrs; i++) { 694 q2 = SIMPLEQ_FIRST(&sc->sc_queue); 695 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map, 696 BUS_DMASYNC_PREWRITE); 697 if (q2->q_dst_map != NULL) 698 bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map, 699 BUS_DMASYNC_PREREAD); 700 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q2, q_next); 701 --sc->sc_nqueue; 702 703 v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) - 704 sizeof(struct ubsec_mcr_add)); 705 bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add)); 706 q->q_stacked_mcr[i] = q2; 707 } 708 q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts); 709 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 710 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map, 711 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 712 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 713 offsetof(struct ubsec_dmachunk, d_mcr)); 714 return (0); 715 716feed1: 717 while (!SIMPLEQ_EMPTY(&sc->sc_queue)) { 718 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 719 if(stat & BS_STAT_DMAERR) { 720 ubsec_totalreset(sc); 721 ubsecstats.hst_dmaerr++; 722 } 723 break; 724 } 725 726 q = SIMPLEQ_FIRST(&sc->sc_queue); 727 728 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 729 BUS_DMASYNC_PREWRITE); 730 if (q->q_dst_map != NULL) 731 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 732 BUS_DMASYNC_PREREAD); 733 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map, 734 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 735 736 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 737 offsetof(struct ubsec_dmachunk, d_mcr)); 738#ifdef UBSEC_DEBUG 739 if (ubsec_debug) 740 printf("feed: q->chip %p %08x stat %08x\n", 741 q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr), 742 stat); 743#endif /* UBSEC_DEBUG */ 744 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next); 745 --sc->sc_nqueue; 746 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 747 } 748 return (0); 749} 750 751/* 752 * Allocate a new 'session' and return an encoded session id. 'sidp' 753 * contains our registration id, and should contain an encoded session 754 * id on successful allocation. 755 */ 756static int 757ubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri) 758{ 759 struct cryptoini *c, *encini = NULL, *macini = NULL; 760 struct ubsec_softc *sc = arg; 761 struct ubsec_session *ses = NULL; 762 MD5_CTX md5ctx; 763 SHA1_CTX sha1ctx; 764 int i, sesn; 765 766 KASSERT(sc != NULL, ("ubsec_newsession: null softc")); 767 if (sidp == NULL || cri == NULL || sc == NULL) 768 return (EINVAL); 769 770 for (c = cri; c != NULL; c = c->cri_next) { 771 if (c->cri_alg == CRYPTO_MD5_HMAC || 772 c->cri_alg == CRYPTO_SHA1_HMAC) { 773 if (macini) 774 return (EINVAL); 775 macini = c; 776 } else if (c->cri_alg == CRYPTO_DES_CBC || 777 c->cri_alg == CRYPTO_3DES_CBC) { 778 if (encini) 779 return (EINVAL); 780 encini = c; 781 } else 782 return (EINVAL); 783 } 784 if (encini == NULL && macini == NULL) 785 return (EINVAL); 786 787 if (sc->sc_sessions == NULL) { 788 ses = sc->sc_sessions = (struct ubsec_session *)malloc( 789 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 790 if (ses == NULL) 791 return (ENOMEM); 792 sesn = 0; 793 sc->sc_nsessions = 1; 794 } else { 795 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 796 if (sc->sc_sessions[sesn].ses_used == 0) { 797 ses = &sc->sc_sessions[sesn]; 798 break; 799 } 800 } 801 802 if (ses == NULL) { 803 sesn = sc->sc_nsessions; 804 ses = (struct ubsec_session *)malloc((sesn + 1) * 805 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 806 if (ses == NULL) 807 return (ENOMEM); 808 bcopy(sc->sc_sessions, ses, sesn * 809 sizeof(struct ubsec_session)); 810 bzero(sc->sc_sessions, sesn * 811 sizeof(struct ubsec_session)); 812 free(sc->sc_sessions, M_DEVBUF); 813 sc->sc_sessions = ses; 814 ses = &sc->sc_sessions[sesn]; 815 sc->sc_nsessions++; 816 } 817 } 818 819 bzero(ses, sizeof(struct ubsec_session)); 820 ses->ses_used = 1; 821 if (encini) { 822 /* get an IV, network byte order */ 823 /* XXX may read fewer than requested */ 824 read_random(ses->ses_iv, sizeof(ses->ses_iv)); 825 826 /* Go ahead and compute key in ubsec's byte order */ 827 if (encini->cri_alg == CRYPTO_DES_CBC) { 828 bcopy(encini->cri_key, &ses->ses_deskey[0], 8); 829 bcopy(encini->cri_key, &ses->ses_deskey[2], 8); 830 bcopy(encini->cri_key, &ses->ses_deskey[4], 8); 831 } else 832 bcopy(encini->cri_key, ses->ses_deskey, 24); 833 834 SWAP32(ses->ses_deskey[0]); 835 SWAP32(ses->ses_deskey[1]); 836 SWAP32(ses->ses_deskey[2]); 837 SWAP32(ses->ses_deskey[3]); 838 SWAP32(ses->ses_deskey[4]); 839 SWAP32(ses->ses_deskey[5]); 840 } 841 842 if (macini) { 843 for (i = 0; i < macini->cri_klen / 8; i++) 844 macini->cri_key[i] ^= HMAC_IPAD_VAL; 845 846 if (macini->cri_alg == CRYPTO_MD5_HMAC) { 847 MD5Init(&md5ctx); 848 MD5Update(&md5ctx, macini->cri_key, 849 macini->cri_klen / 8); 850 MD5Update(&md5ctx, hmac_ipad_buffer, 851 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 852 bcopy(md5ctx.state, ses->ses_hminner, 853 sizeof(md5ctx.state)); 854 } else { 855 SHA1Init(&sha1ctx); 856 SHA1Update(&sha1ctx, macini->cri_key, 857 macini->cri_klen / 8); 858 SHA1Update(&sha1ctx, hmac_ipad_buffer, 859 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 860 bcopy(sha1ctx.h.b32, ses->ses_hminner, 861 sizeof(sha1ctx.h.b32)); 862 } 863 864 for (i = 0; i < macini->cri_klen / 8; i++) 865 macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 866 867 if (macini->cri_alg == CRYPTO_MD5_HMAC) { 868 MD5Init(&md5ctx); 869 MD5Update(&md5ctx, macini->cri_key, 870 macini->cri_klen / 8); 871 MD5Update(&md5ctx, hmac_opad_buffer, 872 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 873 bcopy(md5ctx.state, ses->ses_hmouter, 874 sizeof(md5ctx.state)); 875 } else { 876 SHA1Init(&sha1ctx); 877 SHA1Update(&sha1ctx, macini->cri_key, 878 macini->cri_klen / 8); 879 SHA1Update(&sha1ctx, hmac_opad_buffer, 880 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 881 bcopy(sha1ctx.h.b32, ses->ses_hmouter, 882 sizeof(sha1ctx.h.b32)); 883 } 884 885 for (i = 0; i < macini->cri_klen / 8; i++) 886 macini->cri_key[i] ^= HMAC_OPAD_VAL; 887 } 888 889 *sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn); 890 return (0); 891} 892 893/* 894 * Deallocate a session. 895 */ 896static int 897ubsec_freesession(void *arg, u_int64_t tid) 898{ 899 struct ubsec_softc *sc = arg; 900 int session; 901 u_int32_t sid = ((u_int32_t) tid) & 0xffffffff; 902 903 KASSERT(sc != NULL, ("ubsec_freesession: null softc")); 904 if (sc == NULL) 905 return (EINVAL); 906 907 session = UBSEC_SESSION(sid); 908 if (session >= sc->sc_nsessions) 909 return (EINVAL); 910 911 bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session])); 912 return (0); 913} 914 915static void 916ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error) 917{ 918 struct ubsec_operand *op = arg; 919 920 KASSERT(nsegs <= UBS_MAX_SCATTER, 921 ("Too many DMA segments returned when mapping operand")); 922#ifdef UBSEC_DEBUG 923 if (ubsec_debug) 924 printf("ubsec_op_cb: mapsize %u nsegs %d\n", 925 (u_int) mapsize, nsegs); 926#endif 927 op->mapsize = mapsize; 928 op->nsegs = nsegs; 929 bcopy(seg, op->segs, nsegs * sizeof (seg[0])); 930} 931 932static int 933ubsec_process(void *arg, struct cryptop *crp, int hint) 934{ 935 struct ubsec_q *q = NULL; 936 int err = 0, i, j, nicealign; 937 struct ubsec_softc *sc = arg; 938 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 939 int encoffset = 0, macoffset = 0, cpskip, cpoffset; 940 int sskip, dskip, stheend, dtheend; 941 int16_t coffset; 942 struct ubsec_session *ses; 943 struct ubsec_pktctx ctx; 944 struct ubsec_dma *dmap = NULL; 945 946 if (crp == NULL || crp->crp_callback == NULL || sc == NULL) { 947 ubsecstats.hst_invalid++; 948 return (EINVAL); 949 } 950 if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) { 951 ubsecstats.hst_invalid++; 952 return (EINVAL); 953 } 954 955 UBSEC_LOCK(sc); 956 957 if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 958 ubsecstats.hst_queuefull++; 959 sc->sc_needwakeup |= CRYPTO_SYMQ; 960 UBSEC_UNLOCK(sc); 961 return (ERESTART); 962 } 963 q = SIMPLEQ_FIRST(&sc->sc_freequeue); 964 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q, q_next); 965 UBSEC_UNLOCK(sc); 966 967 dmap = q->q_dma; /* Save dma pointer */ 968 bzero(q, sizeof(struct ubsec_q)); 969 bzero(&ctx, sizeof(ctx)); 970 971 q->q_sesn = UBSEC_SESSION(crp->crp_sid); 972 q->q_dma = dmap; 973 ses = &sc->sc_sessions[q->q_sesn]; 974 975 if (crp->crp_flags & CRYPTO_F_IMBUF) { 976 q->q_src_m = (struct mbuf *)crp->crp_buf; 977 q->q_dst_m = (struct mbuf *)crp->crp_buf; 978 } else if (crp->crp_flags & CRYPTO_F_IOV) { 979 q->q_src_io = (struct uio *)crp->crp_buf; 980 q->q_dst_io = (struct uio *)crp->crp_buf; 981 } else { 982 ubsecstats.hst_invalid++; 983 err = EINVAL; 984 goto errout; /* XXX we don't handle contiguous blocks! */ 985 } 986 987 bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr)); 988 989 dmap->d_dma->d_mcr.mcr_pkts = htole16(1); 990 dmap->d_dma->d_mcr.mcr_flags = 0; 991 q->q_crp = crp; 992 993 crd1 = crp->crp_desc; 994 if (crd1 == NULL) { 995 ubsecstats.hst_invalid++; 996 err = EINVAL; 997 goto errout; 998 } 999 crd2 = crd1->crd_next; 1000 1001 if (crd2 == NULL) { 1002 if (crd1->crd_alg == CRYPTO_MD5_HMAC || 1003 crd1->crd_alg == CRYPTO_SHA1_HMAC) { 1004 maccrd = crd1; 1005 enccrd = NULL; 1006 } else if (crd1->crd_alg == CRYPTO_DES_CBC || 1007 crd1->crd_alg == CRYPTO_3DES_CBC) { 1008 maccrd = NULL; 1009 enccrd = crd1; 1010 } else { 1011 ubsecstats.hst_invalid++; 1012 err = EINVAL; 1013 goto errout; 1014 } 1015 } else { 1016 if ((crd1->crd_alg == CRYPTO_MD5_HMAC || 1017 crd1->crd_alg == CRYPTO_SHA1_HMAC) && 1018 (crd2->crd_alg == CRYPTO_DES_CBC || 1019 crd2->crd_alg == CRYPTO_3DES_CBC) && 1020 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 1021 maccrd = crd1; 1022 enccrd = crd2; 1023 } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 1024 crd1->crd_alg == CRYPTO_3DES_CBC) && 1025 (crd2->crd_alg == CRYPTO_MD5_HMAC || 1026 crd2->crd_alg == CRYPTO_SHA1_HMAC) && 1027 (crd1->crd_flags & CRD_F_ENCRYPT)) { 1028 enccrd = crd1; 1029 maccrd = crd2; 1030 } else { 1031 /* 1032 * We cannot order the ubsec as requested 1033 */ 1034 ubsecstats.hst_invalid++; 1035 err = EINVAL; 1036 goto errout; 1037 } 1038 } 1039 1040 if (enccrd) { 1041 encoffset = enccrd->crd_skip; 1042 ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES); 1043 1044 if (enccrd->crd_flags & CRD_F_ENCRYPT) { 1045 q->q_flags |= UBSEC_QFLAGS_COPYOUTIV; 1046 1047 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1048 bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1049 else { 1050 ctx.pc_iv[0] = ses->ses_iv[0]; 1051 ctx.pc_iv[1] = ses->ses_iv[1]; 1052 } 1053 1054 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { 1055 if (crp->crp_flags & CRYPTO_F_IMBUF) 1056 m_copyback(q->q_src_m, 1057 enccrd->crd_inject, 1058 8, (caddr_t)ctx.pc_iv); 1059 else if (crp->crp_flags & CRYPTO_F_IOV) 1060 cuio_copyback(q->q_src_io, 1061 enccrd->crd_inject, 1062 8, (caddr_t)ctx.pc_iv); 1063 } 1064 } else { 1065 ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND); 1066 1067 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1068 bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1069 else if (crp->crp_flags & CRYPTO_F_IMBUF) 1070 m_copydata(q->q_src_m, enccrd->crd_inject, 1071 8, (caddr_t)ctx.pc_iv); 1072 else if (crp->crp_flags & CRYPTO_F_IOV) 1073 cuio_copydata(q->q_src_io, 1074 enccrd->crd_inject, 8, 1075 (caddr_t)ctx.pc_iv); 1076 } 1077 1078 ctx.pc_deskey[0] = ses->ses_deskey[0]; 1079 ctx.pc_deskey[1] = ses->ses_deskey[1]; 1080 ctx.pc_deskey[2] = ses->ses_deskey[2]; 1081 ctx.pc_deskey[3] = ses->ses_deskey[3]; 1082 ctx.pc_deskey[4] = ses->ses_deskey[4]; 1083 ctx.pc_deskey[5] = ses->ses_deskey[5]; 1084 SWAP32(ctx.pc_iv[0]); 1085 SWAP32(ctx.pc_iv[1]); 1086 } 1087 1088 if (maccrd) { 1089 macoffset = maccrd->crd_skip; 1090 1091 if (maccrd->crd_alg == CRYPTO_MD5_HMAC) 1092 ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5); 1093 else 1094 ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1); 1095 1096 for (i = 0; i < 5; i++) { 1097 ctx.pc_hminner[i] = ses->ses_hminner[i]; 1098 ctx.pc_hmouter[i] = ses->ses_hmouter[i]; 1099 1100 HTOLE32(ctx.pc_hminner[i]); 1101 HTOLE32(ctx.pc_hmouter[i]); 1102 } 1103 } 1104 1105 if (enccrd && maccrd) { 1106 /* 1107 * ubsec cannot handle packets where the end of encryption 1108 * and authentication are not the same, or where the 1109 * encrypted part begins before the authenticated part. 1110 */ 1111 if ((encoffset + enccrd->crd_len) != 1112 (macoffset + maccrd->crd_len)) { 1113 ubsecstats.hst_lenmismatch++; 1114 err = EINVAL; 1115 goto errout; 1116 } 1117 if (enccrd->crd_skip < maccrd->crd_skip) { 1118 ubsecstats.hst_skipmismatch++; 1119 err = EINVAL; 1120 goto errout; 1121 } 1122 sskip = maccrd->crd_skip; 1123 cpskip = dskip = enccrd->crd_skip; 1124 stheend = maccrd->crd_len; 1125 dtheend = enccrd->crd_len; 1126 coffset = enccrd->crd_skip - maccrd->crd_skip; 1127 cpoffset = cpskip + dtheend; 1128#ifdef UBSEC_DEBUG 1129 if (ubsec_debug) { 1130 printf("mac: skip %d, len %d, inject %d\n", 1131 maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject); 1132 printf("enc: skip %d, len %d, inject %d\n", 1133 enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject); 1134 printf("src: skip %d, len %d\n", sskip, stheend); 1135 printf("dst: skip %d, len %d\n", dskip, dtheend); 1136 printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n", 1137 coffset, stheend, cpskip, cpoffset); 1138 } 1139#endif 1140 } else { 1141 cpskip = dskip = sskip = macoffset + encoffset; 1142 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len; 1143 cpoffset = cpskip + dtheend; 1144 coffset = 0; 1145 } 1146 ctx.pc_offset = htole16(coffset >> 2); 1147 1148 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) { 1149 ubsecstats.hst_nomap++; 1150 err = ENOMEM; 1151 goto errout; 1152 } 1153 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1154 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map, 1155 q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1156 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1157 q->q_src_map = NULL; 1158 ubsecstats.hst_noload++; 1159 err = ENOMEM; 1160 goto errout; 1161 } 1162 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1163 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map, 1164 q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1165 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1166 q->q_src_map = NULL; 1167 ubsecstats.hst_noload++; 1168 err = ENOMEM; 1169 goto errout; 1170 } 1171 } 1172 nicealign = ubsec_dmamap_aligned(&q->q_src); 1173 1174 dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend); 1175 1176#ifdef UBSEC_DEBUG 1177 if (ubsec_debug) 1178 printf("src skip: %d nicealign: %u\n", sskip, nicealign); 1179#endif 1180 for (i = j = 0; i < q->q_src_nsegs; i++) { 1181 struct ubsec_pktbuf *pb; 1182 bus_size_t packl = q->q_src_segs[i].ds_len; 1183 bus_addr_t packp = q->q_src_segs[i].ds_addr; 1184 1185 if (sskip >= packl) { 1186 sskip -= packl; 1187 continue; 1188 } 1189 1190 packl -= sskip; 1191 packp += sskip; 1192 sskip = 0; 1193 1194 if (packl > 0xfffc) { 1195 err = EIO; 1196 goto errout; 1197 } 1198 1199 if (j == 0) 1200 pb = &dmap->d_dma->d_mcr.mcr_ipktbuf; 1201 else 1202 pb = &dmap->d_dma->d_sbuf[j - 1]; 1203 1204 pb->pb_addr = htole32(packp); 1205 1206 if (stheend) { 1207 if (packl > stheend) { 1208 pb->pb_len = htole32(stheend); 1209 stheend = 0; 1210 } else { 1211 pb->pb_len = htole32(packl); 1212 stheend -= packl; 1213 } 1214 } else 1215 pb->pb_len = htole32(packl); 1216 1217 if ((i + 1) == q->q_src_nsegs) 1218 pb->pb_next = 0; 1219 else 1220 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1221 offsetof(struct ubsec_dmachunk, d_sbuf[j])); 1222 j++; 1223 } 1224 1225 if (enccrd == NULL && maccrd != NULL) { 1226 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0; 1227 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0; 1228 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr + 1229 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1230#ifdef UBSEC_DEBUG 1231 if (ubsec_debug) 1232 printf("opkt: %x %x %x\n", 1233 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr, 1234 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len, 1235 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next); 1236#endif 1237 } else { 1238 if (crp->crp_flags & CRYPTO_F_IOV) { 1239 if (!nicealign) { 1240 ubsecstats.hst_iovmisaligned++; 1241 err = EINVAL; 1242 goto errout; 1243 } 1244 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, 1245 &q->q_dst_map)) { 1246 ubsecstats.hst_nomap++; 1247 err = ENOMEM; 1248 goto errout; 1249 } 1250 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map, 1251 q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) { 1252 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1253 q->q_dst_map = NULL; 1254 ubsecstats.hst_noload++; 1255 err = ENOMEM; 1256 goto errout; 1257 } 1258 } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1259 if (nicealign) { 1260 q->q_dst = q->q_src; 1261 } else { 1262 int totlen, len; 1263 struct mbuf *m, *top, **mp; 1264 1265 ubsecstats.hst_unaligned++; 1266 totlen = q->q_src_mapsize; 1267 if (q->q_src_m->m_flags & M_PKTHDR) { 1268 len = MHLEN; 1269 MGETHDR(m, M_DONTWAIT, MT_DATA); 1270 } else { 1271 len = MLEN; 1272 MGET(m, M_DONTWAIT, MT_DATA); 1273 } 1274 if (m == NULL) { 1275 ubsecstats.hst_nombuf++; 1276 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1277 goto errout; 1278 } 1279 if (len == MHLEN) 1280 M_COPY_PKTHDR(m, q->q_src_m); 1281 if (totlen >= MINCLSIZE) { 1282 MCLGET(m, M_DONTWAIT); 1283 if ((m->m_flags & M_EXT) == 0) { 1284 m_free(m); 1285 ubsecstats.hst_nomcl++; 1286 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1287 goto errout; 1288 } 1289 len = MCLBYTES; 1290 } 1291 m->m_len = len; 1292 top = NULL; 1293 mp = ⊤ 1294 1295 while (totlen > 0) { 1296 if (top) { 1297 MGET(m, M_DONTWAIT, MT_DATA); 1298 if (m == NULL) { 1299 m_freem(top); 1300 ubsecstats.hst_nombuf++; 1301 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1302 goto errout; 1303 } 1304 len = MLEN; 1305 } 1306 if (top && totlen >= MINCLSIZE) { 1307 MCLGET(m, M_DONTWAIT); 1308 if ((m->m_flags & M_EXT) == 0) { 1309 *mp = m; 1310 m_freem(top); 1311 ubsecstats.hst_nomcl++; 1312 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1313 goto errout; 1314 } 1315 len = MCLBYTES; 1316 } 1317 m->m_len = len = min(totlen, len); 1318 totlen -= len; 1319 *mp = m; 1320 mp = &m->m_next; 1321 } 1322 q->q_dst_m = top; 1323 ubsec_mcopy(q->q_src_m, q->q_dst_m, 1324 cpskip, cpoffset); 1325 if (bus_dmamap_create(sc->sc_dmat, 1326 BUS_DMA_NOWAIT, &q->q_dst_map) != 0) { 1327 ubsecstats.hst_nomap++; 1328 err = ENOMEM; 1329 goto errout; 1330 } 1331 if (bus_dmamap_load_mbuf(sc->sc_dmat, 1332 q->q_dst_map, q->q_dst_m, 1333 ubsec_op_cb, &q->q_dst, 1334 BUS_DMA_NOWAIT) != 0) { 1335 bus_dmamap_destroy(sc->sc_dmat, 1336 q->q_dst_map); 1337 q->q_dst_map = NULL; 1338 ubsecstats.hst_noload++; 1339 err = ENOMEM; 1340 goto errout; 1341 } 1342 } 1343 } else { 1344 ubsecstats.hst_invalid++; 1345 err = EINVAL; 1346 goto errout; 1347 } 1348 1349#ifdef UBSEC_DEBUG 1350 if (ubsec_debug) 1351 printf("dst skip: %d\n", dskip); 1352#endif 1353 for (i = j = 0; i < q->q_dst_nsegs; i++) { 1354 struct ubsec_pktbuf *pb; 1355 bus_size_t packl = q->q_dst_segs[i].ds_len; 1356 bus_addr_t packp = q->q_dst_segs[i].ds_addr; 1357 1358 if (dskip >= packl) { 1359 dskip -= packl; 1360 continue; 1361 } 1362 1363 packl -= dskip; 1364 packp += dskip; 1365 dskip = 0; 1366 1367 if (packl > 0xfffc) { 1368 err = EIO; 1369 goto errout; 1370 } 1371 1372 if (j == 0) 1373 pb = &dmap->d_dma->d_mcr.mcr_opktbuf; 1374 else 1375 pb = &dmap->d_dma->d_dbuf[j - 1]; 1376 1377 pb->pb_addr = htole32(packp); 1378 1379 if (dtheend) { 1380 if (packl > dtheend) { 1381 pb->pb_len = htole32(dtheend); 1382 dtheend = 0; 1383 } else { 1384 pb->pb_len = htole32(packl); 1385 dtheend -= packl; 1386 } 1387 } else 1388 pb->pb_len = htole32(packl); 1389 1390 if ((i + 1) == q->q_dst_nsegs) { 1391 if (maccrd) 1392 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1393 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1394 else 1395 pb->pb_next = 0; 1396 } else 1397 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1398 offsetof(struct ubsec_dmachunk, d_dbuf[j])); 1399 j++; 1400 } 1401 } 1402 1403 dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr + 1404 offsetof(struct ubsec_dmachunk, d_ctx)); 1405 1406 if (sc->sc_flags & UBS_FLAGS_LONGCTX) { 1407 struct ubsec_pktctx_long *ctxl; 1408 1409 ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr + 1410 offsetof(struct ubsec_dmachunk, d_ctx)); 1411 1412 /* transform small context into long context */ 1413 ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long)); 1414 ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC); 1415 ctxl->pc_flags = ctx.pc_flags; 1416 ctxl->pc_offset = ctx.pc_offset; 1417 for (i = 0; i < 6; i++) 1418 ctxl->pc_deskey[i] = ctx.pc_deskey[i]; 1419 for (i = 0; i < 5; i++) 1420 ctxl->pc_hminner[i] = ctx.pc_hminner[i]; 1421 for (i = 0; i < 5; i++) 1422 ctxl->pc_hmouter[i] = ctx.pc_hmouter[i]; 1423 ctxl->pc_iv[0] = ctx.pc_iv[0]; 1424 ctxl->pc_iv[1] = ctx.pc_iv[1]; 1425 } else 1426 bcopy(&ctx, dmap->d_alloc.dma_vaddr + 1427 offsetof(struct ubsec_dmachunk, d_ctx), 1428 sizeof(struct ubsec_pktctx)); 1429 1430 UBSEC_LOCK(sc); 1431 SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next); 1432 sc->sc_nqueue++; 1433 ubsecstats.hst_ipackets++; 1434 ubsecstats.hst_ibytes += dmap->d_alloc.dma_size; 1435 if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= ubsec_maxbatch) 1436 ubsec_feed(sc); 1437 UBSEC_UNLOCK(sc); 1438 return (0); 1439 1440errout: 1441 if (q != NULL) { 1442 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 1443 m_freem(q->q_dst_m); 1444 1445 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1446 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1447 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1448 } 1449 if (q->q_src_map != NULL) { 1450 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1451 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1452 } 1453 1454 UBSEC_LOCK(sc); 1455 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1456 UBSEC_UNLOCK(sc); 1457 } 1458 if (err != ERESTART) { 1459 crp->crp_etype = err; 1460 crypto_done(crp); 1461 } else { 1462 sc->sc_needwakeup |= CRYPTO_SYMQ; 1463 } 1464 return (err); 1465} 1466 1467static void 1468ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q) 1469{ 1470 struct cryptop *crp = (struct cryptop *)q->q_crp; 1471 struct cryptodesc *crd; 1472 struct ubsec_dma *dmap = q->q_dma; 1473 1474 bus_dmamap_sync(sc->sc_dmat, dmap->d_alloc.dma_map, 1475 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1476 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1477 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 1478 BUS_DMASYNC_POSTREAD); 1479 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1480 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1481 } 1482 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE); 1483 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1484 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1485 1486 if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) { 1487 m_freem(q->q_src_m); 1488 crp->crp_buf = (caddr_t)q->q_dst_m; 1489 } 1490 ubsecstats.hst_obytes += ((struct mbuf *)crp->crp_buf)->m_len; 1491 1492 /* copy out IV for future use */ 1493 if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) { 1494 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1495 if (crd->crd_alg != CRYPTO_DES_CBC && 1496 crd->crd_alg != CRYPTO_3DES_CBC) 1497 continue; 1498 if (crp->crp_flags & CRYPTO_F_IMBUF) 1499 m_copydata((struct mbuf *)crp->crp_buf, 1500 crd->crd_skip + crd->crd_len - 8, 8, 1501 (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv); 1502 else if (crp->crp_flags & CRYPTO_F_IOV) { 1503 cuio_copydata((struct uio *)crp->crp_buf, 1504 crd->crd_skip + crd->crd_len - 8, 8, 1505 (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv); 1506 } 1507 break; 1508 } 1509 } 1510 1511 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1512 if (crd->crd_alg != CRYPTO_MD5_HMAC && 1513 crd->crd_alg != CRYPTO_SHA1_HMAC) 1514 continue; 1515 if (crp->crp_flags & CRYPTO_F_IMBUF) 1516 m_copyback((struct mbuf *)crp->crp_buf, 1517 crd->crd_inject, 12, 1518 (caddr_t)dmap->d_dma->d_macbuf); 1519 else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac) 1520 bcopy((caddr_t)dmap->d_dma->d_macbuf, 1521 crp->crp_mac, 12); 1522 break; 1523 } 1524 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1525 crypto_done(crp); 1526} 1527 1528static void 1529ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset) 1530{ 1531 int i, j, dlen, slen; 1532 caddr_t dptr, sptr; 1533 1534 j = 0; 1535 sptr = srcm->m_data; 1536 slen = srcm->m_len; 1537 dptr = dstm->m_data; 1538 dlen = dstm->m_len; 1539 1540 while (1) { 1541 for (i = 0; i < min(slen, dlen); i++) { 1542 if (j < hoffset || j >= toffset) 1543 *dptr++ = *sptr++; 1544 slen--; 1545 dlen--; 1546 j++; 1547 } 1548 if (slen == 0) { 1549 srcm = srcm->m_next; 1550 if (srcm == NULL) 1551 return; 1552 sptr = srcm->m_data; 1553 slen = srcm->m_len; 1554 } 1555 if (dlen == 0) { 1556 dstm = dstm->m_next; 1557 if (dstm == NULL) 1558 return; 1559 dptr = dstm->m_data; 1560 dlen = dstm->m_len; 1561 } 1562 } 1563} 1564 1565/* 1566 * feed the key generator, must be called at splimp() or higher. 1567 */ 1568static int 1569ubsec_feed2(struct ubsec_softc *sc) 1570{ 1571 struct ubsec_q2 *q; 1572 1573 while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) { 1574 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL) 1575 break; 1576 q = SIMPLEQ_FIRST(&sc->sc_queue2); 1577 1578 bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 1579 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1580 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 1581 BUS_DMASYNC_PREWRITE); 1582 1583 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr); 1584 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q, q_next); 1585 --sc->sc_nqueue2; 1586 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next); 1587 } 1588 return (0); 1589} 1590 1591/* 1592 * Callback for handling random numbers 1593 */ 1594static void 1595ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q) 1596{ 1597 struct cryptkop *krp; 1598 struct ubsec_ctx_keyop *ctx; 1599 1600 ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr; 1601 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, BUS_DMASYNC_POSTWRITE); 1602 1603 switch (q->q_type) { 1604#ifndef UBSEC_NO_RNG 1605 case UBS_CTXOP_RNGBYPASS: { 1606 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q; 1607 1608 bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 1609 BUS_DMASYNC_POSTREAD); 1610 random_harvest(rng->rng_buf.dma_vaddr, 1611 UBSEC_RNG_BUFSIZ*sizeof (u_int32_t), 1612 UBSEC_RNG_BUFSIZ*sizeof (u_int32_t)*NBBY, 0, 1613 RANDOM_PURE); 1614 rng->rng_used = 0; 1615 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1616 break; 1617 } 1618#endif 1619 case UBS_CTXOP_MODEXP: { 1620 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 1621 u_int rlen, clen; 1622 1623 krp = me->me_krp; 1624 rlen = (me->me_modbits + 7) / 8; 1625 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8; 1626 1627 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 1628 BUS_DMASYNC_POSTWRITE); 1629 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 1630 BUS_DMASYNC_POSTWRITE); 1631 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 1632 BUS_DMASYNC_POSTREAD); 1633 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 1634 BUS_DMASYNC_POSTWRITE); 1635 1636 if (clen < rlen) 1637 krp->krp_status = E2BIG; 1638 else { 1639 if (sc->sc_flags & UBS_FLAGS_HWNORM) { 1640 bzero(krp->krp_param[krp->krp_iparams].crp_p, 1641 (krp->krp_param[krp->krp_iparams].crp_nbits 1642 + 7) / 8); 1643 bcopy(me->me_C.dma_vaddr, 1644 krp->krp_param[krp->krp_iparams].crp_p, 1645 (me->me_modbits + 7) / 8); 1646 } else 1647 ubsec_kshift_l(me->me_shiftbits, 1648 me->me_C.dma_vaddr, me->me_normbits, 1649 krp->krp_param[krp->krp_iparams].crp_p, 1650 krp->krp_param[krp->krp_iparams].crp_nbits); 1651 } 1652 1653 crypto_kdone(krp); 1654 1655 /* bzero all potentially sensitive data */ 1656 bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 1657 bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 1658 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 1659 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 1660 1661 /* Can't free here, so put us on the free list. */ 1662 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next); 1663 break; 1664 } 1665 case UBS_CTXOP_RSAPRIV: { 1666 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 1667 u_int len; 1668 1669 krp = rp->rpr_krp; 1670 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 1671 BUS_DMASYNC_POSTWRITE); 1672 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 1673 BUS_DMASYNC_POSTREAD); 1674 1675 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8; 1676 bcopy(rp->rpr_msgout.dma_vaddr, 1677 krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len); 1678 1679 crypto_kdone(krp); 1680 1681 bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 1682 bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 1683 bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size); 1684 1685 /* Can't free here, so put us on the free list. */ 1686 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next); 1687 break; 1688 } 1689 default: 1690 device_printf(sc->sc_dev, "unknown ctx op: %x\n", 1691 letoh16(ctx->ctx_op)); 1692 break; 1693 } 1694} 1695 1696#ifndef UBSEC_NO_RNG 1697static void 1698ubsec_rng(void *vsc) 1699{ 1700 struct ubsec_softc *sc = vsc; 1701 struct ubsec_q2_rng *rng = &sc->sc_rng; 1702 struct ubsec_mcr *mcr; 1703 struct ubsec_ctx_rngbypass *ctx; 1704 1705 UBSEC_LOCK(sc); 1706 if (rng->rng_used) { 1707 UBSEC_UNLOCK(sc); 1708 return; 1709 } 1710 sc->sc_nqueue2++; 1711 if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE) 1712 goto out; 1713 1714 mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr; 1715 ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr; 1716 1717 mcr->mcr_pkts = htole16(1); 1718 mcr->mcr_flags = 0; 1719 mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr); 1720 mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0; 1721 mcr->mcr_ipktbuf.pb_len = 0; 1722 mcr->mcr_reserved = mcr->mcr_pktlen = 0; 1723 mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr); 1724 mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) & 1725 UBS_PKTBUF_LEN); 1726 mcr->mcr_opktbuf.pb_next = 0; 1727 1728 ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass)); 1729 ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS); 1730 rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS; 1731 1732 bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, BUS_DMASYNC_PREREAD); 1733 1734 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next); 1735 rng->rng_used = 1; 1736 ubsec_feed2(sc); 1737 ubsecstats.hst_rng++; 1738 UBSEC_UNLOCK(sc); 1739 1740 return; 1741 1742out: 1743 /* 1744 * Something weird happened, generate our own call back. 1745 */ 1746 sc->sc_nqueue2--; 1747 UBSEC_UNLOCK(sc); 1748 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1749} 1750#endif /* UBSEC_NO_RNG */ 1751 1752static void 1753ubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 1754{ 1755 bus_addr_t *paddr = (bus_addr_t*) arg; 1756 *paddr = segs->ds_addr; 1757} 1758 1759static int 1760ubsec_dma_malloc( 1761 struct ubsec_softc *sc, 1762 bus_size_t size, 1763 struct ubsec_dma_alloc *dma, 1764 int mapflags 1765) 1766{ 1767 int r; 1768 1769 r = bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &dma->dma_map); 1770 if (r != 0) 1771 goto fail_0; 1772 1773 r = bus_dmamem_alloc(sc->sc_dmat, (void**) &dma->dma_vaddr, 1774 BUS_DMA_NOWAIT, &dma->dma_map); 1775 if (r != 0) 1776 goto fail_1; 1777 1778 r = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr, 1779 size, 1780 ubsec_dmamap_cb, 1781 &dma->dma_paddr, 1782 mapflags | BUS_DMA_NOWAIT); 1783 if (r != 0) 1784 goto fail_2; 1785 1786 dma->dma_size = size; 1787 return (0); 1788 1789fail_2: 1790 bus_dmamap_unload(sc->sc_dmat, dma->dma_map); 1791fail_1: 1792 bus_dmamem_free(sc->sc_dmat, dma->dma_vaddr, dma->dma_map); 1793fail_0: 1794 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map); 1795 dma->dma_map = NULL; 1796 return (r); 1797} 1798 1799static void 1800ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma) 1801{ 1802 bus_dmamap_unload(sc->sc_dmat, dma->dma_map); 1803 bus_dmamem_free(sc->sc_dmat, dma->dma_vaddr, dma->dma_map); 1804 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map); 1805} 1806 1807/* 1808 * Resets the board. Values in the regesters are left as is 1809 * from the reset (i.e. initial values are assigned elsewhere). 1810 */ 1811static void 1812ubsec_reset_board(struct ubsec_softc *sc) 1813{ 1814 volatile u_int32_t ctrl; 1815 1816 ctrl = READ_REG(sc, BS_CTRL); 1817 ctrl |= BS_CTRL_RESET; 1818 WRITE_REG(sc, BS_CTRL, ctrl); 1819 1820 /* 1821 * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us 1822 */ 1823 DELAY(10); 1824} 1825 1826/* 1827 * Init Broadcom registers 1828 */ 1829static void 1830ubsec_init_board(struct ubsec_softc *sc) 1831{ 1832 u_int32_t ctrl; 1833 1834 ctrl = READ_REG(sc, BS_CTRL); 1835 ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64); 1836 ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT; 1837 1838 if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) 1839 ctrl |= BS_CTRL_MCR2INT; 1840 else 1841 ctrl &= ~BS_CTRL_MCR2INT; 1842 1843 if (sc->sc_flags & UBS_FLAGS_HWNORM) 1844 ctrl &= ~BS_CTRL_SWNORM; 1845 1846 WRITE_REG(sc, BS_CTRL, ctrl); 1847} 1848 1849/* 1850 * Init Broadcom PCI registers 1851 */ 1852static void 1853ubsec_init_pciregs(device_t dev) 1854{ 1855#if 0 1856 u_int32_t misc; 1857 1858 misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT); 1859 misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT)) 1860 | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT); 1861 misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT)) 1862 | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT); 1863 pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc); 1864#endif 1865 1866 /* 1867 * This will set the cache line size to 1, this will 1868 * force the BCM58xx chip just to do burst read/writes. 1869 * Cache line read/writes are to slow 1870 */ 1871 pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1); 1872} 1873 1874/* 1875 * Clean up after a chip crash. 1876 * It is assumed that the caller in splimp() 1877 */ 1878static void 1879ubsec_cleanchip(struct ubsec_softc *sc) 1880{ 1881 struct ubsec_q *q; 1882 1883 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 1884 q = SIMPLEQ_FIRST(&sc->sc_qchip); 1885 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next); 1886 ubsec_free_q(sc, q); 1887 } 1888} 1889 1890/* 1891 * free a ubsec_q 1892 * It is assumed that the caller is within spimp() 1893 */ 1894static int 1895ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q) 1896{ 1897 struct ubsec_q *q2; 1898 struct cryptop *crp; 1899 int npkts; 1900 int i; 1901 1902 npkts = q->q_nstacked_mcrs; 1903 1904 for (i = 0; i < npkts; i++) { 1905 if(q->q_stacked_mcr[i]) { 1906 q2 = q->q_stacked_mcr[i]; 1907 1908 if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m)) 1909 m_freem(q2->q_dst_m); 1910 1911 crp = (struct cryptop *)q2->q_crp; 1912 1913 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next); 1914 1915 crp->crp_etype = EFAULT; 1916 crypto_done(crp); 1917 } else { 1918 break; 1919 } 1920 } 1921 1922 /* 1923 * Free header MCR 1924 */ 1925 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 1926 m_freem(q->q_dst_m); 1927 1928 crp = (struct cryptop *)q->q_crp; 1929 1930 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1931 1932 crp->crp_etype = EFAULT; 1933 crypto_done(crp); 1934 return(0); 1935} 1936 1937/* 1938 * Routine to reset the chip and clean up. 1939 * It is assumed that the caller is in splimp() 1940 */ 1941static void 1942ubsec_totalreset(struct ubsec_softc *sc) 1943{ 1944 ubsec_reset_board(sc); 1945 ubsec_init_board(sc); 1946 ubsec_cleanchip(sc); 1947} 1948 1949static int 1950ubsec_dmamap_aligned(struct ubsec_operand *op) 1951{ 1952 int i; 1953 1954 for (i = 0; i < op->nsegs; i++) { 1955 if (op->segs[i].ds_addr & 3) 1956 return (0); 1957 if ((i != (op->nsegs - 1)) && 1958 (op->segs[i].ds_len & 3)) 1959 return (0); 1960 } 1961 return (1); 1962} 1963 1964static void 1965ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q) 1966{ 1967 switch (q->q_type) { 1968 case UBS_CTXOP_MODEXP: { 1969 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 1970 1971 ubsec_dma_free(sc, &me->me_q.q_mcr); 1972 ubsec_dma_free(sc, &me->me_q.q_ctx); 1973 ubsec_dma_free(sc, &me->me_M); 1974 ubsec_dma_free(sc, &me->me_E); 1975 ubsec_dma_free(sc, &me->me_C); 1976 ubsec_dma_free(sc, &me->me_epb); 1977 free(me, M_DEVBUF); 1978 break; 1979 } 1980 case UBS_CTXOP_RSAPRIV: { 1981 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 1982 1983 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 1984 ubsec_dma_free(sc, &rp->rpr_q.q_ctx); 1985 ubsec_dma_free(sc, &rp->rpr_msgin); 1986 ubsec_dma_free(sc, &rp->rpr_msgout); 1987 free(rp, M_DEVBUF); 1988 break; 1989 } 1990 default: 1991 device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type); 1992 break; 1993 } 1994} 1995 1996static int 1997ubsec_kprocess(void *arg, struct cryptkop *krp, int hint) 1998{ 1999 struct ubsec_softc *sc = arg; 2000 int r; 2001 2002 if (krp == NULL || krp->krp_callback == NULL) 2003 return (EINVAL); 2004 2005 while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) { 2006 struct ubsec_q2 *q; 2007 2008 q = SIMPLEQ_FIRST(&sc->sc_q2free); 2009 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q, q_next); 2010 ubsec_kfree(sc, q); 2011 } 2012 2013 switch (krp->krp_op) { 2014 case CRK_MOD_EXP: 2015 if (sc->sc_flags & UBS_FLAGS_HWNORM) 2016 r = ubsec_kprocess_modexp_hw(sc, krp, hint); 2017 else 2018 r = ubsec_kprocess_modexp_sw(sc, krp, hint); 2019 break; 2020 case CRK_MOD_EXP_CRT: 2021 return (ubsec_kprocess_rsapriv(sc, krp, hint)); 2022 default: 2023 device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n", 2024 krp->krp_op); 2025 krp->krp_status = EOPNOTSUPP; 2026 crypto_kdone(krp); 2027 return (0); 2028 } 2029 return (0); /* silence compiler */ 2030} 2031 2032/* 2033 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization) 2034 */ 2035static int 2036ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2037{ 2038 struct ubsec_q2_modexp *me; 2039 struct ubsec_mcr *mcr; 2040 struct ubsec_ctx_modexp *ctx; 2041 struct ubsec_pktbuf *epb; 2042 int err = 0; 2043 u_int nbits, normbits, mbits, shiftbits, ebits; 2044 2045 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2046 if (me == NULL) { 2047 err = ENOMEM; 2048 goto errout; 2049 } 2050 bzero(me, sizeof *me); 2051 me->me_krp = krp; 2052 me->me_q.q_type = UBS_CTXOP_MODEXP; 2053 2054 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2055 if (nbits <= 512) 2056 normbits = 512; 2057 else if (nbits <= 768) 2058 normbits = 768; 2059 else if (nbits <= 1024) 2060 normbits = 1024; 2061 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2062 normbits = 1536; 2063 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2064 normbits = 2048; 2065 else { 2066 err = E2BIG; 2067 goto errout; 2068 } 2069 2070 shiftbits = normbits - nbits; 2071 2072 me->me_modbits = nbits; 2073 me->me_shiftbits = shiftbits; 2074 me->me_normbits = normbits; 2075 2076 /* Sanity check: result bits must be >= true modulus bits. */ 2077 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2078 err = ERANGE; 2079 goto errout; 2080 } 2081 2082 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2083 &me->me_q.q_mcr, 0)) { 2084 err = ENOMEM; 2085 goto errout; 2086 } 2087 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2088 2089 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2090 &me->me_q.q_ctx, 0)) { 2091 err = ENOMEM; 2092 goto errout; 2093 } 2094 2095 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2096 if (mbits > nbits) { 2097 err = E2BIG; 2098 goto errout; 2099 } 2100 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2101 err = ENOMEM; 2102 goto errout; 2103 } 2104 ubsec_kshift_r(shiftbits, 2105 krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits, 2106 me->me_M.dma_vaddr, normbits); 2107 2108 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2109 err = ENOMEM; 2110 goto errout; 2111 } 2112 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2113 2114 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2115 if (ebits > nbits) { 2116 err = E2BIG; 2117 goto errout; 2118 } 2119 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2120 err = ENOMEM; 2121 goto errout; 2122 } 2123 ubsec_kshift_r(shiftbits, 2124 krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits, 2125 me->me_E.dma_vaddr, normbits); 2126 2127 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2128 &me->me_epb, 0)) { 2129 err = ENOMEM; 2130 goto errout; 2131 } 2132 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2133 epb->pb_addr = htole32(me->me_E.dma_paddr); 2134 epb->pb_next = 0; 2135 epb->pb_len = htole32(normbits / 8); 2136 2137#ifdef UBSEC_DEBUG 2138 if (ubsec_debug) { 2139 printf("Epb "); 2140 ubsec_dump_pb(epb); 2141 } 2142#endif 2143 2144 mcr->mcr_pkts = htole16(1); 2145 mcr->mcr_flags = 0; 2146 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2147 mcr->mcr_reserved = 0; 2148 mcr->mcr_pktlen = 0; 2149 2150 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2151 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2152 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2153 2154 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2155 mcr->mcr_opktbuf.pb_next = 0; 2156 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2157 2158#ifdef DIAGNOSTIC 2159 /* Misaligned output buffer will hang the chip. */ 2160 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2161 panic("%s: modexp invalid addr 0x%x\n", 2162 device_get_nameunit(sc->sc_dev), 2163 letoh32(mcr->mcr_opktbuf.pb_addr)); 2164 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2165 panic("%s: modexp invalid len 0x%x\n", 2166 device_get_nameunit(sc->sc_dev), 2167 letoh32(mcr->mcr_opktbuf.pb_len)); 2168#endif 2169 2170 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2171 bzero(ctx, sizeof(*ctx)); 2172 ubsec_kshift_r(shiftbits, 2173 krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits, 2174 ctx->me_N, normbits); 2175 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2176 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2177 ctx->me_E_len = htole16(nbits); 2178 ctx->me_N_len = htole16(nbits); 2179 2180#ifdef UBSEC_DEBUG 2181 if (ubsec_debug) { 2182 ubsec_dump_mcr(mcr); 2183 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2184 } 2185#endif 2186 2187 /* 2188 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2189 * everything else. 2190 */ 2191 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, BUS_DMASYNC_PREWRITE); 2192 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, BUS_DMASYNC_PREWRITE); 2193 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, BUS_DMASYNC_PREREAD); 2194 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, BUS_DMASYNC_PREWRITE); 2195 2196 /* Enqueue and we're done... */ 2197 UBSEC_LOCK(sc); 2198 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2199 ubsec_feed2(sc); 2200 ubsecstats.hst_modexp++; 2201 UBSEC_UNLOCK(sc); 2202 2203 return (0); 2204 2205errout: 2206 if (me != NULL) { 2207 if (me->me_q.q_mcr.dma_map != NULL) 2208 ubsec_dma_free(sc, &me->me_q.q_mcr); 2209 if (me->me_q.q_ctx.dma_map != NULL) { 2210 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2211 ubsec_dma_free(sc, &me->me_q.q_ctx); 2212 } 2213 if (me->me_M.dma_map != NULL) { 2214 bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2215 ubsec_dma_free(sc, &me->me_M); 2216 } 2217 if (me->me_E.dma_map != NULL) { 2218 bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2219 ubsec_dma_free(sc, &me->me_E); 2220 } 2221 if (me->me_C.dma_map != NULL) { 2222 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2223 ubsec_dma_free(sc, &me->me_C); 2224 } 2225 if (me->me_epb.dma_map != NULL) 2226 ubsec_dma_free(sc, &me->me_epb); 2227 free(me, M_DEVBUF); 2228 } 2229 krp->krp_status = err; 2230 crypto_kdone(krp); 2231 return (0); 2232} 2233 2234/* 2235 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization) 2236 */ 2237int 2238ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2239{ 2240 struct ubsec_q2_modexp *me; 2241 struct ubsec_mcr *mcr; 2242 struct ubsec_ctx_modexp *ctx; 2243 struct ubsec_pktbuf *epb; 2244 int err = 0; 2245 u_int nbits, normbits, mbits, shiftbits, ebits; 2246 2247 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2248 if (me == NULL) { 2249 err = ENOMEM; 2250 goto errout; 2251 } 2252 bzero(me, sizeof *me); 2253 me->me_krp = krp; 2254 me->me_q.q_type = UBS_CTXOP_MODEXP; 2255 2256 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2257 if (nbits <= 512) 2258 normbits = 512; 2259 else if (nbits <= 768) 2260 normbits = 768; 2261 else if (nbits <= 1024) 2262 normbits = 1024; 2263 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2264 normbits = 1536; 2265 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2266 normbits = 2048; 2267 else { 2268 err = E2BIG; 2269 goto errout; 2270 } 2271 2272 shiftbits = normbits - nbits; 2273 2274 /* XXX ??? */ 2275 me->me_modbits = nbits; 2276 me->me_shiftbits = shiftbits; 2277 me->me_normbits = normbits; 2278 2279 /* Sanity check: result bits must be >= true modulus bits. */ 2280 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2281 err = ERANGE; 2282 goto errout; 2283 } 2284 2285 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2286 &me->me_q.q_mcr, 0)) { 2287 err = ENOMEM; 2288 goto errout; 2289 } 2290 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2291 2292 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2293 &me->me_q.q_ctx, 0)) { 2294 err = ENOMEM; 2295 goto errout; 2296 } 2297 2298 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2299 if (mbits > nbits) { 2300 err = E2BIG; 2301 goto errout; 2302 } 2303 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2304 err = ENOMEM; 2305 goto errout; 2306 } 2307 bzero(me->me_M.dma_vaddr, normbits / 8); 2308 bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p, 2309 me->me_M.dma_vaddr, (mbits + 7) / 8); 2310 2311 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2312 err = ENOMEM; 2313 goto errout; 2314 } 2315 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2316 2317 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2318 if (ebits > nbits) { 2319 err = E2BIG; 2320 goto errout; 2321 } 2322 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2323 err = ENOMEM; 2324 goto errout; 2325 } 2326 bzero(me->me_E.dma_vaddr, normbits / 8); 2327 bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p, 2328 me->me_E.dma_vaddr, (ebits + 7) / 8); 2329 2330 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2331 &me->me_epb, 0)) { 2332 err = ENOMEM; 2333 goto errout; 2334 } 2335 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2336 epb->pb_addr = htole32(me->me_E.dma_paddr); 2337 epb->pb_next = 0; 2338 epb->pb_len = htole32((ebits + 7) / 8); 2339 2340#ifdef UBSEC_DEBUG 2341 printf("Epb "); 2342 ubsec_dump_pb(epb); 2343#endif 2344 2345 mcr->mcr_pkts = htole16(1); 2346 mcr->mcr_flags = 0; 2347 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2348 mcr->mcr_reserved = 0; 2349 mcr->mcr_pktlen = 0; 2350 2351 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2352 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2353 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2354 2355 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2356 mcr->mcr_opktbuf.pb_next = 0; 2357 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2358 2359#ifdef DIAGNOSTIC 2360 /* Misaligned output buffer will hang the chip. */ 2361 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2362 panic("%s: modexp invalid addr 0x%x\n", 2363 device_get_nameunit(sc->sc_dev), 2364 letoh32(mcr->mcr_opktbuf.pb_addr)); 2365 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2366 panic("%s: modexp invalid len 0x%x\n", 2367 device_get_nameunit(sc->sc_dev), 2368 letoh32(mcr->mcr_opktbuf.pb_len)); 2369#endif 2370 2371 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2372 bzero(ctx, sizeof(*ctx)); 2373 bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N, 2374 (nbits + 7) / 8); 2375 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2376 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2377 ctx->me_E_len = htole16(ebits); 2378 ctx->me_N_len = htole16(nbits); 2379 2380#ifdef UBSEC_DEBUG 2381 if (ubsec_debug) { 2382 ubsec_dump_mcr(mcr); 2383 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2384 } 2385#endif 2386 2387 /* 2388 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2389 * everything else. 2390 */ 2391 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, BUS_DMASYNC_PREWRITE); 2392 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, BUS_DMASYNC_PREWRITE); 2393 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, BUS_DMASYNC_PREREAD); 2394 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, BUS_DMASYNC_PREWRITE); 2395 2396 /* Enqueue and we're done... */ 2397 UBSEC_LOCK(sc); 2398 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2399 ubsec_feed2(sc); 2400 UBSEC_UNLOCK(sc); 2401 2402 return (0); 2403 2404errout: 2405 if (me != NULL) { 2406 if (me->me_q.q_mcr.dma_map != NULL) 2407 ubsec_dma_free(sc, &me->me_q.q_mcr); 2408 if (me->me_q.q_ctx.dma_map != NULL) { 2409 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2410 ubsec_dma_free(sc, &me->me_q.q_ctx); 2411 } 2412 if (me->me_M.dma_map != NULL) { 2413 bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2414 ubsec_dma_free(sc, &me->me_M); 2415 } 2416 if (me->me_E.dma_map != NULL) { 2417 bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2418 ubsec_dma_free(sc, &me->me_E); 2419 } 2420 if (me->me_C.dma_map != NULL) { 2421 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2422 ubsec_dma_free(sc, &me->me_C); 2423 } 2424 if (me->me_epb.dma_map != NULL) 2425 ubsec_dma_free(sc, &me->me_epb); 2426 free(me, M_DEVBUF); 2427 } 2428 krp->krp_status = err; 2429 crypto_kdone(krp); 2430 return (0); 2431} 2432 2433static int 2434ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2435{ 2436 struct ubsec_q2_rsapriv *rp = NULL; 2437 struct ubsec_mcr *mcr; 2438 struct ubsec_ctx_rsapriv *ctx; 2439 int err = 0; 2440 u_int padlen, msglen; 2441 2442 msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]); 2443 padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]); 2444 if (msglen > padlen) 2445 padlen = msglen; 2446 2447 if (padlen <= 256) 2448 padlen = 256; 2449 else if (padlen <= 384) 2450 padlen = 384; 2451 else if (padlen <= 512) 2452 padlen = 512; 2453 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768) 2454 padlen = 768; 2455 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024) 2456 padlen = 1024; 2457 else { 2458 err = E2BIG; 2459 goto errout; 2460 } 2461 2462 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) { 2463 err = E2BIG; 2464 goto errout; 2465 } 2466 2467 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) { 2468 err = E2BIG; 2469 goto errout; 2470 } 2471 2472 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) { 2473 err = E2BIG; 2474 goto errout; 2475 } 2476 2477 rp = (struct ubsec_q2_rsapriv *)malloc(sizeof *rp, M_DEVBUF, M_NOWAIT); 2478 if (rp == NULL) 2479 return (ENOMEM); 2480 bzero(rp, sizeof *rp); 2481 rp->rpr_krp = krp; 2482 rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV; 2483 2484 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2485 &rp->rpr_q.q_mcr, 0)) { 2486 err = ENOMEM; 2487 goto errout; 2488 } 2489 mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr; 2490 2491 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv), 2492 &rp->rpr_q.q_ctx, 0)) { 2493 err = ENOMEM; 2494 goto errout; 2495 } 2496 ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr; 2497 bzero(ctx, sizeof *ctx); 2498 2499 /* Copy in p */ 2500 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p, 2501 &ctx->rpr_buf[0 * (padlen / 8)], 2502 (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8); 2503 2504 /* Copy in q */ 2505 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p, 2506 &ctx->rpr_buf[1 * (padlen / 8)], 2507 (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8); 2508 2509 /* Copy in dp */ 2510 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p, 2511 &ctx->rpr_buf[2 * (padlen / 8)], 2512 (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8); 2513 2514 /* Copy in dq */ 2515 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p, 2516 &ctx->rpr_buf[3 * (padlen / 8)], 2517 (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8); 2518 2519 /* Copy in pinv */ 2520 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p, 2521 &ctx->rpr_buf[4 * (padlen / 8)], 2522 (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8); 2523 2524 msglen = padlen * 2; 2525 2526 /* Copy in input message (aligned buffer/length). */ 2527 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) { 2528 /* Is this likely? */ 2529 err = E2BIG; 2530 goto errout; 2531 } 2532 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) { 2533 err = ENOMEM; 2534 goto errout; 2535 } 2536 bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8); 2537 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p, 2538 rp->rpr_msgin.dma_vaddr, 2539 (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8); 2540 2541 /* Prepare space for output message (aligned buffer/length). */ 2542 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) { 2543 /* Is this likely? */ 2544 err = E2BIG; 2545 goto errout; 2546 } 2547 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) { 2548 err = ENOMEM; 2549 goto errout; 2550 } 2551 bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8); 2552 2553 mcr->mcr_pkts = htole16(1); 2554 mcr->mcr_flags = 0; 2555 mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr); 2556 mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr); 2557 mcr->mcr_ipktbuf.pb_next = 0; 2558 mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size); 2559 mcr->mcr_reserved = 0; 2560 mcr->mcr_pktlen = htole16(msglen); 2561 mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr); 2562 mcr->mcr_opktbuf.pb_next = 0; 2563 mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size); 2564 2565#ifdef DIAGNOSTIC 2566 if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) { 2567 panic("%s: rsapriv: invalid msgin %x(0x%x)", 2568 device_get_nameunit(sc->sc_dev), 2569 rp->rpr_msgin.dma_paddr, rp->rpr_msgin.dma_size); 2570 } 2571 if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) { 2572 panic("%s: rsapriv: invalid msgout %x(0x%x)", 2573 device_get_nameunit(sc->sc_dev), 2574 rp->rpr_msgout.dma_paddr, rp->rpr_msgout.dma_size); 2575 } 2576#endif 2577 2578 ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8)); 2579 ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV); 2580 ctx->rpr_q_len = htole16(padlen); 2581 ctx->rpr_p_len = htole16(padlen); 2582 2583 /* 2584 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2585 * everything else. 2586 */ 2587 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 2588 BUS_DMASYNC_PREWRITE); 2589 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 2590 BUS_DMASYNC_PREREAD); 2591 2592 /* Enqueue and we're done... */ 2593 UBSEC_LOCK(sc); 2594 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next); 2595 ubsec_feed2(sc); 2596 ubsecstats.hst_modexpcrt++; 2597 UBSEC_UNLOCK(sc); 2598 return (0); 2599 2600errout: 2601 if (rp != NULL) { 2602 if (rp->rpr_q.q_mcr.dma_map != NULL) 2603 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2604 if (rp->rpr_msgin.dma_map != NULL) { 2605 bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 2606 ubsec_dma_free(sc, &rp->rpr_msgin); 2607 } 2608 if (rp->rpr_msgout.dma_map != NULL) { 2609 bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 2610 ubsec_dma_free(sc, &rp->rpr_msgout); 2611 } 2612 free(rp, M_DEVBUF); 2613 } 2614 krp->krp_status = err; 2615 crypto_kdone(krp); 2616 return (0); 2617} 2618 2619#ifdef UBSEC_DEBUG 2620static void 2621ubsec_dump_pb(volatile struct ubsec_pktbuf *pb) 2622{ 2623 printf("addr 0x%x (0x%x) next 0x%x\n", 2624 pb->pb_addr, pb->pb_len, pb->pb_next); 2625} 2626 2627static void 2628ubsec_dump_ctx2(struct ubsec_ctx_keyop *c) 2629{ 2630 printf("CTX (0x%x):\n", c->ctx_len); 2631 switch (letoh16(c->ctx_op)) { 2632 case UBS_CTXOP_RNGBYPASS: 2633 case UBS_CTXOP_RNGSHA1: 2634 break; 2635 case UBS_CTXOP_MODEXP: 2636 { 2637 struct ubsec_ctx_modexp *cx = (void *)c; 2638 int i, len; 2639 2640 printf(" Elen %u, Nlen %u\n", 2641 letoh16(cx->me_E_len), letoh16(cx->me_N_len)); 2642 len = (cx->me_N_len + 7)/8; 2643 for (i = 0; i < len; i++) 2644 printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]); 2645 printf("\n"); 2646 break; 2647 } 2648 default: 2649 printf("unknown context: %x\n", c->ctx_op); 2650 } 2651 printf("END CTX\n"); 2652} 2653 2654static void 2655ubsec_dump_mcr(struct ubsec_mcr *mcr) 2656{ 2657 volatile struct ubsec_mcr_add *ma; 2658 int i; 2659 2660 printf("MCR:\n"); 2661 printf(" pkts: %u, flags 0x%x\n", 2662 letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags)); 2663 ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp; 2664 for (i = 0; i < letoh16(mcr->mcr_pkts); i++) { 2665 printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i, 2666 letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen), 2667 letoh16(ma->mcr_reserved)); 2668 printf(" %d: ipkt ", i); 2669 ubsec_dump_pb(&ma->mcr_ipktbuf); 2670 printf(" %d: opkt ", i); 2671 ubsec_dump_pb(&ma->mcr_opktbuf); 2672 ma++; 2673 } 2674 printf("END MCR\n"); 2675} 2676#endif /* UBSEC_DEBUG */ 2677 2678/* 2679 * Return the number of significant bits of a big number. 2680 */ 2681static int 2682ubsec_ksigbits(struct crparam *cr) 2683{ 2684 u_int plen = (cr->crp_nbits + 7) / 8; 2685 int i, sig = plen * 8; 2686 u_int8_t c, *p = cr->crp_p; 2687 2688 for (i = plen - 1; i >= 0; i--) { 2689 c = p[i]; 2690 if (c != 0) { 2691 while ((c & 0x80) == 0) { 2692 sig--; 2693 c <<= 1; 2694 } 2695 break; 2696 } 2697 sig -= 8; 2698 } 2699 return (sig); 2700} 2701 2702static void 2703ubsec_kshift_r( 2704 u_int shiftbits, 2705 u_int8_t *src, u_int srcbits, 2706 u_int8_t *dst, u_int dstbits) 2707{ 2708 u_int slen, dlen; 2709 int i, si, di, n; 2710 2711 slen = (srcbits + 7) / 8; 2712 dlen = (dstbits + 7) / 8; 2713 2714 for (i = 0; i < slen; i++) 2715 dst[i] = src[i]; 2716 for (i = 0; i < dlen - slen; i++) 2717 dst[slen + i] = 0; 2718 2719 n = shiftbits / 8; 2720 if (n != 0) { 2721 si = dlen - n - 1; 2722 di = dlen - 1; 2723 while (si >= 0) 2724 dst[di--] = dst[si--]; 2725 while (di >= 0) 2726 dst[di--] = 0; 2727 } 2728 2729 n = shiftbits % 8; 2730 if (n != 0) { 2731 for (i = dlen - 1; i > 0; i--) 2732 dst[i] = (dst[i] << n) | 2733 (dst[i - 1] >> (8 - n)); 2734 dst[0] = dst[0] << n; 2735 } 2736} 2737 2738static void 2739ubsec_kshift_l( 2740 u_int shiftbits, 2741 u_int8_t *src, u_int srcbits, 2742 u_int8_t *dst, u_int dstbits) 2743{ 2744 int slen, dlen, i, n; 2745 2746 slen = (srcbits + 7) / 8; 2747 dlen = (dstbits + 7) / 8; 2748 2749 n = shiftbits / 8; 2750 for (i = 0; i < slen; i++) 2751 dst[i] = src[i + n]; 2752 for (i = 0; i < dlen - slen; i++) 2753 dst[slen + i] = 0; 2754 2755 n = shiftbits % 8; 2756 if (n != 0) { 2757 for (i = 0; i < (dlen - 1); i++) 2758 dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n)); 2759 dst[dlen - 1] = dst[dlen - 1] >> n; 2760 } 2761} 2762