ubsec.c revision 267448
1104630Ssam/* $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $ */ 2104478Ssam 3139749Simp/*- 4104478Ssam * Copyright (c) 2000 Jason L. Wright (jason@thought.net) 5104478Ssam * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org) 6104478Ssam * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com) 7162969Sjhb * 8104478Ssam * All rights reserved. 9104478Ssam * 10104478Ssam * Redistribution and use in source and binary forms, with or without 11104478Ssam * modification, are permitted provided that the following conditions 12104478Ssam * are met: 13104478Ssam * 1. Redistributions of source code must retain the above copyright 14104478Ssam * notice, this list of conditions and the following disclaimer. 15104478Ssam * 2. Redistributions in binary form must reproduce the above copyright 16104478Ssam * notice, this list of conditions and the following disclaimer in the 17104478Ssam * documentation and/or other materials provided with the distribution. 18104478Ssam * 3. All advertising materials mentioning features or use of this software 19104478Ssam * must display the following acknowledgement: 20104478Ssam * This product includes software developed by Jason L. Wright 21104478Ssam * 4. The name of the author may not be used to endorse or promote products 22104478Ssam * derived from this software without specific prior written permission. 23104478Ssam * 24104478Ssam * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 25104478Ssam * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 26104478Ssam * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 27104478Ssam * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 28104478Ssam * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 29104478Ssam * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 30104478Ssam * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31104478Ssam * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 32104478Ssam * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 33104478Ssam * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34104478Ssam * POSSIBILITY OF SUCH DAMAGE. 35104478Ssam * 36104478Ssam * Effort sponsored in part by the Defense Advanced Research Projects 37104478Ssam * Agency (DARPA) and Air Force Research Laboratory, Air Force 38104478Ssam * Materiel Command, USAF, under agreement number F30602-01-2-0537. 39104478Ssam */ 40104478Ssam 41119418Sobrien#include <sys/cdefs.h> 42119418Sobrien__FBSDID("$FreeBSD: head/sys/dev/ubsec/ubsec.c 267448 2014-06-13 19:34:34Z jhb $"); 43119418Sobrien 44104478Ssam/* 45104478Ssam * uBsec 5[56]01, 58xx hardware crypto accelerator 46104478Ssam */ 47104478Ssam 48112124Ssam#include "opt_ubsec.h" 49112124Ssam 50104478Ssam#include <sys/param.h> 51104478Ssam#include <sys/systm.h> 52104478Ssam#include <sys/proc.h> 53104478Ssam#include <sys/errno.h> 54104478Ssam#include <sys/malloc.h> 55104478Ssam#include <sys/kernel.h> 56129879Sphk#include <sys/module.h> 57104478Ssam#include <sys/mbuf.h> 58104478Ssam#include <sys/lock.h> 59104478Ssam#include <sys/mutex.h> 60104478Ssam#include <sys/sysctl.h> 61104478Ssam#include <sys/endian.h> 62104478Ssam 63104478Ssam#include <vm/vm.h> 64104478Ssam#include <vm/pmap.h> 65104478Ssam 66104478Ssam#include <machine/bus.h> 67104478Ssam#include <machine/resource.h> 68104478Ssam#include <sys/bus.h> 69104478Ssam#include <sys/rman.h> 70104478Ssam 71104478Ssam#include <crypto/sha1.h> 72104478Ssam#include <opencrypto/cryptodev.h> 73104478Ssam#include <opencrypto/cryptosoft.h> 74104478Ssam#include <sys/md5.h> 75104478Ssam#include <sys/random.h> 76167755Ssam#include <sys/kobj.h> 77104478Ssam 78167755Ssam#include "cryptodev_if.h" 79167755Ssam 80119287Simp#include <dev/pci/pcivar.h> 81119287Simp#include <dev/pci/pcireg.h> 82104478Ssam 83104478Ssam/* grr, #defines for gratuitous incompatibility in queue.h */ 84104478Ssam#define SIMPLEQ_HEAD STAILQ_HEAD 85104478Ssam#define SIMPLEQ_ENTRY STAILQ_ENTRY 86104478Ssam#define SIMPLEQ_INIT STAILQ_INIT 87104478Ssam#define SIMPLEQ_INSERT_TAIL STAILQ_INSERT_TAIL 88104478Ssam#define SIMPLEQ_EMPTY STAILQ_EMPTY 89104478Ssam#define SIMPLEQ_FIRST STAILQ_FIRST 90163648Sru#define SIMPLEQ_REMOVE_HEAD STAILQ_REMOVE_HEAD 91111416Ssam#define SIMPLEQ_FOREACH STAILQ_FOREACH 92104478Ssam/* ditto for endian.h */ 93104478Ssam#define letoh16(x) le16toh(x) 94104478Ssam#define letoh32(x) le32toh(x) 95104478Ssam 96112124Ssam#ifdef UBSEC_RNDTEST 97112124Ssam#include <dev/rndtest/rndtest.h> 98112124Ssam#endif 99104478Ssam#include <dev/ubsec/ubsecreg.h> 100104478Ssam#include <dev/ubsec/ubsecvar.h> 101104478Ssam 102104478Ssam/* 103104478Ssam * Prototypes and count for the pci_device structure 104104478Ssam */ 105104478Ssamstatic int ubsec_probe(device_t); 106104478Ssamstatic int ubsec_attach(device_t); 107104478Ssamstatic int ubsec_detach(device_t); 108104478Ssamstatic int ubsec_suspend(device_t); 109104478Ssamstatic int ubsec_resume(device_t); 110194023Savgstatic int ubsec_shutdown(device_t); 111104478Ssam 112167755Ssamstatic int ubsec_newsession(device_t, u_int32_t *, struct cryptoini *); 113167755Ssamstatic int ubsec_freesession(device_t, u_int64_t); 114167755Ssamstatic int ubsec_process(device_t, struct cryptop *, int); 115167755Ssamstatic int ubsec_kprocess(device_t, struct cryptkop *, int); 116167755Ssam 117104478Ssamstatic device_method_t ubsec_methods[] = { 118104478Ssam /* Device interface */ 119104478Ssam DEVMETHOD(device_probe, ubsec_probe), 120104478Ssam DEVMETHOD(device_attach, ubsec_attach), 121104478Ssam DEVMETHOD(device_detach, ubsec_detach), 122104478Ssam DEVMETHOD(device_suspend, ubsec_suspend), 123104478Ssam DEVMETHOD(device_resume, ubsec_resume), 124104478Ssam DEVMETHOD(device_shutdown, ubsec_shutdown), 125104478Ssam 126167755Ssam /* crypto device methods */ 127167755Ssam DEVMETHOD(cryptodev_newsession, ubsec_newsession), 128167755Ssam DEVMETHOD(cryptodev_freesession,ubsec_freesession), 129167755Ssam DEVMETHOD(cryptodev_process, ubsec_process), 130167755Ssam DEVMETHOD(cryptodev_kprocess, ubsec_kprocess), 131167755Ssam 132227843Smarius DEVMETHOD_END 133104478Ssam}; 134104478Ssamstatic driver_t ubsec_driver = { 135104478Ssam "ubsec", 136104478Ssam ubsec_methods, 137104478Ssam sizeof (struct ubsec_softc) 138104478Ssam}; 139104478Ssamstatic devclass_t ubsec_devclass; 140104478Ssam 141104478SsamDRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0); 142105251SmarkmMODULE_DEPEND(ubsec, crypto, 1, 1, 1); 143112124Ssam#ifdef UBSEC_RNDTEST 144112124SsamMODULE_DEPEND(ubsec, rndtest, 1, 1, 1); 145112124Ssam#endif 146104478Ssam 147104478Ssamstatic void ubsec_intr(void *); 148104478Ssamstatic void ubsec_callback(struct ubsec_softc *, struct ubsec_q *); 149111416Ssamstatic void ubsec_feed(struct ubsec_softc *); 150104478Ssamstatic void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int); 151104478Ssamstatic void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *); 152104478Ssamstatic int ubsec_feed2(struct ubsec_softc *); 153104478Ssamstatic void ubsec_rng(void *); 154104478Ssamstatic int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t, 155104478Ssam struct ubsec_dma_alloc *, int); 156108823Ssam#define ubsec_dma_sync(_dma, _flags) \ 157108823Ssam bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags)) 158104478Ssamstatic void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *); 159104478Ssamstatic int ubsec_dmamap_aligned(struct ubsec_operand *op); 160104478Ssam 161104478Ssamstatic void ubsec_reset_board(struct ubsec_softc *sc); 162104478Ssamstatic void ubsec_init_board(struct ubsec_softc *sc); 163104478Ssamstatic void ubsec_init_pciregs(device_t dev); 164104478Ssamstatic void ubsec_totalreset(struct ubsec_softc *sc); 165104478Ssam 166104478Ssamstatic int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q); 167104478Ssam 168104630Ssamstatic int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int); 169104630Ssamstatic int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int); 170104478Ssamstatic int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int); 171104478Ssamstatic void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *); 172104478Ssamstatic int ubsec_ksigbits(struct crparam *); 173104478Ssamstatic void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 174104478Ssamstatic void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 175104478Ssam 176227309Sedstatic SYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0, 177227309Sed "Broadcom driver parameters"); 178109595Ssam 179104478Ssam#ifdef UBSEC_DEBUG 180104478Ssamstatic void ubsec_dump_pb(volatile struct ubsec_pktbuf *); 181104478Ssamstatic void ubsec_dump_mcr(struct ubsec_mcr *); 182104478Ssamstatic void ubsec_dump_ctx2(struct ubsec_ctx_keyop *); 183104478Ssam 184104478Ssamstatic int ubsec_debug = 0; 185109595SsamSYSCTL_INT(_hw_ubsec, OID_AUTO, debug, CTLFLAG_RW, &ubsec_debug, 186109595Ssam 0, "control debugging msgs"); 187104478Ssam#endif 188104478Ssam 189104478Ssam#define READ_REG(sc,r) \ 190104478Ssam bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r)) 191104478Ssam 192104478Ssam#define WRITE_REG(sc,reg,val) \ 193104478Ssam bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val) 194104478Ssam 195104478Ssam#define SWAP32(x) (x) = htole32(ntohl((x))) 196104478Ssam#define HTOLE32(x) (x) = htole32(x) 197104478Ssam 198104478Ssamstruct ubsec_stats ubsecstats; 199109595SsamSYSCTL_STRUCT(_hw_ubsec, OID_AUTO, stats, CTLFLAG_RD, &ubsecstats, 200109595Ssam ubsec_stats, "driver statistics"); 201104478Ssam 202105215Sphkstatic int 203104478Ssamubsec_probe(device_t dev) 204104478Ssam{ 205114105Ssam if (pci_get_vendor(dev) == PCI_VENDOR_SUN && 206114105Ssam (pci_get_device(dev) == PCI_PRODUCT_SUN_5821 || 207114105Ssam pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K)) 208142880Simp return (BUS_PROBE_DEFAULT); 209104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 210104478Ssam (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 || 211104478Ssam pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)) 212142880Simp return (BUS_PROBE_DEFAULT); 213104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 214111646Ssam (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5801 || 215111646Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 || 216111646Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 || 217104478Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 || 218104478Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 219110522Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 || 220191894Sphilip pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 || 221191894Sphilip pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5825 222110522Ssam )) 223142880Simp return (BUS_PROBE_DEFAULT); 224104478Ssam return (ENXIO); 225104478Ssam} 226104478Ssam 227104478Ssamstatic const char* 228104478Ssamubsec_partname(struct ubsec_softc *sc) 229104478Ssam{ 230104478Ssam /* XXX sprintf numbers when not decoded */ 231104478Ssam switch (pci_get_vendor(sc->sc_dev)) { 232104478Ssam case PCI_VENDOR_BROADCOM: 233104478Ssam switch (pci_get_device(sc->sc_dev)) { 234111646Ssam case PCI_PRODUCT_BROADCOM_5801: return "Broadcom 5801"; 235111646Ssam case PCI_PRODUCT_BROADCOM_5802: return "Broadcom 5802"; 236104478Ssam case PCI_PRODUCT_BROADCOM_5805: return "Broadcom 5805"; 237104478Ssam case PCI_PRODUCT_BROADCOM_5820: return "Broadcom 5820"; 238104478Ssam case PCI_PRODUCT_BROADCOM_5821: return "Broadcom 5821"; 239104478Ssam case PCI_PRODUCT_BROADCOM_5822: return "Broadcom 5822"; 240110522Ssam case PCI_PRODUCT_BROADCOM_5823: return "Broadcom 5823"; 241191894Sphilip case PCI_PRODUCT_BROADCOM_5825: return "Broadcom 5825"; 242104478Ssam } 243104478Ssam return "Broadcom unknown-part"; 244104478Ssam case PCI_VENDOR_BLUESTEEL: 245104478Ssam switch (pci_get_device(sc->sc_dev)) { 246104478Ssam case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601"; 247104478Ssam } 248104478Ssam return "Bluesteel unknown-part"; 249114105Ssam case PCI_VENDOR_SUN: 250114105Ssam switch (pci_get_device(sc->sc_dev)) { 251114105Ssam case PCI_PRODUCT_SUN_5821: return "Sun Crypto 5821"; 252114105Ssam case PCI_PRODUCT_SUN_SCA1K: return "Sun Crypto 1K"; 253114105Ssam } 254114105Ssam return "Sun unknown-part"; 255104478Ssam } 256104478Ssam return "Unknown-vendor unknown-part"; 257104478Ssam} 258104478Ssam 259112124Ssamstatic void 260112124Ssamdefault_harvest(struct rndtest_state *rsp, void *buf, u_int count) 261112124Ssam{ 262256377Smarkm random_harvest(buf, count, count*NBBY/2, RANDOM_PURE_UBSEC); 263112124Ssam} 264112124Ssam 265104478Ssamstatic int 266104478Ssamubsec_attach(device_t dev) 267104478Ssam{ 268104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 269104478Ssam struct ubsec_dma *dmap; 270254263Sscottl u_int32_t i; 271104478Ssam int rid; 272104478Ssam 273104478Ssam bzero(sc, sizeof (*sc)); 274104478Ssam sc->sc_dev = dev; 275104478Ssam 276104478Ssam SIMPLEQ_INIT(&sc->sc_queue); 277104478Ssam SIMPLEQ_INIT(&sc->sc_qchip); 278104478Ssam SIMPLEQ_INIT(&sc->sc_queue2); 279104478Ssam SIMPLEQ_INIT(&sc->sc_qchip2); 280104478Ssam SIMPLEQ_INIT(&sc->sc_q2free); 281104478Ssam 282104478Ssam /* XXX handle power management */ 283104478Ssam 284104478Ssam sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR; 285104478Ssam 286104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 287104478Ssam pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601) 288104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 289104478Ssam 290104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 291111646Ssam (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 || 292111646Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805)) 293104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 294104478Ssam 295104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 296104478Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820) 297104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 298104478Ssam UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 299104478Ssam 300114105Ssam if ((pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 301114105Ssam (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 302114105Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 || 303191894Sphilip pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 || 304191894Sphilip pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5825)) || 305114105Ssam (pci_get_vendor(dev) == PCI_VENDOR_SUN && 306114105Ssam (pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K || 307114105Ssam pci_get_device(dev) == PCI_PRODUCT_SUN_5821))) { 308104478Ssam /* NB: the 5821/5822 defines some additional status bits */ 309104478Ssam sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY | 310104478Ssam BS_STAT_MCR2_ALLEMPTY; 311104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 312104478Ssam UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 313104478Ssam } 314162969Sjhb 315254263Sscottl pci_enable_busmaster(dev); 316104478Ssam 317162969Sjhb /* 318104478Ssam * Setup memory-mapping of PCI registers. 319104478Ssam */ 320104478Ssam rid = BS_BAR; 321127135Snjl sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 322127135Snjl RF_ACTIVE); 323104478Ssam if (sc->sc_sr == NULL) { 324104478Ssam device_printf(dev, "cannot map register space\n"); 325104478Ssam goto bad; 326104478Ssam } 327104478Ssam sc->sc_st = rman_get_bustag(sc->sc_sr); 328104478Ssam sc->sc_sh = rman_get_bushandle(sc->sc_sr); 329104478Ssam 330104478Ssam /* 331104478Ssam * Arrange interrupt line. 332104478Ssam */ 333104478Ssam rid = 0; 334127135Snjl sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 335127135Snjl RF_SHAREABLE|RF_ACTIVE); 336104478Ssam if (sc->sc_irq == NULL) { 337104478Ssam device_printf(dev, "could not map interrupt\n"); 338108823Ssam goto bad1; 339104478Ssam } 340104478Ssam /* 341104478Ssam * NB: Network code assumes we are blocked with splimp() 342104478Ssam * so make sure the IRQ is mapped appropriately. 343104478Ssam */ 344115747Ssam if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE, 345166901Spiso NULL, ubsec_intr, sc, &sc->sc_ih)) { 346104478Ssam device_printf(dev, "could not establish interrupt\n"); 347108823Ssam goto bad2; 348104478Ssam } 349104478Ssam 350167755Ssam sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); 351104478Ssam if (sc->sc_cid < 0) { 352104478Ssam device_printf(dev, "could not get crypto driver id\n"); 353108823Ssam goto bad3; 354104478Ssam } 355104478Ssam 356104478Ssam /* 357104478Ssam * Setup DMA descriptor area. 358104478Ssam */ 359232874Sscottl if (bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 360104478Ssam 1, 0, /* alignment, bounds */ 361104478Ssam BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 362104478Ssam BUS_SPACE_MAXADDR, /* highaddr */ 363104478Ssam NULL, NULL, /* filter, filterarg */ 364108823Ssam 0x3ffff, /* maxsize */ 365104478Ssam UBS_MAX_SCATTER, /* nsegments */ 366108823Ssam 0xffff, /* maxsegsize */ 367104478Ssam BUS_DMA_ALLOCNOW, /* flags */ 368117126Sscottl NULL, NULL, /* lockfunc, lockarg */ 369104478Ssam &sc->sc_dmat)) { 370104478Ssam device_printf(dev, "cannot allocate DMA tag\n"); 371108823Ssam goto bad4; 372104478Ssam } 373104478Ssam SIMPLEQ_INIT(&sc->sc_freequeue); 374104478Ssam dmap = sc->sc_dmaa; 375104478Ssam for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) { 376104478Ssam struct ubsec_q *q; 377104478Ssam 378104478Ssam q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q), 379104478Ssam M_DEVBUF, M_NOWAIT); 380104478Ssam if (q == NULL) { 381104478Ssam device_printf(dev, "cannot allocate queue buffers\n"); 382104478Ssam break; 383104478Ssam } 384104478Ssam 385104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk), 386104478Ssam &dmap->d_alloc, 0)) { 387104478Ssam device_printf(dev, "cannot allocate dma buffers\n"); 388104478Ssam free(q, M_DEVBUF); 389104478Ssam break; 390104478Ssam } 391104478Ssam dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr; 392104478Ssam 393104478Ssam q->q_dma = dmap; 394104478Ssam sc->sc_queuea[i] = q; 395104478Ssam 396104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 397104478Ssam } 398115747Ssam mtx_init(&sc->sc_mcr1lock, device_get_nameunit(dev), 399115747Ssam "mcr1 operations", MTX_DEF); 400115747Ssam mtx_init(&sc->sc_freeqlock, device_get_nameunit(dev), 401115747Ssam "mcr1 free q", MTX_DEF); 402104478Ssam 403104478Ssam device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc)); 404104478Ssam 405167755Ssam crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); 406167755Ssam crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); 407167755Ssam crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 408167755Ssam crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 409104478Ssam 410104478Ssam /* 411104478Ssam * Reset Broadcom chip 412104478Ssam */ 413104478Ssam ubsec_reset_board(sc); 414104478Ssam 415104478Ssam /* 416104478Ssam * Init Broadcom specific PCI settings 417104478Ssam */ 418104478Ssam ubsec_init_pciregs(dev); 419104478Ssam 420104478Ssam /* 421104478Ssam * Init Broadcom chip 422104478Ssam */ 423104478Ssam ubsec_init_board(sc); 424104478Ssam 425104478Ssam#ifndef UBSEC_NO_RNG 426104478Ssam if (sc->sc_flags & UBS_FLAGS_RNG) { 427104478Ssam sc->sc_statmask |= BS_STAT_MCR2_DONE; 428112124Ssam#ifdef UBSEC_RNDTEST 429112124Ssam sc->sc_rndtest = rndtest_attach(dev); 430112124Ssam if (sc->sc_rndtest) 431112124Ssam sc->sc_harvest = rndtest_harvest; 432112124Ssam else 433112124Ssam sc->sc_harvest = default_harvest; 434112124Ssam#else 435112124Ssam sc->sc_harvest = default_harvest; 436112124Ssam#endif 437104478Ssam 438104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 439104478Ssam &sc->sc_rng.rng_q.q_mcr, 0)) 440104478Ssam goto skip_rng; 441104478Ssam 442104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass), 443104478Ssam &sc->sc_rng.rng_q.q_ctx, 0)) { 444104478Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 445104478Ssam goto skip_rng; 446104478Ssam } 447104478Ssam 448104478Ssam if (ubsec_dma_malloc(sc, sizeof(u_int32_t) * 449104478Ssam UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) { 450104478Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 451104478Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 452104478Ssam goto skip_rng; 453104478Ssam } 454104478Ssam 455104478Ssam if (hz >= 100) 456104478Ssam sc->sc_rnghz = hz / 100; 457104478Ssam else 458104478Ssam sc->sc_rnghz = 1; 459119137Ssam callout_init(&sc->sc_rngto, CALLOUT_MPSAFE); 460104478Ssam callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 461104478Ssamskip_rng: 462104478Ssam ; 463104478Ssam } 464104478Ssam#endif /* UBSEC_NO_RNG */ 465115747Ssam mtx_init(&sc->sc_mcr2lock, device_get_nameunit(dev), 466115747Ssam "mcr2 operations", MTX_DEF); 467104478Ssam 468104478Ssam if (sc->sc_flags & UBS_FLAGS_KEY) { 469104478Ssam sc->sc_statmask |= BS_STAT_MCR2_DONE; 470104478Ssam 471167755Ssam crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0); 472104630Ssam#if 0 473167755Ssam crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0); 474104630Ssam#endif 475104478Ssam } 476104478Ssam return (0); 477108823Ssambad4: 478108823Ssam crypto_unregister_all(sc->sc_cid); 479108823Ssambad3: 480108823Ssam bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 481108823Ssambad2: 482108823Ssam bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 483108823Ssambad1: 484108823Ssam bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 485104478Ssambad: 486104478Ssam return (ENXIO); 487104478Ssam} 488104478Ssam 489104478Ssam/* 490104478Ssam * Detach a device that successfully probed. 491104478Ssam */ 492104478Ssamstatic int 493104478Ssamubsec_detach(device_t dev) 494104478Ssam{ 495104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 496104478Ssam 497108823Ssam /* XXX wait/abort active ops */ 498108823Ssam 499115747Ssam /* disable interrupts */ 500115747Ssam WRITE_REG(sc, BS_CTRL, READ_REG(sc, BS_CTRL) &~ 501115747Ssam (BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR)); 502104478Ssam 503104478Ssam callout_stop(&sc->sc_rngto); 504104478Ssam 505104478Ssam crypto_unregister_all(sc->sc_cid); 506104478Ssam 507112124Ssam#ifdef UBSEC_RNDTEST 508112124Ssam if (sc->sc_rndtest) 509112124Ssam rndtest_detach(sc->sc_rndtest); 510112124Ssam#endif 511112124Ssam 512108823Ssam while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 513108823Ssam struct ubsec_q *q; 514108823Ssam 515108823Ssam q = SIMPLEQ_FIRST(&sc->sc_freequeue); 516163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next); 517108823Ssam ubsec_dma_free(sc, &q->q_dma->d_alloc); 518108823Ssam free(q, M_DEVBUF); 519108823Ssam } 520115747Ssam mtx_destroy(&sc->sc_mcr1lock); 521159224Spjd mtx_destroy(&sc->sc_freeqlock); 522108823Ssam#ifndef UBSEC_NO_RNG 523108823Ssam if (sc->sc_flags & UBS_FLAGS_RNG) { 524108823Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 525108823Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 526108823Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_buf); 527108823Ssam } 528108823Ssam#endif /* UBSEC_NO_RNG */ 529115747Ssam mtx_destroy(&sc->sc_mcr2lock); 530108823Ssam 531104478Ssam bus_generic_detach(dev); 532104478Ssam bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 533104478Ssam bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 534104478Ssam 535104478Ssam bus_dma_tag_destroy(sc->sc_dmat); 536104478Ssam bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 537104478Ssam 538104478Ssam return (0); 539104478Ssam} 540104478Ssam 541104478Ssam/* 542104478Ssam * Stop all chip i/o so that the kernel's probe routines don't 543104478Ssam * get confused by errant DMAs when rebooting. 544104478Ssam */ 545194023Savgstatic int 546104478Ssamubsec_shutdown(device_t dev) 547104478Ssam{ 548104478Ssam#ifdef notyet 549104478Ssam ubsec_stop(device_get_softc(dev)); 550104478Ssam#endif 551194023Savg return (0); 552104478Ssam} 553104478Ssam 554104478Ssam/* 555104478Ssam * Device suspend routine. 556104478Ssam */ 557104478Ssamstatic int 558104478Ssamubsec_suspend(device_t dev) 559104478Ssam{ 560104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 561104478Ssam 562104478Ssam#ifdef notyet 563104478Ssam /* XXX stop the device and save PCI settings */ 564104478Ssam#endif 565104478Ssam sc->sc_suspended = 1; 566104478Ssam 567104478Ssam return (0); 568104478Ssam} 569104478Ssam 570104478Ssamstatic int 571104478Ssamubsec_resume(device_t dev) 572104478Ssam{ 573104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 574104478Ssam 575104478Ssam#ifdef notyet 576104478Ssam /* XXX retore PCI settings and start the device */ 577104478Ssam#endif 578104478Ssam sc->sc_suspended = 0; 579104478Ssam return (0); 580104478Ssam} 581104478Ssam 582104478Ssam/* 583104478Ssam * UBSEC Interrupt routine 584104478Ssam */ 585104478Ssamstatic void 586104478Ssamubsec_intr(void *arg) 587104478Ssam{ 588104478Ssam struct ubsec_softc *sc = arg; 589104478Ssam volatile u_int32_t stat; 590104478Ssam struct ubsec_q *q; 591104478Ssam struct ubsec_dma *dmap; 592104478Ssam int npkts = 0, i; 593104478Ssam 594104478Ssam stat = READ_REG(sc, BS_STAT); 595104478Ssam stat &= sc->sc_statmask; 596115747Ssam if (stat == 0) 597104478Ssam return; 598104478Ssam 599104478Ssam WRITE_REG(sc, BS_STAT, stat); /* IACK */ 600104478Ssam 601104478Ssam /* 602104478Ssam * Check to see if we have any packets waiting for us 603104478Ssam */ 604104478Ssam if ((stat & BS_STAT_MCR1_DONE)) { 605115747Ssam mtx_lock(&sc->sc_mcr1lock); 606104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 607104478Ssam q = SIMPLEQ_FIRST(&sc->sc_qchip); 608104478Ssam dmap = q->q_dma; 609104478Ssam 610104478Ssam if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0) 611104478Ssam break; 612104478Ssam 613163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next); 614104478Ssam 615104478Ssam npkts = q->q_nstacked_mcrs; 616108471Ssam sc->sc_nqchip -= 1+npkts; 617104478Ssam /* 618104478Ssam * search for further sc_qchip ubsec_q's that share 619104478Ssam * the same MCR, and complete them too, they must be 620104478Ssam * at the top. 621104478Ssam */ 622104478Ssam for (i = 0; i < npkts; i++) { 623104478Ssam if(q->q_stacked_mcr[i]) { 624104478Ssam ubsec_callback(sc, q->q_stacked_mcr[i]); 625104478Ssam } else { 626104478Ssam break; 627104478Ssam } 628104478Ssam } 629104478Ssam ubsec_callback(sc, q); 630104478Ssam } 631104478Ssam /* 632104478Ssam * Don't send any more packet to chip if there has been 633104478Ssam * a DMAERR. 634104478Ssam */ 635104478Ssam if (!(stat & BS_STAT_DMAERR)) 636104478Ssam ubsec_feed(sc); 637115747Ssam mtx_unlock(&sc->sc_mcr1lock); 638104478Ssam } 639104478Ssam 640104478Ssam /* 641104478Ssam * Check to see if we have any key setups/rng's waiting for us 642104478Ssam */ 643104478Ssam if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) && 644104478Ssam (stat & BS_STAT_MCR2_DONE)) { 645104478Ssam struct ubsec_q2 *q2; 646104478Ssam struct ubsec_mcr *mcr; 647104478Ssam 648115747Ssam mtx_lock(&sc->sc_mcr2lock); 649104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) { 650104478Ssam q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 651104478Ssam 652108823Ssam ubsec_dma_sync(&q2->q_mcr, 653104478Ssam BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 654104478Ssam 655104478Ssam mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 656104478Ssam if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) { 657108823Ssam ubsec_dma_sync(&q2->q_mcr, 658104478Ssam BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 659104478Ssam break; 660104478Ssam } 661163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q_next); 662104478Ssam ubsec_callback2(sc, q2); 663104478Ssam /* 664104478Ssam * Don't send any more packet to chip if there has been 665104478Ssam * a DMAERR. 666104478Ssam */ 667104478Ssam if (!(stat & BS_STAT_DMAERR)) 668104478Ssam ubsec_feed2(sc); 669104478Ssam } 670115747Ssam mtx_unlock(&sc->sc_mcr2lock); 671104478Ssam } 672104478Ssam 673104478Ssam /* 674104478Ssam * Check to see if we got any DMA Error 675104478Ssam */ 676104478Ssam if (stat & BS_STAT_DMAERR) { 677104478Ssam#ifdef UBSEC_DEBUG 678104478Ssam if (ubsec_debug) { 679104478Ssam volatile u_int32_t a = READ_REG(sc, BS_ERR); 680104478Ssam 681104478Ssam printf("dmaerr %s@%08x\n", 682104478Ssam (a & BS_ERR_READ) ? "read" : "write", 683104478Ssam a & BS_ERR_ADDR); 684104478Ssam } 685104478Ssam#endif /* UBSEC_DEBUG */ 686104478Ssam ubsecstats.hst_dmaerr++; 687115747Ssam mtx_lock(&sc->sc_mcr1lock); 688104478Ssam ubsec_totalreset(sc); 689104478Ssam ubsec_feed(sc); 690115747Ssam mtx_unlock(&sc->sc_mcr1lock); 691104478Ssam } 692104478Ssam 693104478Ssam if (sc->sc_needwakeup) { /* XXX check high watermark */ 694158828Spjd int wakeup; 695158828Spjd 696158828Spjd mtx_lock(&sc->sc_freeqlock); 697158828Spjd wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ); 698104478Ssam#ifdef UBSEC_DEBUG 699104478Ssam if (ubsec_debug) 700104478Ssam device_printf(sc->sc_dev, "wakeup crypto (%x)\n", 701104478Ssam sc->sc_needwakeup); 702104478Ssam#endif /* UBSEC_DEBUG */ 703104478Ssam sc->sc_needwakeup &= ~wakeup; 704158828Spjd mtx_unlock(&sc->sc_freeqlock); 705104478Ssam crypto_unblock(sc->sc_cid, wakeup); 706104478Ssam } 707104478Ssam} 708104478Ssam 709104478Ssam/* 710104478Ssam * ubsec_feed() - aggregate and post requests to chip 711104478Ssam */ 712111416Ssamstatic void 713104478Ssamubsec_feed(struct ubsec_softc *sc) 714104478Ssam{ 715104478Ssam struct ubsec_q *q, *q2; 716104478Ssam int npkts, i; 717104478Ssam void *v; 718104478Ssam u_int32_t stat; 719104478Ssam 720108471Ssam /* 721108471Ssam * Decide how many ops to combine in a single MCR. We cannot 722108471Ssam * aggregate more than UBS_MAX_AGGR because this is the number 723111416Ssam * of slots defined in the data structure. Note that 724111416Ssam * aggregation only happens if ops are marked batch'able. 725108471Ssam * Aggregating ops reduces the number of interrupts to the host 726108471Ssam * but also (potentially) increases the latency for processing 727108471Ssam * completed ops as we only get an interrupt when all aggregated 728108471Ssam * ops have completed. 729108471Ssam */ 730111416Ssam if (sc->sc_nqueue == 0) 731111416Ssam return; 732111416Ssam if (sc->sc_nqueue > 1) { 733111416Ssam npkts = 0; 734111416Ssam SIMPLEQ_FOREACH(q, &sc->sc_queue, q_next) { 735111416Ssam npkts++; 736111416Ssam if ((q->q_crp->crp_flags & CRYPTO_F_BATCH) == 0) 737111416Ssam break; 738111416Ssam } 739111416Ssam } else 740111416Ssam npkts = 1; 741111416Ssam /* 742111416Ssam * Check device status before going any further. 743111416Ssam */ 744104478Ssam if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 745108471Ssam if (stat & BS_STAT_DMAERR) { 746104478Ssam ubsec_totalreset(sc); 747104478Ssam ubsecstats.hst_dmaerr++; 748108471Ssam } else 749108471Ssam ubsecstats.hst_mcr1full++; 750111416Ssam return; 751104478Ssam } 752111416Ssam if (sc->sc_nqueue > ubsecstats.hst_maxqueue) 753111416Ssam ubsecstats.hst_maxqueue = sc->sc_nqueue; 754111416Ssam if (npkts > UBS_MAX_AGGR) 755111416Ssam npkts = UBS_MAX_AGGR; 756111416Ssam if (npkts < 2) /* special case 1 op */ 757111416Ssam goto feed1; 758104478Ssam 759111416Ssam ubsecstats.hst_totbatch += npkts-1; 760104478Ssam#ifdef UBSEC_DEBUG 761104478Ssam if (ubsec_debug) 762104478Ssam printf("merging %d records\n", npkts); 763104478Ssam#endif /* UBSEC_DEBUG */ 764104478Ssam 765104478Ssam q = SIMPLEQ_FIRST(&sc->sc_queue); 766163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next); 767104478Ssam --sc->sc_nqueue; 768104478Ssam 769104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE); 770104478Ssam if (q->q_dst_map != NULL) 771104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD); 772104478Ssam 773104478Ssam q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */ 774104478Ssam 775104478Ssam for (i = 0; i < q->q_nstacked_mcrs; i++) { 776104478Ssam q2 = SIMPLEQ_FIRST(&sc->sc_queue); 777104478Ssam bus_dmamap_sync(sc->sc_dmat, q2->q_src_map, 778104478Ssam BUS_DMASYNC_PREWRITE); 779104478Ssam if (q2->q_dst_map != NULL) 780104478Ssam bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map, 781104478Ssam BUS_DMASYNC_PREREAD); 782163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next); 783104478Ssam --sc->sc_nqueue; 784104478Ssam 785104478Ssam v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) - 786104478Ssam sizeof(struct ubsec_mcr_add)); 787104478Ssam bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add)); 788104478Ssam q->q_stacked_mcr[i] = q2; 789104478Ssam } 790104478Ssam q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts); 791104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 792108471Ssam sc->sc_nqchip += npkts; 793108471Ssam if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 794108471Ssam ubsecstats.hst_maxqchip = sc->sc_nqchip; 795108823Ssam ubsec_dma_sync(&q->q_dma->d_alloc, 796104478Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 797104478Ssam WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 798104478Ssam offsetof(struct ubsec_dmachunk, d_mcr)); 799111416Ssam return; 800104478Ssamfeed1: 801111416Ssam q = SIMPLEQ_FIRST(&sc->sc_queue); 802104478Ssam 803111416Ssam bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE); 804111416Ssam if (q->q_dst_map != NULL) 805111416Ssam bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD); 806111416Ssam ubsec_dma_sync(&q->q_dma->d_alloc, 807111416Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 808104478Ssam 809111416Ssam WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 810111416Ssam offsetof(struct ubsec_dmachunk, d_mcr)); 811104478Ssam#ifdef UBSEC_DEBUG 812111416Ssam if (ubsec_debug) 813111416Ssam printf("feed1: q->chip %p %08x stat %08x\n", 814111416Ssam q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr), 815111416Ssam stat); 816104478Ssam#endif /* UBSEC_DEBUG */ 817163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next); 818111416Ssam --sc->sc_nqueue; 819111416Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 820111416Ssam sc->sc_nqchip++; 821108471Ssam if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 822108471Ssam ubsecstats.hst_maxqchip = sc->sc_nqchip; 823111416Ssam return; 824104478Ssam} 825104478Ssam 826159225Spjdstatic void 827159225Spjdubsec_setup_enckey(struct ubsec_session *ses, int algo, caddr_t key) 828159225Spjd{ 829159225Spjd 830159225Spjd /* Go ahead and compute key in ubsec's byte order */ 831159225Spjd if (algo == CRYPTO_DES_CBC) { 832159225Spjd bcopy(key, &ses->ses_deskey[0], 8); 833159225Spjd bcopy(key, &ses->ses_deskey[2], 8); 834159225Spjd bcopy(key, &ses->ses_deskey[4], 8); 835159225Spjd } else 836159225Spjd bcopy(key, ses->ses_deskey, 24); 837159225Spjd 838159225Spjd SWAP32(ses->ses_deskey[0]); 839159225Spjd SWAP32(ses->ses_deskey[1]); 840159225Spjd SWAP32(ses->ses_deskey[2]); 841159225Spjd SWAP32(ses->ses_deskey[3]); 842159225Spjd SWAP32(ses->ses_deskey[4]); 843159225Spjd SWAP32(ses->ses_deskey[5]); 844159225Spjd} 845159225Spjd 846159225Spjdstatic void 847159225Spjdubsec_setup_mackey(struct ubsec_session *ses, int algo, caddr_t key, int klen) 848159225Spjd{ 849159225Spjd MD5_CTX md5ctx; 850159225Spjd SHA1_CTX sha1ctx; 851159225Spjd int i; 852159225Spjd 853159225Spjd for (i = 0; i < klen; i++) 854159225Spjd key[i] ^= HMAC_IPAD_VAL; 855159225Spjd 856159225Spjd if (algo == CRYPTO_MD5_HMAC) { 857159225Spjd MD5Init(&md5ctx); 858159225Spjd MD5Update(&md5ctx, key, klen); 859159232Spjd MD5Update(&md5ctx, hmac_ipad_buffer, MD5_HMAC_BLOCK_LEN - klen); 860159225Spjd bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state)); 861159225Spjd } else { 862159225Spjd SHA1Init(&sha1ctx); 863159225Spjd SHA1Update(&sha1ctx, key, klen); 864159232Spjd SHA1Update(&sha1ctx, hmac_ipad_buffer, 865159232Spjd SHA1_HMAC_BLOCK_LEN - klen); 866159225Spjd bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32)); 867159225Spjd } 868159225Spjd 869159225Spjd for (i = 0; i < klen; i++) 870159225Spjd key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 871159225Spjd 872159225Spjd if (algo == CRYPTO_MD5_HMAC) { 873159225Spjd MD5Init(&md5ctx); 874159225Spjd MD5Update(&md5ctx, key, klen); 875159232Spjd MD5Update(&md5ctx, hmac_opad_buffer, MD5_HMAC_BLOCK_LEN - klen); 876159225Spjd bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state)); 877159225Spjd } else { 878159225Spjd SHA1Init(&sha1ctx); 879159225Spjd SHA1Update(&sha1ctx, key, klen); 880159232Spjd SHA1Update(&sha1ctx, hmac_opad_buffer, 881159232Spjd SHA1_HMAC_BLOCK_LEN - klen); 882159225Spjd bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32)); 883159225Spjd } 884159225Spjd 885159225Spjd for (i = 0; i < klen; i++) 886159225Spjd key[i] ^= HMAC_OPAD_VAL; 887159225Spjd} 888159225Spjd 889104478Ssam/* 890104478Ssam * Allocate a new 'session' and return an encoded session id. 'sidp' 891104478Ssam * contains our registration id, and should contain an encoded session 892104478Ssam * id on successful allocation. 893104478Ssam */ 894104478Ssamstatic int 895167755Ssamubsec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri) 896104478Ssam{ 897167755Ssam struct ubsec_softc *sc = device_get_softc(dev); 898104478Ssam struct cryptoini *c, *encini = NULL, *macini = NULL; 899104478Ssam struct ubsec_session *ses = NULL; 900159225Spjd int sesn; 901104478Ssam 902104478Ssam if (sidp == NULL || cri == NULL || sc == NULL) 903104478Ssam return (EINVAL); 904104478Ssam 905104478Ssam for (c = cri; c != NULL; c = c->cri_next) { 906104478Ssam if (c->cri_alg == CRYPTO_MD5_HMAC || 907104478Ssam c->cri_alg == CRYPTO_SHA1_HMAC) { 908104478Ssam if (macini) 909104478Ssam return (EINVAL); 910104478Ssam macini = c; 911104478Ssam } else if (c->cri_alg == CRYPTO_DES_CBC || 912104478Ssam c->cri_alg == CRYPTO_3DES_CBC) { 913104478Ssam if (encini) 914104478Ssam return (EINVAL); 915104478Ssam encini = c; 916104478Ssam } else 917104478Ssam return (EINVAL); 918104478Ssam } 919104478Ssam if (encini == NULL && macini == NULL) 920104478Ssam return (EINVAL); 921104478Ssam 922104478Ssam if (sc->sc_sessions == NULL) { 923104478Ssam ses = sc->sc_sessions = (struct ubsec_session *)malloc( 924104478Ssam sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 925104478Ssam if (ses == NULL) 926104478Ssam return (ENOMEM); 927104478Ssam sesn = 0; 928104478Ssam sc->sc_nsessions = 1; 929104478Ssam } else { 930104478Ssam for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 931104478Ssam if (sc->sc_sessions[sesn].ses_used == 0) { 932104478Ssam ses = &sc->sc_sessions[sesn]; 933104478Ssam break; 934104478Ssam } 935104478Ssam } 936104478Ssam 937104478Ssam if (ses == NULL) { 938104478Ssam sesn = sc->sc_nsessions; 939104478Ssam ses = (struct ubsec_session *)malloc((sesn + 1) * 940104478Ssam sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 941104478Ssam if (ses == NULL) 942104478Ssam return (ENOMEM); 943104478Ssam bcopy(sc->sc_sessions, ses, sesn * 944104478Ssam sizeof(struct ubsec_session)); 945104478Ssam bzero(sc->sc_sessions, sesn * 946104478Ssam sizeof(struct ubsec_session)); 947104478Ssam free(sc->sc_sessions, M_DEVBUF); 948104478Ssam sc->sc_sessions = ses; 949104478Ssam ses = &sc->sc_sessions[sesn]; 950104478Ssam sc->sc_nsessions++; 951104478Ssam } 952104478Ssam } 953104478Ssam bzero(ses, sizeof(struct ubsec_session)); 954104478Ssam ses->ses_used = 1; 955115747Ssam 956104478Ssam if (encini) { 957104478Ssam /* get an IV, network byte order */ 958104478Ssam /* XXX may read fewer than requested */ 959104478Ssam read_random(ses->ses_iv, sizeof(ses->ses_iv)); 960104478Ssam 961159225Spjd if (encini->cri_key != NULL) { 962159225Spjd ubsec_setup_enckey(ses, encini->cri_alg, 963159225Spjd encini->cri_key); 964159225Spjd } 965104478Ssam } 966104478Ssam 967104478Ssam if (macini) { 968158705Spjd ses->ses_mlen = macini->cri_mlen; 969158705Spjd if (ses->ses_mlen == 0) { 970158705Spjd if (macini->cri_alg == CRYPTO_MD5_HMAC) 971159233Spjd ses->ses_mlen = MD5_HASH_LEN; 972158705Spjd else 973159233Spjd ses->ses_mlen = SHA1_HASH_LEN; 974158705Spjd } 975158705Spjd 976159225Spjd if (macini->cri_key != NULL) { 977159225Spjd ubsec_setup_mackey(ses, macini->cri_alg, 978159225Spjd macini->cri_key, macini->cri_klen / 8); 979104478Ssam } 980104478Ssam } 981104478Ssam 982104478Ssam *sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn); 983104478Ssam return (0); 984104478Ssam} 985104478Ssam 986104478Ssam/* 987104478Ssam * Deallocate a session. 988104478Ssam */ 989104478Ssamstatic int 990167755Ssamubsec_freesession(device_t dev, u_int64_t tid) 991104478Ssam{ 992167755Ssam struct ubsec_softc *sc = device_get_softc(dev); 993115747Ssam int session, ret; 994116924Ssam u_int32_t sid = CRYPTO_SESID2LID(tid); 995104478Ssam 996104478Ssam if (sc == NULL) 997104478Ssam return (EINVAL); 998104478Ssam 999104478Ssam session = UBSEC_SESSION(sid); 1000115747Ssam if (session < sc->sc_nsessions) { 1001115747Ssam bzero(&sc->sc_sessions[session], 1002115747Ssam sizeof(sc->sc_sessions[session])); 1003115747Ssam ret = 0; 1004115747Ssam } else 1005115747Ssam ret = EINVAL; 1006104478Ssam 1007115747Ssam return (ret); 1008104478Ssam} 1009104478Ssam 1010104478Ssamstatic void 1011104478Ssamubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error) 1012104478Ssam{ 1013104478Ssam struct ubsec_operand *op = arg; 1014104478Ssam 1015104478Ssam KASSERT(nsegs <= UBS_MAX_SCATTER, 1016104478Ssam ("Too many DMA segments returned when mapping operand")); 1017104478Ssam#ifdef UBSEC_DEBUG 1018104478Ssam if (ubsec_debug) 1019159341Spjd printf("ubsec_op_cb: mapsize %u nsegs %d error %d\n", 1020159341Spjd (u_int) mapsize, nsegs, error); 1021104478Ssam#endif 1022159341Spjd if (error != 0) 1023159341Spjd return; 1024104478Ssam op->mapsize = mapsize; 1025104478Ssam op->nsegs = nsegs; 1026104478Ssam bcopy(seg, op->segs, nsegs * sizeof (seg[0])); 1027104478Ssam} 1028104478Ssam 1029104478Ssamstatic int 1030167755Ssamubsec_process(device_t dev, struct cryptop *crp, int hint) 1031104478Ssam{ 1032167755Ssam struct ubsec_softc *sc = device_get_softc(dev); 1033104478Ssam struct ubsec_q *q = NULL; 1034104478Ssam int err = 0, i, j, nicealign; 1035104478Ssam struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 1036104478Ssam int encoffset = 0, macoffset = 0, cpskip, cpoffset; 1037104478Ssam int sskip, dskip, stheend, dtheend; 1038104478Ssam int16_t coffset; 1039104478Ssam struct ubsec_session *ses; 1040104478Ssam struct ubsec_pktctx ctx; 1041104478Ssam struct ubsec_dma *dmap = NULL; 1042104478Ssam 1043104478Ssam if (crp == NULL || crp->crp_callback == NULL || sc == NULL) { 1044104478Ssam ubsecstats.hst_invalid++; 1045104478Ssam return (EINVAL); 1046104478Ssam } 1047104478Ssam if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) { 1048108471Ssam ubsecstats.hst_badsession++; 1049104478Ssam return (EINVAL); 1050104478Ssam } 1051104478Ssam 1052115747Ssam mtx_lock(&sc->sc_freeqlock); 1053104478Ssam if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 1054104478Ssam ubsecstats.hst_queuefull++; 1055104478Ssam sc->sc_needwakeup |= CRYPTO_SYMQ; 1056115747Ssam mtx_unlock(&sc->sc_freeqlock); 1057104478Ssam return (ERESTART); 1058104478Ssam } 1059104478Ssam q = SIMPLEQ_FIRST(&sc->sc_freequeue); 1060163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next); 1061115747Ssam mtx_unlock(&sc->sc_freeqlock); 1062104478Ssam 1063104478Ssam dmap = q->q_dma; /* Save dma pointer */ 1064104478Ssam bzero(q, sizeof(struct ubsec_q)); 1065104478Ssam bzero(&ctx, sizeof(ctx)); 1066104478Ssam 1067104478Ssam q->q_sesn = UBSEC_SESSION(crp->crp_sid); 1068104478Ssam q->q_dma = dmap; 1069104478Ssam ses = &sc->sc_sessions[q->q_sesn]; 1070104478Ssam 1071104478Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) { 1072104478Ssam q->q_src_m = (struct mbuf *)crp->crp_buf; 1073104478Ssam q->q_dst_m = (struct mbuf *)crp->crp_buf; 1074104478Ssam } else if (crp->crp_flags & CRYPTO_F_IOV) { 1075104478Ssam q->q_src_io = (struct uio *)crp->crp_buf; 1076104478Ssam q->q_dst_io = (struct uio *)crp->crp_buf; 1077104478Ssam } else { 1078108471Ssam ubsecstats.hst_badflags++; 1079104478Ssam err = EINVAL; 1080104478Ssam goto errout; /* XXX we don't handle contiguous blocks! */ 1081104478Ssam } 1082104478Ssam 1083104478Ssam bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr)); 1084104478Ssam 1085104478Ssam dmap->d_dma->d_mcr.mcr_pkts = htole16(1); 1086104478Ssam dmap->d_dma->d_mcr.mcr_flags = 0; 1087104478Ssam q->q_crp = crp; 1088104478Ssam 1089104478Ssam crd1 = crp->crp_desc; 1090104478Ssam if (crd1 == NULL) { 1091108471Ssam ubsecstats.hst_nodesc++; 1092104478Ssam err = EINVAL; 1093104478Ssam goto errout; 1094104478Ssam } 1095104478Ssam crd2 = crd1->crd_next; 1096104478Ssam 1097104478Ssam if (crd2 == NULL) { 1098104478Ssam if (crd1->crd_alg == CRYPTO_MD5_HMAC || 1099104478Ssam crd1->crd_alg == CRYPTO_SHA1_HMAC) { 1100104478Ssam maccrd = crd1; 1101104478Ssam enccrd = NULL; 1102104478Ssam } else if (crd1->crd_alg == CRYPTO_DES_CBC || 1103104478Ssam crd1->crd_alg == CRYPTO_3DES_CBC) { 1104104478Ssam maccrd = NULL; 1105104478Ssam enccrd = crd1; 1106104478Ssam } else { 1107108471Ssam ubsecstats.hst_badalg++; 1108104478Ssam err = EINVAL; 1109104478Ssam goto errout; 1110104478Ssam } 1111104478Ssam } else { 1112104478Ssam if ((crd1->crd_alg == CRYPTO_MD5_HMAC || 1113104478Ssam crd1->crd_alg == CRYPTO_SHA1_HMAC) && 1114104478Ssam (crd2->crd_alg == CRYPTO_DES_CBC || 1115104478Ssam crd2->crd_alg == CRYPTO_3DES_CBC) && 1116104478Ssam ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 1117104478Ssam maccrd = crd1; 1118104478Ssam enccrd = crd2; 1119104478Ssam } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 1120104478Ssam crd1->crd_alg == CRYPTO_3DES_CBC) && 1121104478Ssam (crd2->crd_alg == CRYPTO_MD5_HMAC || 1122104478Ssam crd2->crd_alg == CRYPTO_SHA1_HMAC) && 1123104478Ssam (crd1->crd_flags & CRD_F_ENCRYPT)) { 1124104478Ssam enccrd = crd1; 1125104478Ssam maccrd = crd2; 1126104478Ssam } else { 1127104478Ssam /* 1128104478Ssam * We cannot order the ubsec as requested 1129104478Ssam */ 1130108471Ssam ubsecstats.hst_badalg++; 1131104478Ssam err = EINVAL; 1132104478Ssam goto errout; 1133104478Ssam } 1134104478Ssam } 1135104478Ssam 1136104478Ssam if (enccrd) { 1137159225Spjd if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) { 1138159225Spjd ubsec_setup_enckey(ses, enccrd->crd_alg, 1139159225Spjd enccrd->crd_key); 1140159225Spjd } 1141159225Spjd 1142104478Ssam encoffset = enccrd->crd_skip; 1143104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES); 1144104478Ssam 1145104478Ssam if (enccrd->crd_flags & CRD_F_ENCRYPT) { 1146104478Ssam q->q_flags |= UBSEC_QFLAGS_COPYOUTIV; 1147104478Ssam 1148104478Ssam if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1149104478Ssam bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1150104478Ssam else { 1151104478Ssam ctx.pc_iv[0] = ses->ses_iv[0]; 1152104478Ssam ctx.pc_iv[1] = ses->ses_iv[1]; 1153104478Ssam } 1154104478Ssam 1155104478Ssam if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { 1156159242Spjd crypto_copyback(crp->crp_flags, crp->crp_buf, 1157159242Spjd enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv); 1158104478Ssam } 1159104478Ssam } else { 1160104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND); 1161104478Ssam 1162104478Ssam if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1163104478Ssam bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1164159242Spjd else { 1165159242Spjd crypto_copydata(crp->crp_flags, crp->crp_buf, 1166159242Spjd enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv); 1167159242Spjd } 1168104478Ssam } 1169104478Ssam 1170104478Ssam ctx.pc_deskey[0] = ses->ses_deskey[0]; 1171104478Ssam ctx.pc_deskey[1] = ses->ses_deskey[1]; 1172104478Ssam ctx.pc_deskey[2] = ses->ses_deskey[2]; 1173104478Ssam ctx.pc_deskey[3] = ses->ses_deskey[3]; 1174104478Ssam ctx.pc_deskey[4] = ses->ses_deskey[4]; 1175104478Ssam ctx.pc_deskey[5] = ses->ses_deskey[5]; 1176104478Ssam SWAP32(ctx.pc_iv[0]); 1177104478Ssam SWAP32(ctx.pc_iv[1]); 1178104478Ssam } 1179104478Ssam 1180104478Ssam if (maccrd) { 1181159225Spjd if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) { 1182159225Spjd ubsec_setup_mackey(ses, maccrd->crd_alg, 1183159225Spjd maccrd->crd_key, maccrd->crd_klen / 8); 1184159225Spjd } 1185159225Spjd 1186104478Ssam macoffset = maccrd->crd_skip; 1187104478Ssam 1188104478Ssam if (maccrd->crd_alg == CRYPTO_MD5_HMAC) 1189104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5); 1190104478Ssam else 1191104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1); 1192104478Ssam 1193104478Ssam for (i = 0; i < 5; i++) { 1194104478Ssam ctx.pc_hminner[i] = ses->ses_hminner[i]; 1195104478Ssam ctx.pc_hmouter[i] = ses->ses_hmouter[i]; 1196104478Ssam 1197104478Ssam HTOLE32(ctx.pc_hminner[i]); 1198104478Ssam HTOLE32(ctx.pc_hmouter[i]); 1199104478Ssam } 1200104478Ssam } 1201104478Ssam 1202104478Ssam if (enccrd && maccrd) { 1203104478Ssam /* 1204104478Ssam * ubsec cannot handle packets where the end of encryption 1205104478Ssam * and authentication are not the same, or where the 1206104478Ssam * encrypted part begins before the authenticated part. 1207104478Ssam */ 1208104478Ssam if ((encoffset + enccrd->crd_len) != 1209104478Ssam (macoffset + maccrd->crd_len)) { 1210104478Ssam ubsecstats.hst_lenmismatch++; 1211104478Ssam err = EINVAL; 1212104478Ssam goto errout; 1213104478Ssam } 1214104478Ssam if (enccrd->crd_skip < maccrd->crd_skip) { 1215104478Ssam ubsecstats.hst_skipmismatch++; 1216104478Ssam err = EINVAL; 1217104478Ssam goto errout; 1218104478Ssam } 1219104478Ssam sskip = maccrd->crd_skip; 1220104478Ssam cpskip = dskip = enccrd->crd_skip; 1221104478Ssam stheend = maccrd->crd_len; 1222104478Ssam dtheend = enccrd->crd_len; 1223104478Ssam coffset = enccrd->crd_skip - maccrd->crd_skip; 1224104478Ssam cpoffset = cpskip + dtheend; 1225104478Ssam#ifdef UBSEC_DEBUG 1226104478Ssam if (ubsec_debug) { 1227104478Ssam printf("mac: skip %d, len %d, inject %d\n", 1228104478Ssam maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject); 1229104478Ssam printf("enc: skip %d, len %d, inject %d\n", 1230104478Ssam enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject); 1231104478Ssam printf("src: skip %d, len %d\n", sskip, stheend); 1232104478Ssam printf("dst: skip %d, len %d\n", dskip, dtheend); 1233104478Ssam printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n", 1234104478Ssam coffset, stheend, cpskip, cpoffset); 1235104478Ssam } 1236104478Ssam#endif 1237104478Ssam } else { 1238104478Ssam cpskip = dskip = sskip = macoffset + encoffset; 1239104478Ssam dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len; 1240104478Ssam cpoffset = cpskip + dtheend; 1241104478Ssam coffset = 0; 1242104478Ssam } 1243104478Ssam ctx.pc_offset = htole16(coffset >> 2); 1244104478Ssam 1245104478Ssam if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) { 1246104478Ssam ubsecstats.hst_nomap++; 1247104478Ssam err = ENOMEM; 1248104478Ssam goto errout; 1249104478Ssam } 1250104478Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) { 1251104478Ssam if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map, 1252104478Ssam q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1253104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1254104478Ssam q->q_src_map = NULL; 1255104478Ssam ubsecstats.hst_noload++; 1256104478Ssam err = ENOMEM; 1257104478Ssam goto errout; 1258104478Ssam } 1259104478Ssam } else if (crp->crp_flags & CRYPTO_F_IOV) { 1260104478Ssam if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map, 1261104478Ssam q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1262104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1263104478Ssam q->q_src_map = NULL; 1264104478Ssam ubsecstats.hst_noload++; 1265104478Ssam err = ENOMEM; 1266104478Ssam goto errout; 1267104478Ssam } 1268104478Ssam } 1269104478Ssam nicealign = ubsec_dmamap_aligned(&q->q_src); 1270104478Ssam 1271104478Ssam dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend); 1272104478Ssam 1273104478Ssam#ifdef UBSEC_DEBUG 1274104478Ssam if (ubsec_debug) 1275104478Ssam printf("src skip: %d nicealign: %u\n", sskip, nicealign); 1276104478Ssam#endif 1277104478Ssam for (i = j = 0; i < q->q_src_nsegs; i++) { 1278104478Ssam struct ubsec_pktbuf *pb; 1279104478Ssam bus_size_t packl = q->q_src_segs[i].ds_len; 1280104478Ssam bus_addr_t packp = q->q_src_segs[i].ds_addr; 1281104478Ssam 1282104478Ssam if (sskip >= packl) { 1283104478Ssam sskip -= packl; 1284104478Ssam continue; 1285104478Ssam } 1286104478Ssam 1287104478Ssam packl -= sskip; 1288104478Ssam packp += sskip; 1289104478Ssam sskip = 0; 1290104478Ssam 1291104478Ssam if (packl > 0xfffc) { 1292104478Ssam err = EIO; 1293104478Ssam goto errout; 1294104478Ssam } 1295104478Ssam 1296104478Ssam if (j == 0) 1297104478Ssam pb = &dmap->d_dma->d_mcr.mcr_ipktbuf; 1298104478Ssam else 1299104478Ssam pb = &dmap->d_dma->d_sbuf[j - 1]; 1300104478Ssam 1301104478Ssam pb->pb_addr = htole32(packp); 1302104478Ssam 1303104478Ssam if (stheend) { 1304104478Ssam if (packl > stheend) { 1305104478Ssam pb->pb_len = htole32(stheend); 1306104478Ssam stheend = 0; 1307104478Ssam } else { 1308104478Ssam pb->pb_len = htole32(packl); 1309104478Ssam stheend -= packl; 1310104478Ssam } 1311104478Ssam } else 1312104478Ssam pb->pb_len = htole32(packl); 1313104478Ssam 1314104478Ssam if ((i + 1) == q->q_src_nsegs) 1315104478Ssam pb->pb_next = 0; 1316104478Ssam else 1317104478Ssam pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1318104478Ssam offsetof(struct ubsec_dmachunk, d_sbuf[j])); 1319104478Ssam j++; 1320104478Ssam } 1321104478Ssam 1322104478Ssam if (enccrd == NULL && maccrd != NULL) { 1323104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0; 1324104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0; 1325104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr + 1326104478Ssam offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1327104478Ssam#ifdef UBSEC_DEBUG 1328104478Ssam if (ubsec_debug) 1329104478Ssam printf("opkt: %x %x %x\n", 1330104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr, 1331104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_len, 1332104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_next); 1333104478Ssam#endif 1334104478Ssam } else { 1335104478Ssam if (crp->crp_flags & CRYPTO_F_IOV) { 1336104478Ssam if (!nicealign) { 1337104478Ssam ubsecstats.hst_iovmisaligned++; 1338104478Ssam err = EINVAL; 1339104478Ssam goto errout; 1340104478Ssam } 1341104478Ssam if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, 1342104478Ssam &q->q_dst_map)) { 1343104478Ssam ubsecstats.hst_nomap++; 1344104478Ssam err = ENOMEM; 1345104478Ssam goto errout; 1346104478Ssam } 1347104478Ssam if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map, 1348104478Ssam q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) { 1349104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1350104478Ssam q->q_dst_map = NULL; 1351104478Ssam ubsecstats.hst_noload++; 1352104478Ssam err = ENOMEM; 1353104478Ssam goto errout; 1354104478Ssam } 1355104478Ssam } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1356104478Ssam if (nicealign) { 1357104478Ssam q->q_dst = q->q_src; 1358104478Ssam } else { 1359104478Ssam int totlen, len; 1360104478Ssam struct mbuf *m, *top, **mp; 1361104478Ssam 1362104478Ssam ubsecstats.hst_unaligned++; 1363104478Ssam totlen = q->q_src_mapsize; 1364160931Sjhb if (totlen >= MINCLSIZE) { 1365243857Sglebius m = m_getcl(M_NOWAIT, MT_DATA, 1366160931Sjhb q->q_src_m->m_flags & M_PKTHDR); 1367160931Sjhb len = MCLBYTES; 1368160931Sjhb } else if (q->q_src_m->m_flags & M_PKTHDR) { 1369243857Sglebius m = m_gethdr(M_NOWAIT, MT_DATA); 1370104478Ssam len = MHLEN; 1371104478Ssam } else { 1372243857Sglebius m = m_get(M_NOWAIT, MT_DATA); 1373104478Ssam len = MLEN; 1374104478Ssam } 1375160931Sjhb if (m && q->q_src_m->m_flags & M_PKTHDR && 1376243857Sglebius !m_dup_pkthdr(m, q->q_src_m, M_NOWAIT)) { 1377160931Sjhb m_free(m); 1378160931Sjhb m = NULL; 1379160931Sjhb } 1380104478Ssam if (m == NULL) { 1381104478Ssam ubsecstats.hst_nombuf++; 1382104478Ssam err = sc->sc_nqueue ? ERESTART : ENOMEM; 1383104478Ssam goto errout; 1384104478Ssam } 1385160931Sjhb m->m_len = len = min(totlen, len); 1386160931Sjhb totlen -= len; 1387160931Sjhb top = m; 1388104478Ssam mp = ⊤ 1389104478Ssam 1390104478Ssam while (totlen > 0) { 1391160931Sjhb if (totlen >= MINCLSIZE) { 1392243857Sglebius m = m_getcl(M_NOWAIT, 1393160931Sjhb MT_DATA, 0); 1394160931Sjhb len = MCLBYTES; 1395160931Sjhb } else { 1396243857Sglebius m = m_get(M_NOWAIT, MT_DATA); 1397104478Ssam len = MLEN; 1398104478Ssam } 1399160931Sjhb if (m == NULL) { 1400160931Sjhb m_freem(top); 1401160931Sjhb ubsecstats.hst_nombuf++; 1402160931Sjhb err = sc->sc_nqueue ? ERESTART : ENOMEM; 1403160931Sjhb goto errout; 1404104478Ssam } 1405104478Ssam m->m_len = len = min(totlen, len); 1406104478Ssam totlen -= len; 1407104478Ssam *mp = m; 1408104478Ssam mp = &m->m_next; 1409104478Ssam } 1410104478Ssam q->q_dst_m = top; 1411104478Ssam ubsec_mcopy(q->q_src_m, q->q_dst_m, 1412104478Ssam cpskip, cpoffset); 1413162969Sjhb if (bus_dmamap_create(sc->sc_dmat, 1414104478Ssam BUS_DMA_NOWAIT, &q->q_dst_map) != 0) { 1415104478Ssam ubsecstats.hst_nomap++; 1416104478Ssam err = ENOMEM; 1417104478Ssam goto errout; 1418104478Ssam } 1419104478Ssam if (bus_dmamap_load_mbuf(sc->sc_dmat, 1420104478Ssam q->q_dst_map, q->q_dst_m, 1421104478Ssam ubsec_op_cb, &q->q_dst, 1422104478Ssam BUS_DMA_NOWAIT) != 0) { 1423104478Ssam bus_dmamap_destroy(sc->sc_dmat, 1424104478Ssam q->q_dst_map); 1425104478Ssam q->q_dst_map = NULL; 1426104478Ssam ubsecstats.hst_noload++; 1427104478Ssam err = ENOMEM; 1428104478Ssam goto errout; 1429104478Ssam } 1430104478Ssam } 1431104478Ssam } else { 1432108471Ssam ubsecstats.hst_badflags++; 1433104478Ssam err = EINVAL; 1434104478Ssam goto errout; 1435104478Ssam } 1436104478Ssam 1437104478Ssam#ifdef UBSEC_DEBUG 1438104478Ssam if (ubsec_debug) 1439104478Ssam printf("dst skip: %d\n", dskip); 1440104478Ssam#endif 1441104478Ssam for (i = j = 0; i < q->q_dst_nsegs; i++) { 1442104478Ssam struct ubsec_pktbuf *pb; 1443104478Ssam bus_size_t packl = q->q_dst_segs[i].ds_len; 1444104478Ssam bus_addr_t packp = q->q_dst_segs[i].ds_addr; 1445104478Ssam 1446104478Ssam if (dskip >= packl) { 1447104478Ssam dskip -= packl; 1448104478Ssam continue; 1449104478Ssam } 1450104478Ssam 1451104478Ssam packl -= dskip; 1452104478Ssam packp += dskip; 1453104478Ssam dskip = 0; 1454104478Ssam 1455104478Ssam if (packl > 0xfffc) { 1456104478Ssam err = EIO; 1457104478Ssam goto errout; 1458104478Ssam } 1459104478Ssam 1460104478Ssam if (j == 0) 1461104478Ssam pb = &dmap->d_dma->d_mcr.mcr_opktbuf; 1462104478Ssam else 1463104478Ssam pb = &dmap->d_dma->d_dbuf[j - 1]; 1464104478Ssam 1465104478Ssam pb->pb_addr = htole32(packp); 1466104478Ssam 1467104478Ssam if (dtheend) { 1468104478Ssam if (packl > dtheend) { 1469104478Ssam pb->pb_len = htole32(dtheend); 1470104478Ssam dtheend = 0; 1471104478Ssam } else { 1472104478Ssam pb->pb_len = htole32(packl); 1473104478Ssam dtheend -= packl; 1474104478Ssam } 1475104478Ssam } else 1476104478Ssam pb->pb_len = htole32(packl); 1477104478Ssam 1478104478Ssam if ((i + 1) == q->q_dst_nsegs) { 1479104478Ssam if (maccrd) 1480104478Ssam pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1481104478Ssam offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1482104478Ssam else 1483104478Ssam pb->pb_next = 0; 1484104478Ssam } else 1485104478Ssam pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1486104478Ssam offsetof(struct ubsec_dmachunk, d_dbuf[j])); 1487104478Ssam j++; 1488104478Ssam } 1489104478Ssam } 1490104478Ssam 1491104478Ssam dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr + 1492104478Ssam offsetof(struct ubsec_dmachunk, d_ctx)); 1493104478Ssam 1494104478Ssam if (sc->sc_flags & UBS_FLAGS_LONGCTX) { 1495104478Ssam struct ubsec_pktctx_long *ctxl; 1496104478Ssam 1497104478Ssam ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr + 1498104478Ssam offsetof(struct ubsec_dmachunk, d_ctx)); 1499162969Sjhb 1500104478Ssam /* transform small context into long context */ 1501104478Ssam ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long)); 1502104478Ssam ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC); 1503104478Ssam ctxl->pc_flags = ctx.pc_flags; 1504104478Ssam ctxl->pc_offset = ctx.pc_offset; 1505104478Ssam for (i = 0; i < 6; i++) 1506104478Ssam ctxl->pc_deskey[i] = ctx.pc_deskey[i]; 1507104478Ssam for (i = 0; i < 5; i++) 1508104478Ssam ctxl->pc_hminner[i] = ctx.pc_hminner[i]; 1509104478Ssam for (i = 0; i < 5; i++) 1510162969Sjhb ctxl->pc_hmouter[i] = ctx.pc_hmouter[i]; 1511104478Ssam ctxl->pc_iv[0] = ctx.pc_iv[0]; 1512104478Ssam ctxl->pc_iv[1] = ctx.pc_iv[1]; 1513104478Ssam } else 1514104478Ssam bcopy(&ctx, dmap->d_alloc.dma_vaddr + 1515104478Ssam offsetof(struct ubsec_dmachunk, d_ctx), 1516104478Ssam sizeof(struct ubsec_pktctx)); 1517104478Ssam 1518115747Ssam mtx_lock(&sc->sc_mcr1lock); 1519104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next); 1520104478Ssam sc->sc_nqueue++; 1521104478Ssam ubsecstats.hst_ipackets++; 1522104478Ssam ubsecstats.hst_ibytes += dmap->d_alloc.dma_size; 1523111416Ssam if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= UBS_MAX_AGGR) 1524104478Ssam ubsec_feed(sc); 1525115747Ssam mtx_unlock(&sc->sc_mcr1lock); 1526104478Ssam return (0); 1527104478Ssam 1528104478Ssamerrout: 1529104478Ssam if (q != NULL) { 1530104478Ssam if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 1531104478Ssam m_freem(q->q_dst_m); 1532104478Ssam 1533104478Ssam if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1534104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1535104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1536104478Ssam } 1537104478Ssam if (q->q_src_map != NULL) { 1538104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1539104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1540104478Ssam } 1541158828Spjd } 1542158828Spjd if (q != NULL || err == ERESTART) { 1543115747Ssam mtx_lock(&sc->sc_freeqlock); 1544158828Spjd if (q != NULL) 1545158828Spjd SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1546158828Spjd if (err == ERESTART) 1547158828Spjd sc->sc_needwakeup |= CRYPTO_SYMQ; 1548115747Ssam mtx_unlock(&sc->sc_freeqlock); 1549104478Ssam } 1550104478Ssam if (err != ERESTART) { 1551104478Ssam crp->crp_etype = err; 1552104478Ssam crypto_done(crp); 1553104478Ssam } 1554104478Ssam return (err); 1555104478Ssam} 1556104478Ssam 1557104478Ssamstatic void 1558104478Ssamubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q) 1559104478Ssam{ 1560104478Ssam struct cryptop *crp = (struct cryptop *)q->q_crp; 1561104478Ssam struct cryptodesc *crd; 1562104478Ssam struct ubsec_dma *dmap = q->q_dma; 1563104478Ssam 1564112099Ssam ubsecstats.hst_opackets++; 1565112099Ssam ubsecstats.hst_obytes += dmap->d_alloc.dma_size; 1566112099Ssam 1567108823Ssam ubsec_dma_sync(&dmap->d_alloc, 1568104478Ssam BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1569104478Ssam if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1570104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 1571104478Ssam BUS_DMASYNC_POSTREAD); 1572104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1573104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1574104478Ssam } 1575104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE); 1576104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1577104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1578104478Ssam 1579104478Ssam if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) { 1580104478Ssam m_freem(q->q_src_m); 1581104478Ssam crp->crp_buf = (caddr_t)q->q_dst_m; 1582104478Ssam } 1583104478Ssam 1584104478Ssam /* copy out IV for future use */ 1585104478Ssam if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) { 1586104478Ssam for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1587104478Ssam if (crd->crd_alg != CRYPTO_DES_CBC && 1588104478Ssam crd->crd_alg != CRYPTO_3DES_CBC) 1589104478Ssam continue; 1590159242Spjd crypto_copydata(crp->crp_flags, crp->crp_buf, 1591159242Spjd crd->crd_skip + crd->crd_len - 8, 8, 1592159242Spjd (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv); 1593104478Ssam break; 1594104478Ssam } 1595104478Ssam } 1596104478Ssam 1597104478Ssam for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1598104478Ssam if (crd->crd_alg != CRYPTO_MD5_HMAC && 1599104478Ssam crd->crd_alg != CRYPTO_SHA1_HMAC) 1600104478Ssam continue; 1601159242Spjd crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, 1602159242Spjd sc->sc_sessions[q->q_sesn].ses_mlen, 1603159242Spjd (caddr_t)dmap->d_dma->d_macbuf); 1604104478Ssam break; 1605104478Ssam } 1606115747Ssam mtx_lock(&sc->sc_freeqlock); 1607104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1608115747Ssam mtx_unlock(&sc->sc_freeqlock); 1609104478Ssam crypto_done(crp); 1610104478Ssam} 1611104478Ssam 1612104478Ssamstatic void 1613104478Ssamubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset) 1614104478Ssam{ 1615104478Ssam int i, j, dlen, slen; 1616104478Ssam caddr_t dptr, sptr; 1617104478Ssam 1618104478Ssam j = 0; 1619104478Ssam sptr = srcm->m_data; 1620104478Ssam slen = srcm->m_len; 1621104478Ssam dptr = dstm->m_data; 1622104478Ssam dlen = dstm->m_len; 1623104478Ssam 1624104478Ssam while (1) { 1625104478Ssam for (i = 0; i < min(slen, dlen); i++) { 1626104478Ssam if (j < hoffset || j >= toffset) 1627104478Ssam *dptr++ = *sptr++; 1628104478Ssam slen--; 1629104478Ssam dlen--; 1630104478Ssam j++; 1631104478Ssam } 1632104478Ssam if (slen == 0) { 1633104478Ssam srcm = srcm->m_next; 1634104478Ssam if (srcm == NULL) 1635104478Ssam return; 1636104478Ssam sptr = srcm->m_data; 1637104478Ssam slen = srcm->m_len; 1638104478Ssam } 1639104478Ssam if (dlen == 0) { 1640104478Ssam dstm = dstm->m_next; 1641104478Ssam if (dstm == NULL) 1642104478Ssam return; 1643104478Ssam dptr = dstm->m_data; 1644104478Ssam dlen = dstm->m_len; 1645104478Ssam } 1646104478Ssam } 1647104478Ssam} 1648104478Ssam 1649104478Ssam/* 1650104478Ssam * feed the key generator, must be called at splimp() or higher. 1651104478Ssam */ 1652104478Ssamstatic int 1653104478Ssamubsec_feed2(struct ubsec_softc *sc) 1654104478Ssam{ 1655104478Ssam struct ubsec_q2 *q; 1656104478Ssam 1657104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) { 1658104478Ssam if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL) 1659104478Ssam break; 1660104478Ssam q = SIMPLEQ_FIRST(&sc->sc_queue2); 1661104478Ssam 1662108823Ssam ubsec_dma_sync(&q->q_mcr, 1663104478Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1664108823Ssam ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_PREWRITE); 1665104478Ssam 1666104478Ssam WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr); 1667163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q_next); 1668104478Ssam --sc->sc_nqueue2; 1669104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next); 1670104478Ssam } 1671104478Ssam return (0); 1672104478Ssam} 1673104478Ssam 1674104478Ssam/* 1675104478Ssam * Callback for handling random numbers 1676104478Ssam */ 1677104478Ssamstatic void 1678104478Ssamubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q) 1679104478Ssam{ 1680104478Ssam struct cryptkop *krp; 1681104478Ssam struct ubsec_ctx_keyop *ctx; 1682104478Ssam 1683104478Ssam ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr; 1684108823Ssam ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_POSTWRITE); 1685104478Ssam 1686104478Ssam switch (q->q_type) { 1687104478Ssam#ifndef UBSEC_NO_RNG 1688104478Ssam case UBS_CTXOP_RNGBYPASS: { 1689104478Ssam struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q; 1690104478Ssam 1691108823Ssam ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_POSTREAD); 1692112124Ssam (*sc->sc_harvest)(sc->sc_rndtest, 1693112124Ssam rng->rng_buf.dma_vaddr, 1694112124Ssam UBSEC_RNG_BUFSIZ*sizeof (u_int32_t)); 1695104478Ssam rng->rng_used = 0; 1696104478Ssam callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1697104478Ssam break; 1698104478Ssam } 1699104478Ssam#endif 1700104478Ssam case UBS_CTXOP_MODEXP: { 1701104478Ssam struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 1702104478Ssam u_int rlen, clen; 1703104478Ssam 1704104478Ssam krp = me->me_krp; 1705104478Ssam rlen = (me->me_modbits + 7) / 8; 1706104630Ssam clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8; 1707104478Ssam 1708108823Ssam ubsec_dma_sync(&me->me_M, BUS_DMASYNC_POSTWRITE); 1709108823Ssam ubsec_dma_sync(&me->me_E, BUS_DMASYNC_POSTWRITE); 1710108823Ssam ubsec_dma_sync(&me->me_C, BUS_DMASYNC_POSTREAD); 1711108823Ssam ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_POSTWRITE); 1712104478Ssam 1713104478Ssam if (clen < rlen) 1714104478Ssam krp->krp_status = E2BIG; 1715104630Ssam else { 1716104630Ssam if (sc->sc_flags & UBS_FLAGS_HWNORM) { 1717104630Ssam bzero(krp->krp_param[krp->krp_iparams].crp_p, 1718104630Ssam (krp->krp_param[krp->krp_iparams].crp_nbits 1719104630Ssam + 7) / 8); 1720104630Ssam bcopy(me->me_C.dma_vaddr, 1721104630Ssam krp->krp_param[krp->krp_iparams].crp_p, 1722104630Ssam (me->me_modbits + 7) / 8); 1723104630Ssam } else 1724104630Ssam ubsec_kshift_l(me->me_shiftbits, 1725104630Ssam me->me_C.dma_vaddr, me->me_normbits, 1726104630Ssam krp->krp_param[krp->krp_iparams].crp_p, 1727104630Ssam krp->krp_param[krp->krp_iparams].crp_nbits); 1728104630Ssam } 1729104478Ssam 1730104478Ssam crypto_kdone(krp); 1731104478Ssam 1732104478Ssam /* bzero all potentially sensitive data */ 1733104478Ssam bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 1734104478Ssam bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 1735104478Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 1736104478Ssam bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 1737104478Ssam 1738104478Ssam /* Can't free here, so put us on the free list. */ 1739104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next); 1740104478Ssam break; 1741104478Ssam } 1742104478Ssam case UBS_CTXOP_RSAPRIV: { 1743104478Ssam struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 1744104478Ssam u_int len; 1745104478Ssam 1746104478Ssam krp = rp->rpr_krp; 1747108823Ssam ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_POSTWRITE); 1748108823Ssam ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_POSTREAD); 1749104478Ssam 1750104478Ssam len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8; 1751104478Ssam bcopy(rp->rpr_msgout.dma_vaddr, 1752104478Ssam krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len); 1753104478Ssam 1754104478Ssam crypto_kdone(krp); 1755104478Ssam 1756104478Ssam bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 1757104478Ssam bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 1758104478Ssam bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size); 1759104478Ssam 1760104478Ssam /* Can't free here, so put us on the free list. */ 1761104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next); 1762104478Ssam break; 1763104478Ssam } 1764104478Ssam default: 1765104478Ssam device_printf(sc->sc_dev, "unknown ctx op: %x\n", 1766104478Ssam letoh16(ctx->ctx_op)); 1767104478Ssam break; 1768104478Ssam } 1769104478Ssam} 1770104478Ssam 1771104478Ssam#ifndef UBSEC_NO_RNG 1772104478Ssamstatic void 1773104478Ssamubsec_rng(void *vsc) 1774104478Ssam{ 1775104478Ssam struct ubsec_softc *sc = vsc; 1776104478Ssam struct ubsec_q2_rng *rng = &sc->sc_rng; 1777104478Ssam struct ubsec_mcr *mcr; 1778104478Ssam struct ubsec_ctx_rngbypass *ctx; 1779104478Ssam 1780115747Ssam mtx_lock(&sc->sc_mcr2lock); 1781104478Ssam if (rng->rng_used) { 1782115747Ssam mtx_unlock(&sc->sc_mcr2lock); 1783104478Ssam return; 1784104478Ssam } 1785104478Ssam sc->sc_nqueue2++; 1786104478Ssam if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE) 1787104478Ssam goto out; 1788104478Ssam 1789104478Ssam mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr; 1790104478Ssam ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr; 1791104478Ssam 1792104478Ssam mcr->mcr_pkts = htole16(1); 1793104478Ssam mcr->mcr_flags = 0; 1794104478Ssam mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr); 1795104478Ssam mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0; 1796104478Ssam mcr->mcr_ipktbuf.pb_len = 0; 1797104478Ssam mcr->mcr_reserved = mcr->mcr_pktlen = 0; 1798104478Ssam mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr); 1799104478Ssam mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) & 1800104478Ssam UBS_PKTBUF_LEN); 1801104478Ssam mcr->mcr_opktbuf.pb_next = 0; 1802104478Ssam 1803104478Ssam ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass)); 1804104478Ssam ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS); 1805104478Ssam rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS; 1806104478Ssam 1807108823Ssam ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_PREREAD); 1808104478Ssam 1809104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next); 1810104478Ssam rng->rng_used = 1; 1811104478Ssam ubsec_feed2(sc); 1812104478Ssam ubsecstats.hst_rng++; 1813115747Ssam mtx_unlock(&sc->sc_mcr2lock); 1814104478Ssam 1815104478Ssam return; 1816104478Ssam 1817104478Ssamout: 1818104478Ssam /* 1819104478Ssam * Something weird happened, generate our own call back. 1820104478Ssam */ 1821104478Ssam sc->sc_nqueue2--; 1822115747Ssam mtx_unlock(&sc->sc_mcr2lock); 1823104478Ssam callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1824104478Ssam} 1825104478Ssam#endif /* UBSEC_NO_RNG */ 1826104478Ssam 1827104478Ssamstatic void 1828104478Ssamubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 1829104478Ssam{ 1830104478Ssam bus_addr_t *paddr = (bus_addr_t*) arg; 1831104478Ssam *paddr = segs->ds_addr; 1832104478Ssam} 1833104478Ssam 1834104478Ssamstatic int 1835104478Ssamubsec_dma_malloc( 1836104478Ssam struct ubsec_softc *sc, 1837104478Ssam bus_size_t size, 1838104478Ssam struct ubsec_dma_alloc *dma, 1839104478Ssam int mapflags 1840104478Ssam) 1841104478Ssam{ 1842104478Ssam int r; 1843104478Ssam 1844108823Ssam /* XXX could specify sc_dmat as parent but that just adds overhead */ 1845232874Sscottl r = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ 1846108823Ssam 1, 0, /* alignment, bounds */ 1847108823Ssam BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 1848108823Ssam BUS_SPACE_MAXADDR, /* highaddr */ 1849108823Ssam NULL, NULL, /* filter, filterarg */ 1850108823Ssam size, /* maxsize */ 1851108823Ssam 1, /* nsegments */ 1852108823Ssam size, /* maxsegsize */ 1853108823Ssam BUS_DMA_ALLOCNOW, /* flags */ 1854117126Sscottl NULL, NULL, /* lockfunc, lockarg */ 1855108823Ssam &dma->dma_tag); 1856108823Ssam if (r != 0) { 1857108823Ssam device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1858108823Ssam "bus_dma_tag_create failed; error %u\n", r); 1859104478Ssam goto fail_1; 1860108823Ssam } 1861104478Ssam 1862108823Ssam r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 1863108823Ssam BUS_DMA_NOWAIT, &dma->dma_map); 1864108823Ssam if (r != 0) { 1865108823Ssam device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1866205844Simp "bus_dmammem_alloc failed; size %ju, error %u\n", 1867205844Simp (intmax_t)size, r); 1868108823Ssam goto fail_2; 1869108823Ssam } 1870108823Ssam 1871108823Ssam r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 1872104478Ssam size, 1873104478Ssam ubsec_dmamap_cb, 1874104478Ssam &dma->dma_paddr, 1875104478Ssam mapflags | BUS_DMA_NOWAIT); 1876108823Ssam if (r != 0) { 1877108823Ssam device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1878108823Ssam "bus_dmamap_load failed; error %u\n", r); 1879108823Ssam goto fail_3; 1880108823Ssam } 1881104478Ssam 1882104478Ssam dma->dma_size = size; 1883104478Ssam return (0); 1884104478Ssam 1885108823Ssamfail_3: 1886108823Ssam bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1887104478Ssamfail_2: 1888108823Ssam bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1889104478Ssamfail_1: 1890108823Ssam bus_dma_tag_destroy(dma->dma_tag); 1891108823Ssam dma->dma_tag = NULL; 1892104478Ssam return (r); 1893104478Ssam} 1894104478Ssam 1895104478Ssamstatic void 1896104478Ssamubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma) 1897104478Ssam{ 1898108823Ssam bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1899108823Ssam bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1900108823Ssam bus_dma_tag_destroy(dma->dma_tag); 1901104478Ssam} 1902104478Ssam 1903104478Ssam/* 1904104478Ssam * Resets the board. Values in the regesters are left as is 1905104478Ssam * from the reset (i.e. initial values are assigned elsewhere). 1906104478Ssam */ 1907104478Ssamstatic void 1908104478Ssamubsec_reset_board(struct ubsec_softc *sc) 1909104478Ssam{ 1910104478Ssam volatile u_int32_t ctrl; 1911104478Ssam 1912104478Ssam ctrl = READ_REG(sc, BS_CTRL); 1913104478Ssam ctrl |= BS_CTRL_RESET; 1914104478Ssam WRITE_REG(sc, BS_CTRL, ctrl); 1915104478Ssam 1916104478Ssam /* 1917104478Ssam * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us 1918104478Ssam */ 1919104478Ssam DELAY(10); 1920104478Ssam} 1921104478Ssam 1922104478Ssam/* 1923104478Ssam * Init Broadcom registers 1924104478Ssam */ 1925104478Ssamstatic void 1926104478Ssamubsec_init_board(struct ubsec_softc *sc) 1927104478Ssam{ 1928104630Ssam u_int32_t ctrl; 1929104630Ssam 1930104630Ssam ctrl = READ_REG(sc, BS_CTRL); 1931104630Ssam ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64); 1932104630Ssam ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT; 1933104630Ssam 1934104630Ssam if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) 1935104630Ssam ctrl |= BS_CTRL_MCR2INT; 1936104630Ssam else 1937104630Ssam ctrl &= ~BS_CTRL_MCR2INT; 1938104630Ssam 1939104630Ssam if (sc->sc_flags & UBS_FLAGS_HWNORM) 1940104630Ssam ctrl &= ~BS_CTRL_SWNORM; 1941104630Ssam 1942104630Ssam WRITE_REG(sc, BS_CTRL, ctrl); 1943104478Ssam} 1944104478Ssam 1945104478Ssam/* 1946104478Ssam * Init Broadcom PCI registers 1947104478Ssam */ 1948104478Ssamstatic void 1949104478Ssamubsec_init_pciregs(device_t dev) 1950104478Ssam{ 1951104478Ssam#if 0 1952104478Ssam u_int32_t misc; 1953104478Ssam 1954104478Ssam misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT); 1955104478Ssam misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT)) 1956104478Ssam | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT); 1957104478Ssam misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT)) 1958104478Ssam | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT); 1959104478Ssam pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc); 1960104478Ssam#endif 1961104478Ssam 1962104478Ssam /* 1963104478Ssam * This will set the cache line size to 1, this will 1964104478Ssam * force the BCM58xx chip just to do burst read/writes. 1965104478Ssam * Cache line read/writes are to slow 1966104478Ssam */ 1967104478Ssam pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1); 1968104478Ssam} 1969104478Ssam 1970104478Ssam/* 1971104478Ssam * Clean up after a chip crash. 1972104478Ssam * It is assumed that the caller in splimp() 1973104478Ssam */ 1974104478Ssamstatic void 1975104478Ssamubsec_cleanchip(struct ubsec_softc *sc) 1976104478Ssam{ 1977104478Ssam struct ubsec_q *q; 1978104478Ssam 1979104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 1980104478Ssam q = SIMPLEQ_FIRST(&sc->sc_qchip); 1981163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next); 1982104478Ssam ubsec_free_q(sc, q); 1983104478Ssam } 1984108471Ssam sc->sc_nqchip = 0; 1985104478Ssam} 1986104478Ssam 1987104478Ssam/* 1988108823Ssam * free a ubsec_q 1989108823Ssam * It is assumed that the caller is within splimp(). 1990104478Ssam */ 1991104478Ssamstatic int 1992104478Ssamubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q) 1993104478Ssam{ 1994104478Ssam struct ubsec_q *q2; 1995104478Ssam struct cryptop *crp; 1996104478Ssam int npkts; 1997104478Ssam int i; 1998104478Ssam 1999104478Ssam npkts = q->q_nstacked_mcrs; 2000104478Ssam 2001104478Ssam for (i = 0; i < npkts; i++) { 2002104478Ssam if(q->q_stacked_mcr[i]) { 2003104478Ssam q2 = q->q_stacked_mcr[i]; 2004104478Ssam 2005162969Sjhb if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m)) 2006104478Ssam m_freem(q2->q_dst_m); 2007104478Ssam 2008104478Ssam crp = (struct cryptop *)q2->q_crp; 2009162969Sjhb 2010104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next); 2011162969Sjhb 2012104478Ssam crp->crp_etype = EFAULT; 2013104478Ssam crypto_done(crp); 2014104478Ssam } else { 2015104478Ssam break; 2016104478Ssam } 2017104478Ssam } 2018104478Ssam 2019104478Ssam /* 2020104478Ssam * Free header MCR 2021104478Ssam */ 2022104478Ssam if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 2023104478Ssam m_freem(q->q_dst_m); 2024104478Ssam 2025104478Ssam crp = (struct cryptop *)q->q_crp; 2026162969Sjhb 2027104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 2028162969Sjhb 2029104478Ssam crp->crp_etype = EFAULT; 2030104478Ssam crypto_done(crp); 2031104478Ssam return(0); 2032104478Ssam} 2033104478Ssam 2034104478Ssam/* 2035104478Ssam * Routine to reset the chip and clean up. 2036104478Ssam * It is assumed that the caller is in splimp() 2037104478Ssam */ 2038104478Ssamstatic void 2039104478Ssamubsec_totalreset(struct ubsec_softc *sc) 2040104478Ssam{ 2041104478Ssam ubsec_reset_board(sc); 2042104478Ssam ubsec_init_board(sc); 2043104478Ssam ubsec_cleanchip(sc); 2044104478Ssam} 2045104478Ssam 2046104478Ssamstatic int 2047104478Ssamubsec_dmamap_aligned(struct ubsec_operand *op) 2048104478Ssam{ 2049104478Ssam int i; 2050104478Ssam 2051104478Ssam for (i = 0; i < op->nsegs; i++) { 2052104478Ssam if (op->segs[i].ds_addr & 3) 2053104478Ssam return (0); 2054104478Ssam if ((i != (op->nsegs - 1)) && 2055104478Ssam (op->segs[i].ds_len & 3)) 2056104478Ssam return (0); 2057104478Ssam } 2058104478Ssam return (1); 2059104478Ssam} 2060104478Ssam 2061104478Ssamstatic void 2062104478Ssamubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q) 2063104478Ssam{ 2064104478Ssam switch (q->q_type) { 2065104478Ssam case UBS_CTXOP_MODEXP: { 2066104478Ssam struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 2067104478Ssam 2068104478Ssam ubsec_dma_free(sc, &me->me_q.q_mcr); 2069104478Ssam ubsec_dma_free(sc, &me->me_q.q_ctx); 2070104478Ssam ubsec_dma_free(sc, &me->me_M); 2071104478Ssam ubsec_dma_free(sc, &me->me_E); 2072104478Ssam ubsec_dma_free(sc, &me->me_C); 2073104478Ssam ubsec_dma_free(sc, &me->me_epb); 2074104478Ssam free(me, M_DEVBUF); 2075104478Ssam break; 2076104478Ssam } 2077104478Ssam case UBS_CTXOP_RSAPRIV: { 2078104478Ssam struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 2079104478Ssam 2080104478Ssam ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2081104478Ssam ubsec_dma_free(sc, &rp->rpr_q.q_ctx); 2082104478Ssam ubsec_dma_free(sc, &rp->rpr_msgin); 2083104478Ssam ubsec_dma_free(sc, &rp->rpr_msgout); 2084104478Ssam free(rp, M_DEVBUF); 2085104478Ssam break; 2086104478Ssam } 2087104478Ssam default: 2088104478Ssam device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type); 2089104478Ssam break; 2090104478Ssam } 2091104478Ssam} 2092104478Ssam 2093104478Ssamstatic int 2094167755Ssamubsec_kprocess(device_t dev, struct cryptkop *krp, int hint) 2095104478Ssam{ 2096167755Ssam struct ubsec_softc *sc = device_get_softc(dev); 2097104630Ssam int r; 2098104478Ssam 2099104478Ssam if (krp == NULL || krp->krp_callback == NULL) 2100104478Ssam return (EINVAL); 2101104478Ssam 2102104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) { 2103104478Ssam struct ubsec_q2 *q; 2104104478Ssam 2105104478Ssam q = SIMPLEQ_FIRST(&sc->sc_q2free); 2106163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q_next); 2107104478Ssam ubsec_kfree(sc, q); 2108104478Ssam } 2109104478Ssam 2110104478Ssam switch (krp->krp_op) { 2111104478Ssam case CRK_MOD_EXP: 2112104630Ssam if (sc->sc_flags & UBS_FLAGS_HWNORM) 2113104630Ssam r = ubsec_kprocess_modexp_hw(sc, krp, hint); 2114104630Ssam else 2115104630Ssam r = ubsec_kprocess_modexp_sw(sc, krp, hint); 2116104630Ssam break; 2117104478Ssam case CRK_MOD_EXP_CRT: 2118104478Ssam return (ubsec_kprocess_rsapriv(sc, krp, hint)); 2119104478Ssam default: 2120104478Ssam device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n", 2121104478Ssam krp->krp_op); 2122104478Ssam krp->krp_status = EOPNOTSUPP; 2123104478Ssam crypto_kdone(krp); 2124104478Ssam return (0); 2125104478Ssam } 2126104630Ssam return (0); /* silence compiler */ 2127104478Ssam} 2128104478Ssam 2129104478Ssam/* 2130104630Ssam * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization) 2131104478Ssam */ 2132104478Ssamstatic int 2133104630Ssamubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2134104478Ssam{ 2135104478Ssam struct ubsec_q2_modexp *me; 2136104478Ssam struct ubsec_mcr *mcr; 2137104478Ssam struct ubsec_ctx_modexp *ctx; 2138104478Ssam struct ubsec_pktbuf *epb; 2139104478Ssam int err = 0; 2140104478Ssam u_int nbits, normbits, mbits, shiftbits, ebits; 2141104478Ssam 2142104478Ssam me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2143104478Ssam if (me == NULL) { 2144104478Ssam err = ENOMEM; 2145104478Ssam goto errout; 2146104478Ssam } 2147104478Ssam bzero(me, sizeof *me); 2148104478Ssam me->me_krp = krp; 2149104478Ssam me->me_q.q_type = UBS_CTXOP_MODEXP; 2150104478Ssam 2151104478Ssam nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2152104478Ssam if (nbits <= 512) 2153104478Ssam normbits = 512; 2154104478Ssam else if (nbits <= 768) 2155104478Ssam normbits = 768; 2156104478Ssam else if (nbits <= 1024) 2157104478Ssam normbits = 1024; 2158104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2159104478Ssam normbits = 1536; 2160104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2161104478Ssam normbits = 2048; 2162104478Ssam else { 2163104478Ssam err = E2BIG; 2164104478Ssam goto errout; 2165104478Ssam } 2166104478Ssam 2167104630Ssam shiftbits = normbits - nbits; 2168104478Ssam 2169104630Ssam me->me_modbits = nbits; 2170104478Ssam me->me_shiftbits = shiftbits; 2171104630Ssam me->me_normbits = normbits; 2172104478Ssam 2173104478Ssam /* Sanity check: result bits must be >= true modulus bits. */ 2174104630Ssam if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2175104478Ssam err = ERANGE; 2176104478Ssam goto errout; 2177104478Ssam } 2178104478Ssam 2179104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2180104478Ssam &me->me_q.q_mcr, 0)) { 2181104478Ssam err = ENOMEM; 2182104478Ssam goto errout; 2183104478Ssam } 2184104478Ssam mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2185104478Ssam 2186104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2187104478Ssam &me->me_q.q_ctx, 0)) { 2188104478Ssam err = ENOMEM; 2189104478Ssam goto errout; 2190104478Ssam } 2191104478Ssam 2192104478Ssam mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2193104478Ssam if (mbits > nbits) { 2194104478Ssam err = E2BIG; 2195104478Ssam goto errout; 2196104478Ssam } 2197104478Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2198104478Ssam err = ENOMEM; 2199104478Ssam goto errout; 2200104478Ssam } 2201104478Ssam ubsec_kshift_r(shiftbits, 2202104478Ssam krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits, 2203104478Ssam me->me_M.dma_vaddr, normbits); 2204104478Ssam 2205104478Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2206104478Ssam err = ENOMEM; 2207104478Ssam goto errout; 2208104478Ssam } 2209104478Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2210104478Ssam 2211104478Ssam ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2212104478Ssam if (ebits > nbits) { 2213104478Ssam err = E2BIG; 2214104478Ssam goto errout; 2215104478Ssam } 2216104478Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2217104478Ssam err = ENOMEM; 2218104478Ssam goto errout; 2219104478Ssam } 2220104478Ssam ubsec_kshift_r(shiftbits, 2221104478Ssam krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits, 2222104478Ssam me->me_E.dma_vaddr, normbits); 2223104478Ssam 2224104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2225104478Ssam &me->me_epb, 0)) { 2226104478Ssam err = ENOMEM; 2227104478Ssam goto errout; 2228104478Ssam } 2229104478Ssam epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2230104478Ssam epb->pb_addr = htole32(me->me_E.dma_paddr); 2231104478Ssam epb->pb_next = 0; 2232104478Ssam epb->pb_len = htole32(normbits / 8); 2233104478Ssam 2234104478Ssam#ifdef UBSEC_DEBUG 2235104478Ssam if (ubsec_debug) { 2236104478Ssam printf("Epb "); 2237104478Ssam ubsec_dump_pb(epb); 2238104478Ssam } 2239104478Ssam#endif 2240104478Ssam 2241104478Ssam mcr->mcr_pkts = htole16(1); 2242104478Ssam mcr->mcr_flags = 0; 2243104478Ssam mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2244104478Ssam mcr->mcr_reserved = 0; 2245104478Ssam mcr->mcr_pktlen = 0; 2246104478Ssam 2247104478Ssam mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2248104478Ssam mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2249104478Ssam mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2250104478Ssam 2251104478Ssam mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2252104478Ssam mcr->mcr_opktbuf.pb_next = 0; 2253104478Ssam mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2254104478Ssam 2255104478Ssam#ifdef DIAGNOSTIC 2256104478Ssam /* Misaligned output buffer will hang the chip. */ 2257104478Ssam if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2258104478Ssam panic("%s: modexp invalid addr 0x%x\n", 2259104478Ssam device_get_nameunit(sc->sc_dev), 2260104478Ssam letoh32(mcr->mcr_opktbuf.pb_addr)); 2261104478Ssam if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2262104478Ssam panic("%s: modexp invalid len 0x%x\n", 2263104478Ssam device_get_nameunit(sc->sc_dev), 2264104478Ssam letoh32(mcr->mcr_opktbuf.pb_len)); 2265104478Ssam#endif 2266104478Ssam 2267104478Ssam ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2268104478Ssam bzero(ctx, sizeof(*ctx)); 2269104478Ssam ubsec_kshift_r(shiftbits, 2270104478Ssam krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits, 2271104478Ssam ctx->me_N, normbits); 2272104478Ssam ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2273104478Ssam ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2274104630Ssam ctx->me_E_len = htole16(nbits); 2275104630Ssam ctx->me_N_len = htole16(nbits); 2276104478Ssam 2277104478Ssam#ifdef UBSEC_DEBUG 2278104478Ssam if (ubsec_debug) { 2279104478Ssam ubsec_dump_mcr(mcr); 2280104478Ssam ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2281104478Ssam } 2282104478Ssam#endif 2283104478Ssam 2284104478Ssam /* 2285104478Ssam * ubsec_feed2 will sync mcr and ctx, we just need to sync 2286104478Ssam * everything else. 2287104478Ssam */ 2288108823Ssam ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE); 2289108823Ssam ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE); 2290108823Ssam ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD); 2291108823Ssam ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE); 2292104478Ssam 2293104478Ssam /* Enqueue and we're done... */ 2294115747Ssam mtx_lock(&sc->sc_mcr2lock); 2295104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2296104478Ssam ubsec_feed2(sc); 2297104478Ssam ubsecstats.hst_modexp++; 2298115747Ssam mtx_unlock(&sc->sc_mcr2lock); 2299104478Ssam 2300104478Ssam return (0); 2301104478Ssam 2302104478Ssamerrout: 2303104478Ssam if (me != NULL) { 2304267448Sjhb if (me->me_q.q_mcr.dma_tag != NULL) 2305104478Ssam ubsec_dma_free(sc, &me->me_q.q_mcr); 2306267448Sjhb if (me->me_q.q_ctx.dma_tag != NULL) { 2307104478Ssam bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2308104478Ssam ubsec_dma_free(sc, &me->me_q.q_ctx); 2309104478Ssam } 2310267448Sjhb if (me->me_M.dma_tag != NULL) { 2311104478Ssam bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2312104478Ssam ubsec_dma_free(sc, &me->me_M); 2313104478Ssam } 2314267448Sjhb if (me->me_E.dma_tag != NULL) { 2315104478Ssam bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2316104478Ssam ubsec_dma_free(sc, &me->me_E); 2317104478Ssam } 2318267448Sjhb if (me->me_C.dma_tag != NULL) { 2319104478Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2320104478Ssam ubsec_dma_free(sc, &me->me_C); 2321104478Ssam } 2322267448Sjhb if (me->me_epb.dma_tag != NULL) 2323104478Ssam ubsec_dma_free(sc, &me->me_epb); 2324104478Ssam free(me, M_DEVBUF); 2325104478Ssam } 2326104478Ssam krp->krp_status = err; 2327104478Ssam crypto_kdone(krp); 2328104478Ssam return (0); 2329104478Ssam} 2330104478Ssam 2331104630Ssam/* 2332104630Ssam * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization) 2333104630Ssam */ 2334105215Sphkstatic int 2335104630Ssamubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2336104630Ssam{ 2337104630Ssam struct ubsec_q2_modexp *me; 2338104630Ssam struct ubsec_mcr *mcr; 2339104630Ssam struct ubsec_ctx_modexp *ctx; 2340104630Ssam struct ubsec_pktbuf *epb; 2341104630Ssam int err = 0; 2342104630Ssam u_int nbits, normbits, mbits, shiftbits, ebits; 2343104630Ssam 2344104630Ssam me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2345104630Ssam if (me == NULL) { 2346104630Ssam err = ENOMEM; 2347104630Ssam goto errout; 2348104630Ssam } 2349104630Ssam bzero(me, sizeof *me); 2350104630Ssam me->me_krp = krp; 2351104630Ssam me->me_q.q_type = UBS_CTXOP_MODEXP; 2352104630Ssam 2353104630Ssam nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2354104630Ssam if (nbits <= 512) 2355104630Ssam normbits = 512; 2356104630Ssam else if (nbits <= 768) 2357104630Ssam normbits = 768; 2358104630Ssam else if (nbits <= 1024) 2359104630Ssam normbits = 1024; 2360104630Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2361104630Ssam normbits = 1536; 2362104630Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2363104630Ssam normbits = 2048; 2364104630Ssam else { 2365104630Ssam err = E2BIG; 2366104630Ssam goto errout; 2367104630Ssam } 2368104630Ssam 2369104630Ssam shiftbits = normbits - nbits; 2370104630Ssam 2371104630Ssam /* XXX ??? */ 2372104630Ssam me->me_modbits = nbits; 2373104630Ssam me->me_shiftbits = shiftbits; 2374104630Ssam me->me_normbits = normbits; 2375104630Ssam 2376104630Ssam /* Sanity check: result bits must be >= true modulus bits. */ 2377104630Ssam if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2378104630Ssam err = ERANGE; 2379104630Ssam goto errout; 2380104630Ssam } 2381104630Ssam 2382104630Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2383104630Ssam &me->me_q.q_mcr, 0)) { 2384104630Ssam err = ENOMEM; 2385104630Ssam goto errout; 2386104630Ssam } 2387104630Ssam mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2388104630Ssam 2389104630Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2390104630Ssam &me->me_q.q_ctx, 0)) { 2391104630Ssam err = ENOMEM; 2392104630Ssam goto errout; 2393104630Ssam } 2394104630Ssam 2395104630Ssam mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2396104630Ssam if (mbits > nbits) { 2397104630Ssam err = E2BIG; 2398104630Ssam goto errout; 2399104630Ssam } 2400104630Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2401104630Ssam err = ENOMEM; 2402104630Ssam goto errout; 2403104630Ssam } 2404104630Ssam bzero(me->me_M.dma_vaddr, normbits / 8); 2405104630Ssam bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p, 2406104630Ssam me->me_M.dma_vaddr, (mbits + 7) / 8); 2407104630Ssam 2408104630Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2409104630Ssam err = ENOMEM; 2410104630Ssam goto errout; 2411104630Ssam } 2412104630Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2413104630Ssam 2414104630Ssam ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2415104630Ssam if (ebits > nbits) { 2416104630Ssam err = E2BIG; 2417104630Ssam goto errout; 2418104630Ssam } 2419104630Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2420104630Ssam err = ENOMEM; 2421104630Ssam goto errout; 2422104630Ssam } 2423104630Ssam bzero(me->me_E.dma_vaddr, normbits / 8); 2424104630Ssam bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p, 2425104630Ssam me->me_E.dma_vaddr, (ebits + 7) / 8); 2426104630Ssam 2427104630Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2428104630Ssam &me->me_epb, 0)) { 2429104630Ssam err = ENOMEM; 2430104630Ssam goto errout; 2431104630Ssam } 2432104630Ssam epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2433104630Ssam epb->pb_addr = htole32(me->me_E.dma_paddr); 2434104630Ssam epb->pb_next = 0; 2435104630Ssam epb->pb_len = htole32((ebits + 7) / 8); 2436104630Ssam 2437104630Ssam#ifdef UBSEC_DEBUG 2438108823Ssam if (ubsec_debug) { 2439108823Ssam printf("Epb "); 2440108823Ssam ubsec_dump_pb(epb); 2441108823Ssam } 2442104630Ssam#endif 2443104630Ssam 2444104630Ssam mcr->mcr_pkts = htole16(1); 2445104630Ssam mcr->mcr_flags = 0; 2446104630Ssam mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2447104630Ssam mcr->mcr_reserved = 0; 2448104630Ssam mcr->mcr_pktlen = 0; 2449104630Ssam 2450104630Ssam mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2451104630Ssam mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2452104630Ssam mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2453104630Ssam 2454104630Ssam mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2455104630Ssam mcr->mcr_opktbuf.pb_next = 0; 2456104630Ssam mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2457104630Ssam 2458104630Ssam#ifdef DIAGNOSTIC 2459104630Ssam /* Misaligned output buffer will hang the chip. */ 2460104630Ssam if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2461104630Ssam panic("%s: modexp invalid addr 0x%x\n", 2462104630Ssam device_get_nameunit(sc->sc_dev), 2463104630Ssam letoh32(mcr->mcr_opktbuf.pb_addr)); 2464104630Ssam if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2465104630Ssam panic("%s: modexp invalid len 0x%x\n", 2466104630Ssam device_get_nameunit(sc->sc_dev), 2467104630Ssam letoh32(mcr->mcr_opktbuf.pb_len)); 2468104630Ssam#endif 2469104630Ssam 2470104630Ssam ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2471104630Ssam bzero(ctx, sizeof(*ctx)); 2472104630Ssam bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N, 2473104630Ssam (nbits + 7) / 8); 2474104630Ssam ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2475104630Ssam ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2476104630Ssam ctx->me_E_len = htole16(ebits); 2477104630Ssam ctx->me_N_len = htole16(nbits); 2478104630Ssam 2479104630Ssam#ifdef UBSEC_DEBUG 2480104630Ssam if (ubsec_debug) { 2481104630Ssam ubsec_dump_mcr(mcr); 2482104630Ssam ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2483104630Ssam } 2484104630Ssam#endif 2485104630Ssam 2486104630Ssam /* 2487104630Ssam * ubsec_feed2 will sync mcr and ctx, we just need to sync 2488104630Ssam * everything else. 2489104630Ssam */ 2490108823Ssam ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE); 2491108823Ssam ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE); 2492108823Ssam ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD); 2493108823Ssam ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE); 2494104630Ssam 2495104630Ssam /* Enqueue and we're done... */ 2496115747Ssam mtx_lock(&sc->sc_mcr2lock); 2497104630Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2498104630Ssam ubsec_feed2(sc); 2499115747Ssam mtx_unlock(&sc->sc_mcr2lock); 2500104630Ssam 2501104630Ssam return (0); 2502104630Ssam 2503104630Ssamerrout: 2504104630Ssam if (me != NULL) { 2505267448Sjhb if (me->me_q.q_mcr.dma_tag != NULL) 2506104630Ssam ubsec_dma_free(sc, &me->me_q.q_mcr); 2507267448Sjhb if (me->me_q.q_ctx.dma_tag != NULL) { 2508104630Ssam bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2509104630Ssam ubsec_dma_free(sc, &me->me_q.q_ctx); 2510104630Ssam } 2511267448Sjhb if (me->me_M.dma_tag != NULL) { 2512104630Ssam bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2513104630Ssam ubsec_dma_free(sc, &me->me_M); 2514104630Ssam } 2515267448Sjhb if (me->me_E.dma_tag != NULL) { 2516104630Ssam bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2517104630Ssam ubsec_dma_free(sc, &me->me_E); 2518104630Ssam } 2519267448Sjhb if (me->me_C.dma_tag != NULL) { 2520104630Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2521104630Ssam ubsec_dma_free(sc, &me->me_C); 2522104630Ssam } 2523267448Sjhb if (me->me_epb.dma_tag != NULL) 2524104630Ssam ubsec_dma_free(sc, &me->me_epb); 2525104630Ssam free(me, M_DEVBUF); 2526104630Ssam } 2527104630Ssam krp->krp_status = err; 2528104630Ssam crypto_kdone(krp); 2529104630Ssam return (0); 2530104630Ssam} 2531104630Ssam 2532104478Ssamstatic int 2533104478Ssamubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2534104478Ssam{ 2535104478Ssam struct ubsec_q2_rsapriv *rp = NULL; 2536104478Ssam struct ubsec_mcr *mcr; 2537104478Ssam struct ubsec_ctx_rsapriv *ctx; 2538104478Ssam int err = 0; 2539104478Ssam u_int padlen, msglen; 2540104478Ssam 2541104478Ssam msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]); 2542104478Ssam padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]); 2543104478Ssam if (msglen > padlen) 2544104478Ssam padlen = msglen; 2545104478Ssam 2546104478Ssam if (padlen <= 256) 2547104478Ssam padlen = 256; 2548104478Ssam else if (padlen <= 384) 2549104478Ssam padlen = 384; 2550104478Ssam else if (padlen <= 512) 2551104478Ssam padlen = 512; 2552104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768) 2553104478Ssam padlen = 768; 2554104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024) 2555104478Ssam padlen = 1024; 2556104478Ssam else { 2557104478Ssam err = E2BIG; 2558104478Ssam goto errout; 2559104478Ssam } 2560104478Ssam 2561104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) { 2562104478Ssam err = E2BIG; 2563104478Ssam goto errout; 2564104478Ssam } 2565104478Ssam 2566104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) { 2567104478Ssam err = E2BIG; 2568104478Ssam goto errout; 2569104478Ssam } 2570104478Ssam 2571104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) { 2572104478Ssam err = E2BIG; 2573104478Ssam goto errout; 2574104478Ssam } 2575104478Ssam 2576104478Ssam rp = (struct ubsec_q2_rsapriv *)malloc(sizeof *rp, M_DEVBUF, M_NOWAIT); 2577104478Ssam if (rp == NULL) 2578104478Ssam return (ENOMEM); 2579104478Ssam bzero(rp, sizeof *rp); 2580104478Ssam rp->rpr_krp = krp; 2581104478Ssam rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV; 2582104478Ssam 2583104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2584104478Ssam &rp->rpr_q.q_mcr, 0)) { 2585104478Ssam err = ENOMEM; 2586104478Ssam goto errout; 2587104478Ssam } 2588104478Ssam mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr; 2589104478Ssam 2590104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv), 2591104478Ssam &rp->rpr_q.q_ctx, 0)) { 2592104478Ssam err = ENOMEM; 2593104478Ssam goto errout; 2594104478Ssam } 2595104478Ssam ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr; 2596104478Ssam bzero(ctx, sizeof *ctx); 2597104478Ssam 2598104478Ssam /* Copy in p */ 2599104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p, 2600104478Ssam &ctx->rpr_buf[0 * (padlen / 8)], 2601104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8); 2602104478Ssam 2603104478Ssam /* Copy in q */ 2604104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p, 2605104478Ssam &ctx->rpr_buf[1 * (padlen / 8)], 2606104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8); 2607104478Ssam 2608104478Ssam /* Copy in dp */ 2609104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p, 2610104478Ssam &ctx->rpr_buf[2 * (padlen / 8)], 2611104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8); 2612104478Ssam 2613104478Ssam /* Copy in dq */ 2614104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p, 2615104478Ssam &ctx->rpr_buf[3 * (padlen / 8)], 2616104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8); 2617104478Ssam 2618104478Ssam /* Copy in pinv */ 2619104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p, 2620104478Ssam &ctx->rpr_buf[4 * (padlen / 8)], 2621104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8); 2622104478Ssam 2623104478Ssam msglen = padlen * 2; 2624104478Ssam 2625104478Ssam /* Copy in input message (aligned buffer/length). */ 2626104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) { 2627104478Ssam /* Is this likely? */ 2628104478Ssam err = E2BIG; 2629104478Ssam goto errout; 2630104478Ssam } 2631104478Ssam if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) { 2632104478Ssam err = ENOMEM; 2633104478Ssam goto errout; 2634104478Ssam } 2635104478Ssam bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8); 2636104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p, 2637104478Ssam rp->rpr_msgin.dma_vaddr, 2638104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8); 2639104478Ssam 2640104478Ssam /* Prepare space for output message (aligned buffer/length). */ 2641104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) { 2642104478Ssam /* Is this likely? */ 2643104478Ssam err = E2BIG; 2644104478Ssam goto errout; 2645104478Ssam } 2646104478Ssam if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) { 2647104478Ssam err = ENOMEM; 2648104478Ssam goto errout; 2649104478Ssam } 2650104478Ssam bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8); 2651104478Ssam 2652104478Ssam mcr->mcr_pkts = htole16(1); 2653104478Ssam mcr->mcr_flags = 0; 2654104478Ssam mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr); 2655104478Ssam mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr); 2656104478Ssam mcr->mcr_ipktbuf.pb_next = 0; 2657104478Ssam mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size); 2658104478Ssam mcr->mcr_reserved = 0; 2659104478Ssam mcr->mcr_pktlen = htole16(msglen); 2660104478Ssam mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr); 2661104478Ssam mcr->mcr_opktbuf.pb_next = 0; 2662104478Ssam mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size); 2663104478Ssam 2664104478Ssam#ifdef DIAGNOSTIC 2665104478Ssam if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) { 2666106579Sjhb panic("%s: rsapriv: invalid msgin %x(0x%jx)", 2667104478Ssam device_get_nameunit(sc->sc_dev), 2668106579Sjhb rp->rpr_msgin.dma_paddr, (uintmax_t)rp->rpr_msgin.dma_size); 2669104478Ssam } 2670104478Ssam if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) { 2671106579Sjhb panic("%s: rsapriv: invalid msgout %x(0x%jx)", 2672104478Ssam device_get_nameunit(sc->sc_dev), 2673106579Sjhb rp->rpr_msgout.dma_paddr, (uintmax_t)rp->rpr_msgout.dma_size); 2674104478Ssam } 2675104478Ssam#endif 2676104478Ssam 2677104478Ssam ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8)); 2678104478Ssam ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV); 2679104478Ssam ctx->rpr_q_len = htole16(padlen); 2680104478Ssam ctx->rpr_p_len = htole16(padlen); 2681104478Ssam 2682104478Ssam /* 2683104478Ssam * ubsec_feed2 will sync mcr and ctx, we just need to sync 2684104478Ssam * everything else. 2685104478Ssam */ 2686108823Ssam ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_PREWRITE); 2687108823Ssam ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_PREREAD); 2688104478Ssam 2689104478Ssam /* Enqueue and we're done... */ 2690115747Ssam mtx_lock(&sc->sc_mcr2lock); 2691104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next); 2692104478Ssam ubsec_feed2(sc); 2693104478Ssam ubsecstats.hst_modexpcrt++; 2694115747Ssam mtx_unlock(&sc->sc_mcr2lock); 2695104478Ssam return (0); 2696104478Ssam 2697104478Ssamerrout: 2698104478Ssam if (rp != NULL) { 2699267448Sjhb if (rp->rpr_q.q_mcr.dma_tag != NULL) 2700104478Ssam ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2701267448Sjhb if (rp->rpr_msgin.dma_tag != NULL) { 2702104478Ssam bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 2703104478Ssam ubsec_dma_free(sc, &rp->rpr_msgin); 2704104478Ssam } 2705267448Sjhb if (rp->rpr_msgout.dma_tag != NULL) { 2706104478Ssam bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 2707104478Ssam ubsec_dma_free(sc, &rp->rpr_msgout); 2708104478Ssam } 2709104478Ssam free(rp, M_DEVBUF); 2710104478Ssam } 2711104478Ssam krp->krp_status = err; 2712104478Ssam crypto_kdone(krp); 2713104478Ssam return (0); 2714104478Ssam} 2715104478Ssam 2716104478Ssam#ifdef UBSEC_DEBUG 2717104478Ssamstatic void 2718104478Ssamubsec_dump_pb(volatile struct ubsec_pktbuf *pb) 2719104478Ssam{ 2720104478Ssam printf("addr 0x%x (0x%x) next 0x%x\n", 2721104478Ssam pb->pb_addr, pb->pb_len, pb->pb_next); 2722104478Ssam} 2723104478Ssam 2724104478Ssamstatic void 2725104478Ssamubsec_dump_ctx2(struct ubsec_ctx_keyop *c) 2726104478Ssam{ 2727104478Ssam printf("CTX (0x%x):\n", c->ctx_len); 2728104478Ssam switch (letoh16(c->ctx_op)) { 2729104478Ssam case UBS_CTXOP_RNGBYPASS: 2730104478Ssam case UBS_CTXOP_RNGSHA1: 2731104478Ssam break; 2732104478Ssam case UBS_CTXOP_MODEXP: 2733104478Ssam { 2734104478Ssam struct ubsec_ctx_modexp *cx = (void *)c; 2735104478Ssam int i, len; 2736104478Ssam 2737104478Ssam printf(" Elen %u, Nlen %u\n", 2738104478Ssam letoh16(cx->me_E_len), letoh16(cx->me_N_len)); 2739104478Ssam len = (cx->me_N_len + 7)/8; 2740104478Ssam for (i = 0; i < len; i++) 2741104478Ssam printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]); 2742104478Ssam printf("\n"); 2743104478Ssam break; 2744104478Ssam } 2745104478Ssam default: 2746104478Ssam printf("unknown context: %x\n", c->ctx_op); 2747104478Ssam } 2748104478Ssam printf("END CTX\n"); 2749104478Ssam} 2750104478Ssam 2751104478Ssamstatic void 2752104478Ssamubsec_dump_mcr(struct ubsec_mcr *mcr) 2753104478Ssam{ 2754104478Ssam volatile struct ubsec_mcr_add *ma; 2755104478Ssam int i; 2756104478Ssam 2757104478Ssam printf("MCR:\n"); 2758104478Ssam printf(" pkts: %u, flags 0x%x\n", 2759104478Ssam letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags)); 2760104478Ssam ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp; 2761104478Ssam for (i = 0; i < letoh16(mcr->mcr_pkts); i++) { 2762104478Ssam printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i, 2763104478Ssam letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen), 2764104478Ssam letoh16(ma->mcr_reserved)); 2765104478Ssam printf(" %d: ipkt ", i); 2766104478Ssam ubsec_dump_pb(&ma->mcr_ipktbuf); 2767104478Ssam printf(" %d: opkt ", i); 2768104478Ssam ubsec_dump_pb(&ma->mcr_opktbuf); 2769104478Ssam ma++; 2770104478Ssam } 2771104478Ssam printf("END MCR\n"); 2772104478Ssam} 2773104478Ssam#endif /* UBSEC_DEBUG */ 2774104478Ssam 2775104478Ssam/* 2776104478Ssam * Return the number of significant bits of a big number. 2777104478Ssam */ 2778104478Ssamstatic int 2779104478Ssamubsec_ksigbits(struct crparam *cr) 2780104478Ssam{ 2781104478Ssam u_int plen = (cr->crp_nbits + 7) / 8; 2782104478Ssam int i, sig = plen * 8; 2783104478Ssam u_int8_t c, *p = cr->crp_p; 2784104478Ssam 2785104478Ssam for (i = plen - 1; i >= 0; i--) { 2786104478Ssam c = p[i]; 2787104478Ssam if (c != 0) { 2788104478Ssam while ((c & 0x80) == 0) { 2789104478Ssam sig--; 2790104478Ssam c <<= 1; 2791104478Ssam } 2792104478Ssam break; 2793104478Ssam } 2794104478Ssam sig -= 8; 2795104478Ssam } 2796104478Ssam return (sig); 2797104478Ssam} 2798104478Ssam 2799104478Ssamstatic void 2800104478Ssamubsec_kshift_r( 2801104478Ssam u_int shiftbits, 2802104478Ssam u_int8_t *src, u_int srcbits, 2803104478Ssam u_int8_t *dst, u_int dstbits) 2804104478Ssam{ 2805104478Ssam u_int slen, dlen; 2806104478Ssam int i, si, di, n; 2807104478Ssam 2808104478Ssam slen = (srcbits + 7) / 8; 2809104478Ssam dlen = (dstbits + 7) / 8; 2810104478Ssam 2811104478Ssam for (i = 0; i < slen; i++) 2812104478Ssam dst[i] = src[i]; 2813104478Ssam for (i = 0; i < dlen - slen; i++) 2814104478Ssam dst[slen + i] = 0; 2815104478Ssam 2816104478Ssam n = shiftbits / 8; 2817104478Ssam if (n != 0) { 2818104478Ssam si = dlen - n - 1; 2819104478Ssam di = dlen - 1; 2820104478Ssam while (si >= 0) 2821104478Ssam dst[di--] = dst[si--]; 2822104478Ssam while (di >= 0) 2823104478Ssam dst[di--] = 0; 2824104478Ssam } 2825104478Ssam 2826104478Ssam n = shiftbits % 8; 2827104478Ssam if (n != 0) { 2828104478Ssam for (i = dlen - 1; i > 0; i--) 2829104478Ssam dst[i] = (dst[i] << n) | 2830104478Ssam (dst[i - 1] >> (8 - n)); 2831104478Ssam dst[0] = dst[0] << n; 2832104478Ssam } 2833104478Ssam} 2834104478Ssam 2835104478Ssamstatic void 2836104478Ssamubsec_kshift_l( 2837104478Ssam u_int shiftbits, 2838104478Ssam u_int8_t *src, u_int srcbits, 2839104478Ssam u_int8_t *dst, u_int dstbits) 2840104478Ssam{ 2841104478Ssam int slen, dlen, i, n; 2842104478Ssam 2843104478Ssam slen = (srcbits + 7) / 8; 2844104478Ssam dlen = (dstbits + 7) / 8; 2845104478Ssam 2846104478Ssam n = shiftbits / 8; 2847104478Ssam for (i = 0; i < slen; i++) 2848104478Ssam dst[i] = src[i + n]; 2849104478Ssam for (i = 0; i < dlen - slen; i++) 2850104478Ssam dst[slen + i] = 0; 2851104478Ssam 2852104478Ssam n = shiftbits % 8; 2853104478Ssam if (n != 0) { 2854104478Ssam for (i = 0; i < (dlen - 1); i++) 2855104478Ssam dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n)); 2856104478Ssam dst[dlen - 1] = dst[dlen - 1] >> n; 2857104478Ssam } 2858104478Ssam} 2859