ubsec.c revision 243857
1104630Ssam/* $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $ */ 2104478Ssam 3139749Simp/*- 4104478Ssam * Copyright (c) 2000 Jason L. Wright (jason@thought.net) 5104478Ssam * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org) 6104478Ssam * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com) 7162969Sjhb * 8104478Ssam * All rights reserved. 9104478Ssam * 10104478Ssam * Redistribution and use in source and binary forms, with or without 11104478Ssam * modification, are permitted provided that the following conditions 12104478Ssam * are met: 13104478Ssam * 1. Redistributions of source code must retain the above copyright 14104478Ssam * notice, this list of conditions and the following disclaimer. 15104478Ssam * 2. Redistributions in binary form must reproduce the above copyright 16104478Ssam * notice, this list of conditions and the following disclaimer in the 17104478Ssam * documentation and/or other materials provided with the distribution. 18104478Ssam * 3. All advertising materials mentioning features or use of this software 19104478Ssam * must display the following acknowledgement: 20104478Ssam * This product includes software developed by Jason L. Wright 21104478Ssam * 4. The name of the author may not be used to endorse or promote products 22104478Ssam * derived from this software without specific prior written permission. 23104478Ssam * 24104478Ssam * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 25104478Ssam * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 26104478Ssam * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 27104478Ssam * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 28104478Ssam * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 29104478Ssam * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 30104478Ssam * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31104478Ssam * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 32104478Ssam * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 33104478Ssam * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34104478Ssam * POSSIBILITY OF SUCH DAMAGE. 35104478Ssam * 36104478Ssam * Effort sponsored in part by the Defense Advanced Research Projects 37104478Ssam * Agency (DARPA) and Air Force Research Laboratory, Air Force 38104478Ssam * Materiel Command, USAF, under agreement number F30602-01-2-0537. 39104478Ssam */ 40104478Ssam 41119418Sobrien#include <sys/cdefs.h> 42119418Sobrien__FBSDID("$FreeBSD: head/sys/dev/ubsec/ubsec.c 243857 2012-12-04 09:32:43Z glebius $"); 43119418Sobrien 44104478Ssam/* 45104478Ssam * uBsec 5[56]01, 58xx hardware crypto accelerator 46104478Ssam */ 47104478Ssam 48112124Ssam#include "opt_ubsec.h" 49112124Ssam 50104478Ssam#include <sys/param.h> 51104478Ssam#include <sys/systm.h> 52104478Ssam#include <sys/proc.h> 53104478Ssam#include <sys/errno.h> 54104478Ssam#include <sys/malloc.h> 55104478Ssam#include <sys/kernel.h> 56129879Sphk#include <sys/module.h> 57104478Ssam#include <sys/mbuf.h> 58104478Ssam#include <sys/lock.h> 59104478Ssam#include <sys/mutex.h> 60104478Ssam#include <sys/sysctl.h> 61104478Ssam#include <sys/endian.h> 62104478Ssam 63104478Ssam#include <vm/vm.h> 64104478Ssam#include <vm/pmap.h> 65104478Ssam 66104478Ssam#include <machine/bus.h> 67104478Ssam#include <machine/resource.h> 68104478Ssam#include <sys/bus.h> 69104478Ssam#include <sys/rman.h> 70104478Ssam 71104478Ssam#include <crypto/sha1.h> 72104478Ssam#include <opencrypto/cryptodev.h> 73104478Ssam#include <opencrypto/cryptosoft.h> 74104478Ssam#include <sys/md5.h> 75104478Ssam#include <sys/random.h> 76167755Ssam#include <sys/kobj.h> 77104478Ssam 78167755Ssam#include "cryptodev_if.h" 79167755Ssam 80119287Simp#include <dev/pci/pcivar.h> 81119287Simp#include <dev/pci/pcireg.h> 82104478Ssam 83104478Ssam/* grr, #defines for gratuitous incompatibility in queue.h */ 84104478Ssam#define SIMPLEQ_HEAD STAILQ_HEAD 85104478Ssam#define SIMPLEQ_ENTRY STAILQ_ENTRY 86104478Ssam#define SIMPLEQ_INIT STAILQ_INIT 87104478Ssam#define SIMPLEQ_INSERT_TAIL STAILQ_INSERT_TAIL 88104478Ssam#define SIMPLEQ_EMPTY STAILQ_EMPTY 89104478Ssam#define SIMPLEQ_FIRST STAILQ_FIRST 90163648Sru#define SIMPLEQ_REMOVE_HEAD STAILQ_REMOVE_HEAD 91111416Ssam#define SIMPLEQ_FOREACH STAILQ_FOREACH 92104478Ssam/* ditto for endian.h */ 93104478Ssam#define letoh16(x) le16toh(x) 94104478Ssam#define letoh32(x) le32toh(x) 95104478Ssam 96112124Ssam#ifdef UBSEC_RNDTEST 97112124Ssam#include <dev/rndtest/rndtest.h> 98112124Ssam#endif 99104478Ssam#include <dev/ubsec/ubsecreg.h> 100104478Ssam#include <dev/ubsec/ubsecvar.h> 101104478Ssam 102104478Ssam/* 103104478Ssam * Prototypes and count for the pci_device structure 104104478Ssam */ 105104478Ssamstatic int ubsec_probe(device_t); 106104478Ssamstatic int ubsec_attach(device_t); 107104478Ssamstatic int ubsec_detach(device_t); 108104478Ssamstatic int ubsec_suspend(device_t); 109104478Ssamstatic int ubsec_resume(device_t); 110194023Savgstatic int ubsec_shutdown(device_t); 111104478Ssam 112167755Ssamstatic int ubsec_newsession(device_t, u_int32_t *, struct cryptoini *); 113167755Ssamstatic int ubsec_freesession(device_t, u_int64_t); 114167755Ssamstatic int ubsec_process(device_t, struct cryptop *, int); 115167755Ssamstatic int ubsec_kprocess(device_t, struct cryptkop *, int); 116167755Ssam 117104478Ssamstatic device_method_t ubsec_methods[] = { 118104478Ssam /* Device interface */ 119104478Ssam DEVMETHOD(device_probe, ubsec_probe), 120104478Ssam DEVMETHOD(device_attach, ubsec_attach), 121104478Ssam DEVMETHOD(device_detach, ubsec_detach), 122104478Ssam DEVMETHOD(device_suspend, ubsec_suspend), 123104478Ssam DEVMETHOD(device_resume, ubsec_resume), 124104478Ssam DEVMETHOD(device_shutdown, ubsec_shutdown), 125104478Ssam 126167755Ssam /* crypto device methods */ 127167755Ssam DEVMETHOD(cryptodev_newsession, ubsec_newsession), 128167755Ssam DEVMETHOD(cryptodev_freesession,ubsec_freesession), 129167755Ssam DEVMETHOD(cryptodev_process, ubsec_process), 130167755Ssam DEVMETHOD(cryptodev_kprocess, ubsec_kprocess), 131167755Ssam 132227843Smarius DEVMETHOD_END 133104478Ssam}; 134104478Ssamstatic driver_t ubsec_driver = { 135104478Ssam "ubsec", 136104478Ssam ubsec_methods, 137104478Ssam sizeof (struct ubsec_softc) 138104478Ssam}; 139104478Ssamstatic devclass_t ubsec_devclass; 140104478Ssam 141104478SsamDRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0); 142105251SmarkmMODULE_DEPEND(ubsec, crypto, 1, 1, 1); 143112124Ssam#ifdef UBSEC_RNDTEST 144112124SsamMODULE_DEPEND(ubsec, rndtest, 1, 1, 1); 145112124Ssam#endif 146104478Ssam 147104478Ssamstatic void ubsec_intr(void *); 148104478Ssamstatic void ubsec_callback(struct ubsec_softc *, struct ubsec_q *); 149111416Ssamstatic void ubsec_feed(struct ubsec_softc *); 150104478Ssamstatic void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int); 151104478Ssamstatic void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *); 152104478Ssamstatic int ubsec_feed2(struct ubsec_softc *); 153104478Ssamstatic void ubsec_rng(void *); 154104478Ssamstatic int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t, 155104478Ssam struct ubsec_dma_alloc *, int); 156108823Ssam#define ubsec_dma_sync(_dma, _flags) \ 157108823Ssam bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags)) 158104478Ssamstatic void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *); 159104478Ssamstatic int ubsec_dmamap_aligned(struct ubsec_operand *op); 160104478Ssam 161104478Ssamstatic void ubsec_reset_board(struct ubsec_softc *sc); 162104478Ssamstatic void ubsec_init_board(struct ubsec_softc *sc); 163104478Ssamstatic void ubsec_init_pciregs(device_t dev); 164104478Ssamstatic void ubsec_totalreset(struct ubsec_softc *sc); 165104478Ssam 166104478Ssamstatic int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q); 167104478Ssam 168104630Ssamstatic int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int); 169104630Ssamstatic int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int); 170104478Ssamstatic int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int); 171104478Ssamstatic void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *); 172104478Ssamstatic int ubsec_ksigbits(struct crparam *); 173104478Ssamstatic void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 174104478Ssamstatic void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 175104478Ssam 176227309Sedstatic SYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0, 177227309Sed "Broadcom driver parameters"); 178109595Ssam 179104478Ssam#ifdef UBSEC_DEBUG 180104478Ssamstatic void ubsec_dump_pb(volatile struct ubsec_pktbuf *); 181104478Ssamstatic void ubsec_dump_mcr(struct ubsec_mcr *); 182104478Ssamstatic void ubsec_dump_ctx2(struct ubsec_ctx_keyop *); 183104478Ssam 184104478Ssamstatic int ubsec_debug = 0; 185109595SsamSYSCTL_INT(_hw_ubsec, OID_AUTO, debug, CTLFLAG_RW, &ubsec_debug, 186109595Ssam 0, "control debugging msgs"); 187104478Ssam#endif 188104478Ssam 189104478Ssam#define READ_REG(sc,r) \ 190104478Ssam bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r)) 191104478Ssam 192104478Ssam#define WRITE_REG(sc,reg,val) \ 193104478Ssam bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val) 194104478Ssam 195104478Ssam#define SWAP32(x) (x) = htole32(ntohl((x))) 196104478Ssam#define HTOLE32(x) (x) = htole32(x) 197104478Ssam 198104478Ssamstruct ubsec_stats ubsecstats; 199109595SsamSYSCTL_STRUCT(_hw_ubsec, OID_AUTO, stats, CTLFLAG_RD, &ubsecstats, 200109595Ssam ubsec_stats, "driver statistics"); 201104478Ssam 202105215Sphkstatic int 203104478Ssamubsec_probe(device_t dev) 204104478Ssam{ 205114105Ssam if (pci_get_vendor(dev) == PCI_VENDOR_SUN && 206114105Ssam (pci_get_device(dev) == PCI_PRODUCT_SUN_5821 || 207114105Ssam pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K)) 208142880Simp return (BUS_PROBE_DEFAULT); 209104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 210104478Ssam (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 || 211104478Ssam pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)) 212142880Simp return (BUS_PROBE_DEFAULT); 213104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 214111646Ssam (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5801 || 215111646Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 || 216111646Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 || 217104478Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 || 218104478Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 219110522Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 || 220191894Sphilip pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 || 221191894Sphilip pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5825 222110522Ssam )) 223142880Simp return (BUS_PROBE_DEFAULT); 224104478Ssam return (ENXIO); 225104478Ssam} 226104478Ssam 227104478Ssamstatic const char* 228104478Ssamubsec_partname(struct ubsec_softc *sc) 229104478Ssam{ 230104478Ssam /* XXX sprintf numbers when not decoded */ 231104478Ssam switch (pci_get_vendor(sc->sc_dev)) { 232104478Ssam case PCI_VENDOR_BROADCOM: 233104478Ssam switch (pci_get_device(sc->sc_dev)) { 234111646Ssam case PCI_PRODUCT_BROADCOM_5801: return "Broadcom 5801"; 235111646Ssam case PCI_PRODUCT_BROADCOM_5802: return "Broadcom 5802"; 236104478Ssam case PCI_PRODUCT_BROADCOM_5805: return "Broadcom 5805"; 237104478Ssam case PCI_PRODUCT_BROADCOM_5820: return "Broadcom 5820"; 238104478Ssam case PCI_PRODUCT_BROADCOM_5821: return "Broadcom 5821"; 239104478Ssam case PCI_PRODUCT_BROADCOM_5822: return "Broadcom 5822"; 240110522Ssam case PCI_PRODUCT_BROADCOM_5823: return "Broadcom 5823"; 241191894Sphilip case PCI_PRODUCT_BROADCOM_5825: return "Broadcom 5825"; 242104478Ssam } 243104478Ssam return "Broadcom unknown-part"; 244104478Ssam case PCI_VENDOR_BLUESTEEL: 245104478Ssam switch (pci_get_device(sc->sc_dev)) { 246104478Ssam case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601"; 247104478Ssam } 248104478Ssam return "Bluesteel unknown-part"; 249114105Ssam case PCI_VENDOR_SUN: 250114105Ssam switch (pci_get_device(sc->sc_dev)) { 251114105Ssam case PCI_PRODUCT_SUN_5821: return "Sun Crypto 5821"; 252114105Ssam case PCI_PRODUCT_SUN_SCA1K: return "Sun Crypto 1K"; 253114105Ssam } 254114105Ssam return "Sun unknown-part"; 255104478Ssam } 256104478Ssam return "Unknown-vendor unknown-part"; 257104478Ssam} 258104478Ssam 259112124Ssamstatic void 260112124Ssamdefault_harvest(struct rndtest_state *rsp, void *buf, u_int count) 261112124Ssam{ 262112124Ssam random_harvest(buf, count, count*NBBY, 0, RANDOM_PURE); 263112124Ssam} 264112124Ssam 265104478Ssamstatic int 266104478Ssamubsec_attach(device_t dev) 267104478Ssam{ 268104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 269104478Ssam struct ubsec_dma *dmap; 270104478Ssam u_int32_t cmd, i; 271104478Ssam int rid; 272104478Ssam 273104478Ssam bzero(sc, sizeof (*sc)); 274104478Ssam sc->sc_dev = dev; 275104478Ssam 276104478Ssam SIMPLEQ_INIT(&sc->sc_queue); 277104478Ssam SIMPLEQ_INIT(&sc->sc_qchip); 278104478Ssam SIMPLEQ_INIT(&sc->sc_queue2); 279104478Ssam SIMPLEQ_INIT(&sc->sc_qchip2); 280104478Ssam SIMPLEQ_INIT(&sc->sc_q2free); 281104478Ssam 282104478Ssam /* XXX handle power management */ 283104478Ssam 284104478Ssam sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR; 285104478Ssam 286104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 287104478Ssam pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601) 288104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 289104478Ssam 290104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 291111646Ssam (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 || 292111646Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805)) 293104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 294104478Ssam 295104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 296104478Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820) 297104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 298104478Ssam UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 299104478Ssam 300114105Ssam if ((pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 301114105Ssam (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 302114105Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 || 303191894Sphilip pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 || 304191894Sphilip pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5825)) || 305114105Ssam (pci_get_vendor(dev) == PCI_VENDOR_SUN && 306114105Ssam (pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K || 307114105Ssam pci_get_device(dev) == PCI_PRODUCT_SUN_5821))) { 308104478Ssam /* NB: the 5821/5822 defines some additional status bits */ 309104478Ssam sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY | 310104478Ssam BS_STAT_MCR2_ALLEMPTY; 311104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 312104478Ssam UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 313104478Ssam } 314162969Sjhb 315104478Ssam cmd = pci_read_config(dev, PCIR_COMMAND, 4); 316104478Ssam cmd |= PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN; 317104478Ssam pci_write_config(dev, PCIR_COMMAND, cmd, 4); 318104478Ssam cmd = pci_read_config(dev, PCIR_COMMAND, 4); 319104478Ssam 320104478Ssam if (!(cmd & PCIM_CMD_MEMEN)) { 321104478Ssam device_printf(dev, "failed to enable memory mapping\n"); 322104478Ssam goto bad; 323104478Ssam } 324104478Ssam 325104478Ssam if (!(cmd & PCIM_CMD_BUSMASTEREN)) { 326104478Ssam device_printf(dev, "failed to enable bus mastering\n"); 327104478Ssam goto bad; 328104478Ssam } 329104478Ssam 330162969Sjhb /* 331104478Ssam * Setup memory-mapping of PCI registers. 332104478Ssam */ 333104478Ssam rid = BS_BAR; 334127135Snjl sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 335127135Snjl RF_ACTIVE); 336104478Ssam if (sc->sc_sr == NULL) { 337104478Ssam device_printf(dev, "cannot map register space\n"); 338104478Ssam goto bad; 339104478Ssam } 340104478Ssam sc->sc_st = rman_get_bustag(sc->sc_sr); 341104478Ssam sc->sc_sh = rman_get_bushandle(sc->sc_sr); 342104478Ssam 343104478Ssam /* 344104478Ssam * Arrange interrupt line. 345104478Ssam */ 346104478Ssam rid = 0; 347127135Snjl sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 348127135Snjl RF_SHAREABLE|RF_ACTIVE); 349104478Ssam if (sc->sc_irq == NULL) { 350104478Ssam device_printf(dev, "could not map interrupt\n"); 351108823Ssam goto bad1; 352104478Ssam } 353104478Ssam /* 354104478Ssam * NB: Network code assumes we are blocked with splimp() 355104478Ssam * so make sure the IRQ is mapped appropriately. 356104478Ssam */ 357115747Ssam if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE, 358166901Spiso NULL, ubsec_intr, sc, &sc->sc_ih)) { 359104478Ssam device_printf(dev, "could not establish interrupt\n"); 360108823Ssam goto bad2; 361104478Ssam } 362104478Ssam 363167755Ssam sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); 364104478Ssam if (sc->sc_cid < 0) { 365104478Ssam device_printf(dev, "could not get crypto driver id\n"); 366108823Ssam goto bad3; 367104478Ssam } 368104478Ssam 369104478Ssam /* 370104478Ssam * Setup DMA descriptor area. 371104478Ssam */ 372232874Sscottl if (bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 373104478Ssam 1, 0, /* alignment, bounds */ 374104478Ssam BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 375104478Ssam BUS_SPACE_MAXADDR, /* highaddr */ 376104478Ssam NULL, NULL, /* filter, filterarg */ 377108823Ssam 0x3ffff, /* maxsize */ 378104478Ssam UBS_MAX_SCATTER, /* nsegments */ 379108823Ssam 0xffff, /* maxsegsize */ 380104478Ssam BUS_DMA_ALLOCNOW, /* flags */ 381117126Sscottl NULL, NULL, /* lockfunc, lockarg */ 382104478Ssam &sc->sc_dmat)) { 383104478Ssam device_printf(dev, "cannot allocate DMA tag\n"); 384108823Ssam goto bad4; 385104478Ssam } 386104478Ssam SIMPLEQ_INIT(&sc->sc_freequeue); 387104478Ssam dmap = sc->sc_dmaa; 388104478Ssam for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) { 389104478Ssam struct ubsec_q *q; 390104478Ssam 391104478Ssam q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q), 392104478Ssam M_DEVBUF, M_NOWAIT); 393104478Ssam if (q == NULL) { 394104478Ssam device_printf(dev, "cannot allocate queue buffers\n"); 395104478Ssam break; 396104478Ssam } 397104478Ssam 398104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk), 399104478Ssam &dmap->d_alloc, 0)) { 400104478Ssam device_printf(dev, "cannot allocate dma buffers\n"); 401104478Ssam free(q, M_DEVBUF); 402104478Ssam break; 403104478Ssam } 404104478Ssam dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr; 405104478Ssam 406104478Ssam q->q_dma = dmap; 407104478Ssam sc->sc_queuea[i] = q; 408104478Ssam 409104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 410104478Ssam } 411115747Ssam mtx_init(&sc->sc_mcr1lock, device_get_nameunit(dev), 412115747Ssam "mcr1 operations", MTX_DEF); 413115747Ssam mtx_init(&sc->sc_freeqlock, device_get_nameunit(dev), 414115747Ssam "mcr1 free q", MTX_DEF); 415104478Ssam 416104478Ssam device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc)); 417104478Ssam 418167755Ssam crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); 419167755Ssam crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); 420167755Ssam crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 421167755Ssam crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 422104478Ssam 423104478Ssam /* 424104478Ssam * Reset Broadcom chip 425104478Ssam */ 426104478Ssam ubsec_reset_board(sc); 427104478Ssam 428104478Ssam /* 429104478Ssam * Init Broadcom specific PCI settings 430104478Ssam */ 431104478Ssam ubsec_init_pciregs(dev); 432104478Ssam 433104478Ssam /* 434104478Ssam * Init Broadcom chip 435104478Ssam */ 436104478Ssam ubsec_init_board(sc); 437104478Ssam 438104478Ssam#ifndef UBSEC_NO_RNG 439104478Ssam if (sc->sc_flags & UBS_FLAGS_RNG) { 440104478Ssam sc->sc_statmask |= BS_STAT_MCR2_DONE; 441112124Ssam#ifdef UBSEC_RNDTEST 442112124Ssam sc->sc_rndtest = rndtest_attach(dev); 443112124Ssam if (sc->sc_rndtest) 444112124Ssam sc->sc_harvest = rndtest_harvest; 445112124Ssam else 446112124Ssam sc->sc_harvest = default_harvest; 447112124Ssam#else 448112124Ssam sc->sc_harvest = default_harvest; 449112124Ssam#endif 450104478Ssam 451104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 452104478Ssam &sc->sc_rng.rng_q.q_mcr, 0)) 453104478Ssam goto skip_rng; 454104478Ssam 455104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass), 456104478Ssam &sc->sc_rng.rng_q.q_ctx, 0)) { 457104478Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 458104478Ssam goto skip_rng; 459104478Ssam } 460104478Ssam 461104478Ssam if (ubsec_dma_malloc(sc, sizeof(u_int32_t) * 462104478Ssam UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) { 463104478Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 464104478Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 465104478Ssam goto skip_rng; 466104478Ssam } 467104478Ssam 468104478Ssam if (hz >= 100) 469104478Ssam sc->sc_rnghz = hz / 100; 470104478Ssam else 471104478Ssam sc->sc_rnghz = 1; 472119137Ssam callout_init(&sc->sc_rngto, CALLOUT_MPSAFE); 473104478Ssam callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 474104478Ssamskip_rng: 475104478Ssam ; 476104478Ssam } 477104478Ssam#endif /* UBSEC_NO_RNG */ 478115747Ssam mtx_init(&sc->sc_mcr2lock, device_get_nameunit(dev), 479115747Ssam "mcr2 operations", MTX_DEF); 480104478Ssam 481104478Ssam if (sc->sc_flags & UBS_FLAGS_KEY) { 482104478Ssam sc->sc_statmask |= BS_STAT_MCR2_DONE; 483104478Ssam 484167755Ssam crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0); 485104630Ssam#if 0 486167755Ssam crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0); 487104630Ssam#endif 488104478Ssam } 489104478Ssam return (0); 490108823Ssambad4: 491108823Ssam crypto_unregister_all(sc->sc_cid); 492108823Ssambad3: 493108823Ssam bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 494108823Ssambad2: 495108823Ssam bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 496108823Ssambad1: 497108823Ssam bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 498104478Ssambad: 499104478Ssam return (ENXIO); 500104478Ssam} 501104478Ssam 502104478Ssam/* 503104478Ssam * Detach a device that successfully probed. 504104478Ssam */ 505104478Ssamstatic int 506104478Ssamubsec_detach(device_t dev) 507104478Ssam{ 508104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 509104478Ssam 510108823Ssam /* XXX wait/abort active ops */ 511108823Ssam 512115747Ssam /* disable interrupts */ 513115747Ssam WRITE_REG(sc, BS_CTRL, READ_REG(sc, BS_CTRL) &~ 514115747Ssam (BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR)); 515104478Ssam 516104478Ssam callout_stop(&sc->sc_rngto); 517104478Ssam 518104478Ssam crypto_unregister_all(sc->sc_cid); 519104478Ssam 520112124Ssam#ifdef UBSEC_RNDTEST 521112124Ssam if (sc->sc_rndtest) 522112124Ssam rndtest_detach(sc->sc_rndtest); 523112124Ssam#endif 524112124Ssam 525108823Ssam while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 526108823Ssam struct ubsec_q *q; 527108823Ssam 528108823Ssam q = SIMPLEQ_FIRST(&sc->sc_freequeue); 529163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next); 530108823Ssam ubsec_dma_free(sc, &q->q_dma->d_alloc); 531108823Ssam free(q, M_DEVBUF); 532108823Ssam } 533115747Ssam mtx_destroy(&sc->sc_mcr1lock); 534159224Spjd mtx_destroy(&sc->sc_freeqlock); 535108823Ssam#ifndef UBSEC_NO_RNG 536108823Ssam if (sc->sc_flags & UBS_FLAGS_RNG) { 537108823Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 538108823Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 539108823Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_buf); 540108823Ssam } 541108823Ssam#endif /* UBSEC_NO_RNG */ 542115747Ssam mtx_destroy(&sc->sc_mcr2lock); 543108823Ssam 544104478Ssam bus_generic_detach(dev); 545104478Ssam bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 546104478Ssam bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 547104478Ssam 548104478Ssam bus_dma_tag_destroy(sc->sc_dmat); 549104478Ssam bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 550104478Ssam 551104478Ssam return (0); 552104478Ssam} 553104478Ssam 554104478Ssam/* 555104478Ssam * Stop all chip i/o so that the kernel's probe routines don't 556104478Ssam * get confused by errant DMAs when rebooting. 557104478Ssam */ 558194023Savgstatic int 559104478Ssamubsec_shutdown(device_t dev) 560104478Ssam{ 561104478Ssam#ifdef notyet 562104478Ssam ubsec_stop(device_get_softc(dev)); 563104478Ssam#endif 564194023Savg return (0); 565104478Ssam} 566104478Ssam 567104478Ssam/* 568104478Ssam * Device suspend routine. 569104478Ssam */ 570104478Ssamstatic int 571104478Ssamubsec_suspend(device_t dev) 572104478Ssam{ 573104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 574104478Ssam 575104478Ssam#ifdef notyet 576104478Ssam /* XXX stop the device and save PCI settings */ 577104478Ssam#endif 578104478Ssam sc->sc_suspended = 1; 579104478Ssam 580104478Ssam return (0); 581104478Ssam} 582104478Ssam 583104478Ssamstatic int 584104478Ssamubsec_resume(device_t dev) 585104478Ssam{ 586104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 587104478Ssam 588104478Ssam#ifdef notyet 589104478Ssam /* XXX retore PCI settings and start the device */ 590104478Ssam#endif 591104478Ssam sc->sc_suspended = 0; 592104478Ssam return (0); 593104478Ssam} 594104478Ssam 595104478Ssam/* 596104478Ssam * UBSEC Interrupt routine 597104478Ssam */ 598104478Ssamstatic void 599104478Ssamubsec_intr(void *arg) 600104478Ssam{ 601104478Ssam struct ubsec_softc *sc = arg; 602104478Ssam volatile u_int32_t stat; 603104478Ssam struct ubsec_q *q; 604104478Ssam struct ubsec_dma *dmap; 605104478Ssam int npkts = 0, i; 606104478Ssam 607104478Ssam stat = READ_REG(sc, BS_STAT); 608104478Ssam stat &= sc->sc_statmask; 609115747Ssam if (stat == 0) 610104478Ssam return; 611104478Ssam 612104478Ssam WRITE_REG(sc, BS_STAT, stat); /* IACK */ 613104478Ssam 614104478Ssam /* 615104478Ssam * Check to see if we have any packets waiting for us 616104478Ssam */ 617104478Ssam if ((stat & BS_STAT_MCR1_DONE)) { 618115747Ssam mtx_lock(&sc->sc_mcr1lock); 619104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 620104478Ssam q = SIMPLEQ_FIRST(&sc->sc_qchip); 621104478Ssam dmap = q->q_dma; 622104478Ssam 623104478Ssam if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0) 624104478Ssam break; 625104478Ssam 626163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next); 627104478Ssam 628104478Ssam npkts = q->q_nstacked_mcrs; 629108471Ssam sc->sc_nqchip -= 1+npkts; 630104478Ssam /* 631104478Ssam * search for further sc_qchip ubsec_q's that share 632104478Ssam * the same MCR, and complete them too, they must be 633104478Ssam * at the top. 634104478Ssam */ 635104478Ssam for (i = 0; i < npkts; i++) { 636104478Ssam if(q->q_stacked_mcr[i]) { 637104478Ssam ubsec_callback(sc, q->q_stacked_mcr[i]); 638104478Ssam } else { 639104478Ssam break; 640104478Ssam } 641104478Ssam } 642104478Ssam ubsec_callback(sc, q); 643104478Ssam } 644104478Ssam /* 645104478Ssam * Don't send any more packet to chip if there has been 646104478Ssam * a DMAERR. 647104478Ssam */ 648104478Ssam if (!(stat & BS_STAT_DMAERR)) 649104478Ssam ubsec_feed(sc); 650115747Ssam mtx_unlock(&sc->sc_mcr1lock); 651104478Ssam } 652104478Ssam 653104478Ssam /* 654104478Ssam * Check to see if we have any key setups/rng's waiting for us 655104478Ssam */ 656104478Ssam if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) && 657104478Ssam (stat & BS_STAT_MCR2_DONE)) { 658104478Ssam struct ubsec_q2 *q2; 659104478Ssam struct ubsec_mcr *mcr; 660104478Ssam 661115747Ssam mtx_lock(&sc->sc_mcr2lock); 662104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) { 663104478Ssam q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 664104478Ssam 665108823Ssam ubsec_dma_sync(&q2->q_mcr, 666104478Ssam BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 667104478Ssam 668104478Ssam mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 669104478Ssam if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) { 670108823Ssam ubsec_dma_sync(&q2->q_mcr, 671104478Ssam BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 672104478Ssam break; 673104478Ssam } 674163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q_next); 675104478Ssam ubsec_callback2(sc, q2); 676104478Ssam /* 677104478Ssam * Don't send any more packet to chip if there has been 678104478Ssam * a DMAERR. 679104478Ssam */ 680104478Ssam if (!(stat & BS_STAT_DMAERR)) 681104478Ssam ubsec_feed2(sc); 682104478Ssam } 683115747Ssam mtx_unlock(&sc->sc_mcr2lock); 684104478Ssam } 685104478Ssam 686104478Ssam /* 687104478Ssam * Check to see if we got any DMA Error 688104478Ssam */ 689104478Ssam if (stat & BS_STAT_DMAERR) { 690104478Ssam#ifdef UBSEC_DEBUG 691104478Ssam if (ubsec_debug) { 692104478Ssam volatile u_int32_t a = READ_REG(sc, BS_ERR); 693104478Ssam 694104478Ssam printf("dmaerr %s@%08x\n", 695104478Ssam (a & BS_ERR_READ) ? "read" : "write", 696104478Ssam a & BS_ERR_ADDR); 697104478Ssam } 698104478Ssam#endif /* UBSEC_DEBUG */ 699104478Ssam ubsecstats.hst_dmaerr++; 700115747Ssam mtx_lock(&sc->sc_mcr1lock); 701104478Ssam ubsec_totalreset(sc); 702104478Ssam ubsec_feed(sc); 703115747Ssam mtx_unlock(&sc->sc_mcr1lock); 704104478Ssam } 705104478Ssam 706104478Ssam if (sc->sc_needwakeup) { /* XXX check high watermark */ 707158828Spjd int wakeup; 708158828Spjd 709158828Spjd mtx_lock(&sc->sc_freeqlock); 710158828Spjd wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ); 711104478Ssam#ifdef UBSEC_DEBUG 712104478Ssam if (ubsec_debug) 713104478Ssam device_printf(sc->sc_dev, "wakeup crypto (%x)\n", 714104478Ssam sc->sc_needwakeup); 715104478Ssam#endif /* UBSEC_DEBUG */ 716104478Ssam sc->sc_needwakeup &= ~wakeup; 717158828Spjd mtx_unlock(&sc->sc_freeqlock); 718104478Ssam crypto_unblock(sc->sc_cid, wakeup); 719104478Ssam } 720104478Ssam} 721104478Ssam 722104478Ssam/* 723104478Ssam * ubsec_feed() - aggregate and post requests to chip 724104478Ssam */ 725111416Ssamstatic void 726104478Ssamubsec_feed(struct ubsec_softc *sc) 727104478Ssam{ 728104478Ssam struct ubsec_q *q, *q2; 729104478Ssam int npkts, i; 730104478Ssam void *v; 731104478Ssam u_int32_t stat; 732104478Ssam 733108471Ssam /* 734108471Ssam * Decide how many ops to combine in a single MCR. We cannot 735108471Ssam * aggregate more than UBS_MAX_AGGR because this is the number 736111416Ssam * of slots defined in the data structure. Note that 737111416Ssam * aggregation only happens if ops are marked batch'able. 738108471Ssam * Aggregating ops reduces the number of interrupts to the host 739108471Ssam * but also (potentially) increases the latency for processing 740108471Ssam * completed ops as we only get an interrupt when all aggregated 741108471Ssam * ops have completed. 742108471Ssam */ 743111416Ssam if (sc->sc_nqueue == 0) 744111416Ssam return; 745111416Ssam if (sc->sc_nqueue > 1) { 746111416Ssam npkts = 0; 747111416Ssam SIMPLEQ_FOREACH(q, &sc->sc_queue, q_next) { 748111416Ssam npkts++; 749111416Ssam if ((q->q_crp->crp_flags & CRYPTO_F_BATCH) == 0) 750111416Ssam break; 751111416Ssam } 752111416Ssam } else 753111416Ssam npkts = 1; 754111416Ssam /* 755111416Ssam * Check device status before going any further. 756111416Ssam */ 757104478Ssam if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 758108471Ssam if (stat & BS_STAT_DMAERR) { 759104478Ssam ubsec_totalreset(sc); 760104478Ssam ubsecstats.hst_dmaerr++; 761108471Ssam } else 762108471Ssam ubsecstats.hst_mcr1full++; 763111416Ssam return; 764104478Ssam } 765111416Ssam if (sc->sc_nqueue > ubsecstats.hst_maxqueue) 766111416Ssam ubsecstats.hst_maxqueue = sc->sc_nqueue; 767111416Ssam if (npkts > UBS_MAX_AGGR) 768111416Ssam npkts = UBS_MAX_AGGR; 769111416Ssam if (npkts < 2) /* special case 1 op */ 770111416Ssam goto feed1; 771104478Ssam 772111416Ssam ubsecstats.hst_totbatch += npkts-1; 773104478Ssam#ifdef UBSEC_DEBUG 774104478Ssam if (ubsec_debug) 775104478Ssam printf("merging %d records\n", npkts); 776104478Ssam#endif /* UBSEC_DEBUG */ 777104478Ssam 778104478Ssam q = SIMPLEQ_FIRST(&sc->sc_queue); 779163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next); 780104478Ssam --sc->sc_nqueue; 781104478Ssam 782104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE); 783104478Ssam if (q->q_dst_map != NULL) 784104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD); 785104478Ssam 786104478Ssam q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */ 787104478Ssam 788104478Ssam for (i = 0; i < q->q_nstacked_mcrs; i++) { 789104478Ssam q2 = SIMPLEQ_FIRST(&sc->sc_queue); 790104478Ssam bus_dmamap_sync(sc->sc_dmat, q2->q_src_map, 791104478Ssam BUS_DMASYNC_PREWRITE); 792104478Ssam if (q2->q_dst_map != NULL) 793104478Ssam bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map, 794104478Ssam BUS_DMASYNC_PREREAD); 795163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next); 796104478Ssam --sc->sc_nqueue; 797104478Ssam 798104478Ssam v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) - 799104478Ssam sizeof(struct ubsec_mcr_add)); 800104478Ssam bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add)); 801104478Ssam q->q_stacked_mcr[i] = q2; 802104478Ssam } 803104478Ssam q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts); 804104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 805108471Ssam sc->sc_nqchip += npkts; 806108471Ssam if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 807108471Ssam ubsecstats.hst_maxqchip = sc->sc_nqchip; 808108823Ssam ubsec_dma_sync(&q->q_dma->d_alloc, 809104478Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 810104478Ssam WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 811104478Ssam offsetof(struct ubsec_dmachunk, d_mcr)); 812111416Ssam return; 813104478Ssamfeed1: 814111416Ssam q = SIMPLEQ_FIRST(&sc->sc_queue); 815104478Ssam 816111416Ssam bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE); 817111416Ssam if (q->q_dst_map != NULL) 818111416Ssam bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD); 819111416Ssam ubsec_dma_sync(&q->q_dma->d_alloc, 820111416Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 821104478Ssam 822111416Ssam WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 823111416Ssam offsetof(struct ubsec_dmachunk, d_mcr)); 824104478Ssam#ifdef UBSEC_DEBUG 825111416Ssam if (ubsec_debug) 826111416Ssam printf("feed1: q->chip %p %08x stat %08x\n", 827111416Ssam q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr), 828111416Ssam stat); 829104478Ssam#endif /* UBSEC_DEBUG */ 830163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next); 831111416Ssam --sc->sc_nqueue; 832111416Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 833111416Ssam sc->sc_nqchip++; 834108471Ssam if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 835108471Ssam ubsecstats.hst_maxqchip = sc->sc_nqchip; 836111416Ssam return; 837104478Ssam} 838104478Ssam 839159225Spjdstatic void 840159225Spjdubsec_setup_enckey(struct ubsec_session *ses, int algo, caddr_t key) 841159225Spjd{ 842159225Spjd 843159225Spjd /* Go ahead and compute key in ubsec's byte order */ 844159225Spjd if (algo == CRYPTO_DES_CBC) { 845159225Spjd bcopy(key, &ses->ses_deskey[0], 8); 846159225Spjd bcopy(key, &ses->ses_deskey[2], 8); 847159225Spjd bcopy(key, &ses->ses_deskey[4], 8); 848159225Spjd } else 849159225Spjd bcopy(key, ses->ses_deskey, 24); 850159225Spjd 851159225Spjd SWAP32(ses->ses_deskey[0]); 852159225Spjd SWAP32(ses->ses_deskey[1]); 853159225Spjd SWAP32(ses->ses_deskey[2]); 854159225Spjd SWAP32(ses->ses_deskey[3]); 855159225Spjd SWAP32(ses->ses_deskey[4]); 856159225Spjd SWAP32(ses->ses_deskey[5]); 857159225Spjd} 858159225Spjd 859159225Spjdstatic void 860159225Spjdubsec_setup_mackey(struct ubsec_session *ses, int algo, caddr_t key, int klen) 861159225Spjd{ 862159225Spjd MD5_CTX md5ctx; 863159225Spjd SHA1_CTX sha1ctx; 864159225Spjd int i; 865159225Spjd 866159225Spjd for (i = 0; i < klen; i++) 867159225Spjd key[i] ^= HMAC_IPAD_VAL; 868159225Spjd 869159225Spjd if (algo == CRYPTO_MD5_HMAC) { 870159225Spjd MD5Init(&md5ctx); 871159225Spjd MD5Update(&md5ctx, key, klen); 872159232Spjd MD5Update(&md5ctx, hmac_ipad_buffer, MD5_HMAC_BLOCK_LEN - klen); 873159225Spjd bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state)); 874159225Spjd } else { 875159225Spjd SHA1Init(&sha1ctx); 876159225Spjd SHA1Update(&sha1ctx, key, klen); 877159232Spjd SHA1Update(&sha1ctx, hmac_ipad_buffer, 878159232Spjd SHA1_HMAC_BLOCK_LEN - klen); 879159225Spjd bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32)); 880159225Spjd } 881159225Spjd 882159225Spjd for (i = 0; i < klen; i++) 883159225Spjd key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 884159225Spjd 885159225Spjd if (algo == CRYPTO_MD5_HMAC) { 886159225Spjd MD5Init(&md5ctx); 887159225Spjd MD5Update(&md5ctx, key, klen); 888159232Spjd MD5Update(&md5ctx, hmac_opad_buffer, MD5_HMAC_BLOCK_LEN - klen); 889159225Spjd bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state)); 890159225Spjd } else { 891159225Spjd SHA1Init(&sha1ctx); 892159225Spjd SHA1Update(&sha1ctx, key, klen); 893159232Spjd SHA1Update(&sha1ctx, hmac_opad_buffer, 894159232Spjd SHA1_HMAC_BLOCK_LEN - klen); 895159225Spjd bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32)); 896159225Spjd } 897159225Spjd 898159225Spjd for (i = 0; i < klen; i++) 899159225Spjd key[i] ^= HMAC_OPAD_VAL; 900159225Spjd} 901159225Spjd 902104478Ssam/* 903104478Ssam * Allocate a new 'session' and return an encoded session id. 'sidp' 904104478Ssam * contains our registration id, and should contain an encoded session 905104478Ssam * id on successful allocation. 906104478Ssam */ 907104478Ssamstatic int 908167755Ssamubsec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri) 909104478Ssam{ 910167755Ssam struct ubsec_softc *sc = device_get_softc(dev); 911104478Ssam struct cryptoini *c, *encini = NULL, *macini = NULL; 912104478Ssam struct ubsec_session *ses = NULL; 913159225Spjd int sesn; 914104478Ssam 915104478Ssam if (sidp == NULL || cri == NULL || sc == NULL) 916104478Ssam return (EINVAL); 917104478Ssam 918104478Ssam for (c = cri; c != NULL; c = c->cri_next) { 919104478Ssam if (c->cri_alg == CRYPTO_MD5_HMAC || 920104478Ssam c->cri_alg == CRYPTO_SHA1_HMAC) { 921104478Ssam if (macini) 922104478Ssam return (EINVAL); 923104478Ssam macini = c; 924104478Ssam } else if (c->cri_alg == CRYPTO_DES_CBC || 925104478Ssam c->cri_alg == CRYPTO_3DES_CBC) { 926104478Ssam if (encini) 927104478Ssam return (EINVAL); 928104478Ssam encini = c; 929104478Ssam } else 930104478Ssam return (EINVAL); 931104478Ssam } 932104478Ssam if (encini == NULL && macini == NULL) 933104478Ssam return (EINVAL); 934104478Ssam 935104478Ssam if (sc->sc_sessions == NULL) { 936104478Ssam ses = sc->sc_sessions = (struct ubsec_session *)malloc( 937104478Ssam sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 938104478Ssam if (ses == NULL) 939104478Ssam return (ENOMEM); 940104478Ssam sesn = 0; 941104478Ssam sc->sc_nsessions = 1; 942104478Ssam } else { 943104478Ssam for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 944104478Ssam if (sc->sc_sessions[sesn].ses_used == 0) { 945104478Ssam ses = &sc->sc_sessions[sesn]; 946104478Ssam break; 947104478Ssam } 948104478Ssam } 949104478Ssam 950104478Ssam if (ses == NULL) { 951104478Ssam sesn = sc->sc_nsessions; 952104478Ssam ses = (struct ubsec_session *)malloc((sesn + 1) * 953104478Ssam sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 954104478Ssam if (ses == NULL) 955104478Ssam return (ENOMEM); 956104478Ssam bcopy(sc->sc_sessions, ses, sesn * 957104478Ssam sizeof(struct ubsec_session)); 958104478Ssam bzero(sc->sc_sessions, sesn * 959104478Ssam sizeof(struct ubsec_session)); 960104478Ssam free(sc->sc_sessions, M_DEVBUF); 961104478Ssam sc->sc_sessions = ses; 962104478Ssam ses = &sc->sc_sessions[sesn]; 963104478Ssam sc->sc_nsessions++; 964104478Ssam } 965104478Ssam } 966104478Ssam bzero(ses, sizeof(struct ubsec_session)); 967104478Ssam ses->ses_used = 1; 968115747Ssam 969104478Ssam if (encini) { 970104478Ssam /* get an IV, network byte order */ 971104478Ssam /* XXX may read fewer than requested */ 972104478Ssam read_random(ses->ses_iv, sizeof(ses->ses_iv)); 973104478Ssam 974159225Spjd if (encini->cri_key != NULL) { 975159225Spjd ubsec_setup_enckey(ses, encini->cri_alg, 976159225Spjd encini->cri_key); 977159225Spjd } 978104478Ssam } 979104478Ssam 980104478Ssam if (macini) { 981158705Spjd ses->ses_mlen = macini->cri_mlen; 982158705Spjd if (ses->ses_mlen == 0) { 983158705Spjd if (macini->cri_alg == CRYPTO_MD5_HMAC) 984159233Spjd ses->ses_mlen = MD5_HASH_LEN; 985158705Spjd else 986159233Spjd ses->ses_mlen = SHA1_HASH_LEN; 987158705Spjd } 988158705Spjd 989159225Spjd if (macini->cri_key != NULL) { 990159225Spjd ubsec_setup_mackey(ses, macini->cri_alg, 991159225Spjd macini->cri_key, macini->cri_klen / 8); 992104478Ssam } 993104478Ssam } 994104478Ssam 995104478Ssam *sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn); 996104478Ssam return (0); 997104478Ssam} 998104478Ssam 999104478Ssam/* 1000104478Ssam * Deallocate a session. 1001104478Ssam */ 1002104478Ssamstatic int 1003167755Ssamubsec_freesession(device_t dev, u_int64_t tid) 1004104478Ssam{ 1005167755Ssam struct ubsec_softc *sc = device_get_softc(dev); 1006115747Ssam int session, ret; 1007116924Ssam u_int32_t sid = CRYPTO_SESID2LID(tid); 1008104478Ssam 1009104478Ssam if (sc == NULL) 1010104478Ssam return (EINVAL); 1011104478Ssam 1012104478Ssam session = UBSEC_SESSION(sid); 1013115747Ssam if (session < sc->sc_nsessions) { 1014115747Ssam bzero(&sc->sc_sessions[session], 1015115747Ssam sizeof(sc->sc_sessions[session])); 1016115747Ssam ret = 0; 1017115747Ssam } else 1018115747Ssam ret = EINVAL; 1019104478Ssam 1020115747Ssam return (ret); 1021104478Ssam} 1022104478Ssam 1023104478Ssamstatic void 1024104478Ssamubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error) 1025104478Ssam{ 1026104478Ssam struct ubsec_operand *op = arg; 1027104478Ssam 1028104478Ssam KASSERT(nsegs <= UBS_MAX_SCATTER, 1029104478Ssam ("Too many DMA segments returned when mapping operand")); 1030104478Ssam#ifdef UBSEC_DEBUG 1031104478Ssam if (ubsec_debug) 1032159341Spjd printf("ubsec_op_cb: mapsize %u nsegs %d error %d\n", 1033159341Spjd (u_int) mapsize, nsegs, error); 1034104478Ssam#endif 1035159341Spjd if (error != 0) 1036159341Spjd return; 1037104478Ssam op->mapsize = mapsize; 1038104478Ssam op->nsegs = nsegs; 1039104478Ssam bcopy(seg, op->segs, nsegs * sizeof (seg[0])); 1040104478Ssam} 1041104478Ssam 1042104478Ssamstatic int 1043167755Ssamubsec_process(device_t dev, struct cryptop *crp, int hint) 1044104478Ssam{ 1045167755Ssam struct ubsec_softc *sc = device_get_softc(dev); 1046104478Ssam struct ubsec_q *q = NULL; 1047104478Ssam int err = 0, i, j, nicealign; 1048104478Ssam struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 1049104478Ssam int encoffset = 0, macoffset = 0, cpskip, cpoffset; 1050104478Ssam int sskip, dskip, stheend, dtheend; 1051104478Ssam int16_t coffset; 1052104478Ssam struct ubsec_session *ses; 1053104478Ssam struct ubsec_pktctx ctx; 1054104478Ssam struct ubsec_dma *dmap = NULL; 1055104478Ssam 1056104478Ssam if (crp == NULL || crp->crp_callback == NULL || sc == NULL) { 1057104478Ssam ubsecstats.hst_invalid++; 1058104478Ssam return (EINVAL); 1059104478Ssam } 1060104478Ssam if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) { 1061108471Ssam ubsecstats.hst_badsession++; 1062104478Ssam return (EINVAL); 1063104478Ssam } 1064104478Ssam 1065115747Ssam mtx_lock(&sc->sc_freeqlock); 1066104478Ssam if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 1067104478Ssam ubsecstats.hst_queuefull++; 1068104478Ssam sc->sc_needwakeup |= CRYPTO_SYMQ; 1069115747Ssam mtx_unlock(&sc->sc_freeqlock); 1070104478Ssam return (ERESTART); 1071104478Ssam } 1072104478Ssam q = SIMPLEQ_FIRST(&sc->sc_freequeue); 1073163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next); 1074115747Ssam mtx_unlock(&sc->sc_freeqlock); 1075104478Ssam 1076104478Ssam dmap = q->q_dma; /* Save dma pointer */ 1077104478Ssam bzero(q, sizeof(struct ubsec_q)); 1078104478Ssam bzero(&ctx, sizeof(ctx)); 1079104478Ssam 1080104478Ssam q->q_sesn = UBSEC_SESSION(crp->crp_sid); 1081104478Ssam q->q_dma = dmap; 1082104478Ssam ses = &sc->sc_sessions[q->q_sesn]; 1083104478Ssam 1084104478Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) { 1085104478Ssam q->q_src_m = (struct mbuf *)crp->crp_buf; 1086104478Ssam q->q_dst_m = (struct mbuf *)crp->crp_buf; 1087104478Ssam } else if (crp->crp_flags & CRYPTO_F_IOV) { 1088104478Ssam q->q_src_io = (struct uio *)crp->crp_buf; 1089104478Ssam q->q_dst_io = (struct uio *)crp->crp_buf; 1090104478Ssam } else { 1091108471Ssam ubsecstats.hst_badflags++; 1092104478Ssam err = EINVAL; 1093104478Ssam goto errout; /* XXX we don't handle contiguous blocks! */ 1094104478Ssam } 1095104478Ssam 1096104478Ssam bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr)); 1097104478Ssam 1098104478Ssam dmap->d_dma->d_mcr.mcr_pkts = htole16(1); 1099104478Ssam dmap->d_dma->d_mcr.mcr_flags = 0; 1100104478Ssam q->q_crp = crp; 1101104478Ssam 1102104478Ssam crd1 = crp->crp_desc; 1103104478Ssam if (crd1 == NULL) { 1104108471Ssam ubsecstats.hst_nodesc++; 1105104478Ssam err = EINVAL; 1106104478Ssam goto errout; 1107104478Ssam } 1108104478Ssam crd2 = crd1->crd_next; 1109104478Ssam 1110104478Ssam if (crd2 == NULL) { 1111104478Ssam if (crd1->crd_alg == CRYPTO_MD5_HMAC || 1112104478Ssam crd1->crd_alg == CRYPTO_SHA1_HMAC) { 1113104478Ssam maccrd = crd1; 1114104478Ssam enccrd = NULL; 1115104478Ssam } else if (crd1->crd_alg == CRYPTO_DES_CBC || 1116104478Ssam crd1->crd_alg == CRYPTO_3DES_CBC) { 1117104478Ssam maccrd = NULL; 1118104478Ssam enccrd = crd1; 1119104478Ssam } else { 1120108471Ssam ubsecstats.hst_badalg++; 1121104478Ssam err = EINVAL; 1122104478Ssam goto errout; 1123104478Ssam } 1124104478Ssam } else { 1125104478Ssam if ((crd1->crd_alg == CRYPTO_MD5_HMAC || 1126104478Ssam crd1->crd_alg == CRYPTO_SHA1_HMAC) && 1127104478Ssam (crd2->crd_alg == CRYPTO_DES_CBC || 1128104478Ssam crd2->crd_alg == CRYPTO_3DES_CBC) && 1129104478Ssam ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 1130104478Ssam maccrd = crd1; 1131104478Ssam enccrd = crd2; 1132104478Ssam } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 1133104478Ssam crd1->crd_alg == CRYPTO_3DES_CBC) && 1134104478Ssam (crd2->crd_alg == CRYPTO_MD5_HMAC || 1135104478Ssam crd2->crd_alg == CRYPTO_SHA1_HMAC) && 1136104478Ssam (crd1->crd_flags & CRD_F_ENCRYPT)) { 1137104478Ssam enccrd = crd1; 1138104478Ssam maccrd = crd2; 1139104478Ssam } else { 1140104478Ssam /* 1141104478Ssam * We cannot order the ubsec as requested 1142104478Ssam */ 1143108471Ssam ubsecstats.hst_badalg++; 1144104478Ssam err = EINVAL; 1145104478Ssam goto errout; 1146104478Ssam } 1147104478Ssam } 1148104478Ssam 1149104478Ssam if (enccrd) { 1150159225Spjd if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) { 1151159225Spjd ubsec_setup_enckey(ses, enccrd->crd_alg, 1152159225Spjd enccrd->crd_key); 1153159225Spjd } 1154159225Spjd 1155104478Ssam encoffset = enccrd->crd_skip; 1156104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES); 1157104478Ssam 1158104478Ssam if (enccrd->crd_flags & CRD_F_ENCRYPT) { 1159104478Ssam q->q_flags |= UBSEC_QFLAGS_COPYOUTIV; 1160104478Ssam 1161104478Ssam if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1162104478Ssam bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1163104478Ssam else { 1164104478Ssam ctx.pc_iv[0] = ses->ses_iv[0]; 1165104478Ssam ctx.pc_iv[1] = ses->ses_iv[1]; 1166104478Ssam } 1167104478Ssam 1168104478Ssam if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { 1169159242Spjd crypto_copyback(crp->crp_flags, crp->crp_buf, 1170159242Spjd enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv); 1171104478Ssam } 1172104478Ssam } else { 1173104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND); 1174104478Ssam 1175104478Ssam if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1176104478Ssam bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1177159242Spjd else { 1178159242Spjd crypto_copydata(crp->crp_flags, crp->crp_buf, 1179159242Spjd enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv); 1180159242Spjd } 1181104478Ssam } 1182104478Ssam 1183104478Ssam ctx.pc_deskey[0] = ses->ses_deskey[0]; 1184104478Ssam ctx.pc_deskey[1] = ses->ses_deskey[1]; 1185104478Ssam ctx.pc_deskey[2] = ses->ses_deskey[2]; 1186104478Ssam ctx.pc_deskey[3] = ses->ses_deskey[3]; 1187104478Ssam ctx.pc_deskey[4] = ses->ses_deskey[4]; 1188104478Ssam ctx.pc_deskey[5] = ses->ses_deskey[5]; 1189104478Ssam SWAP32(ctx.pc_iv[0]); 1190104478Ssam SWAP32(ctx.pc_iv[1]); 1191104478Ssam } 1192104478Ssam 1193104478Ssam if (maccrd) { 1194159225Spjd if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) { 1195159225Spjd ubsec_setup_mackey(ses, maccrd->crd_alg, 1196159225Spjd maccrd->crd_key, maccrd->crd_klen / 8); 1197159225Spjd } 1198159225Spjd 1199104478Ssam macoffset = maccrd->crd_skip; 1200104478Ssam 1201104478Ssam if (maccrd->crd_alg == CRYPTO_MD5_HMAC) 1202104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5); 1203104478Ssam else 1204104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1); 1205104478Ssam 1206104478Ssam for (i = 0; i < 5; i++) { 1207104478Ssam ctx.pc_hminner[i] = ses->ses_hminner[i]; 1208104478Ssam ctx.pc_hmouter[i] = ses->ses_hmouter[i]; 1209104478Ssam 1210104478Ssam HTOLE32(ctx.pc_hminner[i]); 1211104478Ssam HTOLE32(ctx.pc_hmouter[i]); 1212104478Ssam } 1213104478Ssam } 1214104478Ssam 1215104478Ssam if (enccrd && maccrd) { 1216104478Ssam /* 1217104478Ssam * ubsec cannot handle packets where the end of encryption 1218104478Ssam * and authentication are not the same, or where the 1219104478Ssam * encrypted part begins before the authenticated part. 1220104478Ssam */ 1221104478Ssam if ((encoffset + enccrd->crd_len) != 1222104478Ssam (macoffset + maccrd->crd_len)) { 1223104478Ssam ubsecstats.hst_lenmismatch++; 1224104478Ssam err = EINVAL; 1225104478Ssam goto errout; 1226104478Ssam } 1227104478Ssam if (enccrd->crd_skip < maccrd->crd_skip) { 1228104478Ssam ubsecstats.hst_skipmismatch++; 1229104478Ssam err = EINVAL; 1230104478Ssam goto errout; 1231104478Ssam } 1232104478Ssam sskip = maccrd->crd_skip; 1233104478Ssam cpskip = dskip = enccrd->crd_skip; 1234104478Ssam stheend = maccrd->crd_len; 1235104478Ssam dtheend = enccrd->crd_len; 1236104478Ssam coffset = enccrd->crd_skip - maccrd->crd_skip; 1237104478Ssam cpoffset = cpskip + dtheend; 1238104478Ssam#ifdef UBSEC_DEBUG 1239104478Ssam if (ubsec_debug) { 1240104478Ssam printf("mac: skip %d, len %d, inject %d\n", 1241104478Ssam maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject); 1242104478Ssam printf("enc: skip %d, len %d, inject %d\n", 1243104478Ssam enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject); 1244104478Ssam printf("src: skip %d, len %d\n", sskip, stheend); 1245104478Ssam printf("dst: skip %d, len %d\n", dskip, dtheend); 1246104478Ssam printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n", 1247104478Ssam coffset, stheend, cpskip, cpoffset); 1248104478Ssam } 1249104478Ssam#endif 1250104478Ssam } else { 1251104478Ssam cpskip = dskip = sskip = macoffset + encoffset; 1252104478Ssam dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len; 1253104478Ssam cpoffset = cpskip + dtheend; 1254104478Ssam coffset = 0; 1255104478Ssam } 1256104478Ssam ctx.pc_offset = htole16(coffset >> 2); 1257104478Ssam 1258104478Ssam if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) { 1259104478Ssam ubsecstats.hst_nomap++; 1260104478Ssam err = ENOMEM; 1261104478Ssam goto errout; 1262104478Ssam } 1263104478Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) { 1264104478Ssam if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map, 1265104478Ssam q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1266104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1267104478Ssam q->q_src_map = NULL; 1268104478Ssam ubsecstats.hst_noload++; 1269104478Ssam err = ENOMEM; 1270104478Ssam goto errout; 1271104478Ssam } 1272104478Ssam } else if (crp->crp_flags & CRYPTO_F_IOV) { 1273104478Ssam if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map, 1274104478Ssam q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1275104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1276104478Ssam q->q_src_map = NULL; 1277104478Ssam ubsecstats.hst_noload++; 1278104478Ssam err = ENOMEM; 1279104478Ssam goto errout; 1280104478Ssam } 1281104478Ssam } 1282104478Ssam nicealign = ubsec_dmamap_aligned(&q->q_src); 1283104478Ssam 1284104478Ssam dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend); 1285104478Ssam 1286104478Ssam#ifdef UBSEC_DEBUG 1287104478Ssam if (ubsec_debug) 1288104478Ssam printf("src skip: %d nicealign: %u\n", sskip, nicealign); 1289104478Ssam#endif 1290104478Ssam for (i = j = 0; i < q->q_src_nsegs; i++) { 1291104478Ssam struct ubsec_pktbuf *pb; 1292104478Ssam bus_size_t packl = q->q_src_segs[i].ds_len; 1293104478Ssam bus_addr_t packp = q->q_src_segs[i].ds_addr; 1294104478Ssam 1295104478Ssam if (sskip >= packl) { 1296104478Ssam sskip -= packl; 1297104478Ssam continue; 1298104478Ssam } 1299104478Ssam 1300104478Ssam packl -= sskip; 1301104478Ssam packp += sskip; 1302104478Ssam sskip = 0; 1303104478Ssam 1304104478Ssam if (packl > 0xfffc) { 1305104478Ssam err = EIO; 1306104478Ssam goto errout; 1307104478Ssam } 1308104478Ssam 1309104478Ssam if (j == 0) 1310104478Ssam pb = &dmap->d_dma->d_mcr.mcr_ipktbuf; 1311104478Ssam else 1312104478Ssam pb = &dmap->d_dma->d_sbuf[j - 1]; 1313104478Ssam 1314104478Ssam pb->pb_addr = htole32(packp); 1315104478Ssam 1316104478Ssam if (stheend) { 1317104478Ssam if (packl > stheend) { 1318104478Ssam pb->pb_len = htole32(stheend); 1319104478Ssam stheend = 0; 1320104478Ssam } else { 1321104478Ssam pb->pb_len = htole32(packl); 1322104478Ssam stheend -= packl; 1323104478Ssam } 1324104478Ssam } else 1325104478Ssam pb->pb_len = htole32(packl); 1326104478Ssam 1327104478Ssam if ((i + 1) == q->q_src_nsegs) 1328104478Ssam pb->pb_next = 0; 1329104478Ssam else 1330104478Ssam pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1331104478Ssam offsetof(struct ubsec_dmachunk, d_sbuf[j])); 1332104478Ssam j++; 1333104478Ssam } 1334104478Ssam 1335104478Ssam if (enccrd == NULL && maccrd != NULL) { 1336104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0; 1337104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0; 1338104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr + 1339104478Ssam offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1340104478Ssam#ifdef UBSEC_DEBUG 1341104478Ssam if (ubsec_debug) 1342104478Ssam printf("opkt: %x %x %x\n", 1343104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr, 1344104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_len, 1345104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_next); 1346104478Ssam#endif 1347104478Ssam } else { 1348104478Ssam if (crp->crp_flags & CRYPTO_F_IOV) { 1349104478Ssam if (!nicealign) { 1350104478Ssam ubsecstats.hst_iovmisaligned++; 1351104478Ssam err = EINVAL; 1352104478Ssam goto errout; 1353104478Ssam } 1354104478Ssam if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, 1355104478Ssam &q->q_dst_map)) { 1356104478Ssam ubsecstats.hst_nomap++; 1357104478Ssam err = ENOMEM; 1358104478Ssam goto errout; 1359104478Ssam } 1360104478Ssam if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map, 1361104478Ssam q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) { 1362104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1363104478Ssam q->q_dst_map = NULL; 1364104478Ssam ubsecstats.hst_noload++; 1365104478Ssam err = ENOMEM; 1366104478Ssam goto errout; 1367104478Ssam } 1368104478Ssam } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1369104478Ssam if (nicealign) { 1370104478Ssam q->q_dst = q->q_src; 1371104478Ssam } else { 1372104478Ssam int totlen, len; 1373104478Ssam struct mbuf *m, *top, **mp; 1374104478Ssam 1375104478Ssam ubsecstats.hst_unaligned++; 1376104478Ssam totlen = q->q_src_mapsize; 1377160931Sjhb if (totlen >= MINCLSIZE) { 1378243857Sglebius m = m_getcl(M_NOWAIT, MT_DATA, 1379160931Sjhb q->q_src_m->m_flags & M_PKTHDR); 1380160931Sjhb len = MCLBYTES; 1381160931Sjhb } else if (q->q_src_m->m_flags & M_PKTHDR) { 1382243857Sglebius m = m_gethdr(M_NOWAIT, MT_DATA); 1383104478Ssam len = MHLEN; 1384104478Ssam } else { 1385243857Sglebius m = m_get(M_NOWAIT, MT_DATA); 1386104478Ssam len = MLEN; 1387104478Ssam } 1388160931Sjhb if (m && q->q_src_m->m_flags & M_PKTHDR && 1389243857Sglebius !m_dup_pkthdr(m, q->q_src_m, M_NOWAIT)) { 1390160931Sjhb m_free(m); 1391160931Sjhb m = NULL; 1392160931Sjhb } 1393104478Ssam if (m == NULL) { 1394104478Ssam ubsecstats.hst_nombuf++; 1395104478Ssam err = sc->sc_nqueue ? ERESTART : ENOMEM; 1396104478Ssam goto errout; 1397104478Ssam } 1398160931Sjhb m->m_len = len = min(totlen, len); 1399160931Sjhb totlen -= len; 1400160931Sjhb top = m; 1401104478Ssam mp = ⊤ 1402104478Ssam 1403104478Ssam while (totlen > 0) { 1404160931Sjhb if (totlen >= MINCLSIZE) { 1405243857Sglebius m = m_getcl(M_NOWAIT, 1406160931Sjhb MT_DATA, 0); 1407160931Sjhb len = MCLBYTES; 1408160931Sjhb } else { 1409243857Sglebius m = m_get(M_NOWAIT, MT_DATA); 1410104478Ssam len = MLEN; 1411104478Ssam } 1412160931Sjhb if (m == NULL) { 1413160931Sjhb m_freem(top); 1414160931Sjhb ubsecstats.hst_nombuf++; 1415160931Sjhb err = sc->sc_nqueue ? ERESTART : ENOMEM; 1416160931Sjhb goto errout; 1417104478Ssam } 1418104478Ssam m->m_len = len = min(totlen, len); 1419104478Ssam totlen -= len; 1420104478Ssam *mp = m; 1421104478Ssam mp = &m->m_next; 1422104478Ssam } 1423104478Ssam q->q_dst_m = top; 1424104478Ssam ubsec_mcopy(q->q_src_m, q->q_dst_m, 1425104478Ssam cpskip, cpoffset); 1426162969Sjhb if (bus_dmamap_create(sc->sc_dmat, 1427104478Ssam BUS_DMA_NOWAIT, &q->q_dst_map) != 0) { 1428104478Ssam ubsecstats.hst_nomap++; 1429104478Ssam err = ENOMEM; 1430104478Ssam goto errout; 1431104478Ssam } 1432104478Ssam if (bus_dmamap_load_mbuf(sc->sc_dmat, 1433104478Ssam q->q_dst_map, q->q_dst_m, 1434104478Ssam ubsec_op_cb, &q->q_dst, 1435104478Ssam BUS_DMA_NOWAIT) != 0) { 1436104478Ssam bus_dmamap_destroy(sc->sc_dmat, 1437104478Ssam q->q_dst_map); 1438104478Ssam q->q_dst_map = NULL; 1439104478Ssam ubsecstats.hst_noload++; 1440104478Ssam err = ENOMEM; 1441104478Ssam goto errout; 1442104478Ssam } 1443104478Ssam } 1444104478Ssam } else { 1445108471Ssam ubsecstats.hst_badflags++; 1446104478Ssam err = EINVAL; 1447104478Ssam goto errout; 1448104478Ssam } 1449104478Ssam 1450104478Ssam#ifdef UBSEC_DEBUG 1451104478Ssam if (ubsec_debug) 1452104478Ssam printf("dst skip: %d\n", dskip); 1453104478Ssam#endif 1454104478Ssam for (i = j = 0; i < q->q_dst_nsegs; i++) { 1455104478Ssam struct ubsec_pktbuf *pb; 1456104478Ssam bus_size_t packl = q->q_dst_segs[i].ds_len; 1457104478Ssam bus_addr_t packp = q->q_dst_segs[i].ds_addr; 1458104478Ssam 1459104478Ssam if (dskip >= packl) { 1460104478Ssam dskip -= packl; 1461104478Ssam continue; 1462104478Ssam } 1463104478Ssam 1464104478Ssam packl -= dskip; 1465104478Ssam packp += dskip; 1466104478Ssam dskip = 0; 1467104478Ssam 1468104478Ssam if (packl > 0xfffc) { 1469104478Ssam err = EIO; 1470104478Ssam goto errout; 1471104478Ssam } 1472104478Ssam 1473104478Ssam if (j == 0) 1474104478Ssam pb = &dmap->d_dma->d_mcr.mcr_opktbuf; 1475104478Ssam else 1476104478Ssam pb = &dmap->d_dma->d_dbuf[j - 1]; 1477104478Ssam 1478104478Ssam pb->pb_addr = htole32(packp); 1479104478Ssam 1480104478Ssam if (dtheend) { 1481104478Ssam if (packl > dtheend) { 1482104478Ssam pb->pb_len = htole32(dtheend); 1483104478Ssam dtheend = 0; 1484104478Ssam } else { 1485104478Ssam pb->pb_len = htole32(packl); 1486104478Ssam dtheend -= packl; 1487104478Ssam } 1488104478Ssam } else 1489104478Ssam pb->pb_len = htole32(packl); 1490104478Ssam 1491104478Ssam if ((i + 1) == q->q_dst_nsegs) { 1492104478Ssam if (maccrd) 1493104478Ssam pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1494104478Ssam offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1495104478Ssam else 1496104478Ssam pb->pb_next = 0; 1497104478Ssam } else 1498104478Ssam pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1499104478Ssam offsetof(struct ubsec_dmachunk, d_dbuf[j])); 1500104478Ssam j++; 1501104478Ssam } 1502104478Ssam } 1503104478Ssam 1504104478Ssam dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr + 1505104478Ssam offsetof(struct ubsec_dmachunk, d_ctx)); 1506104478Ssam 1507104478Ssam if (sc->sc_flags & UBS_FLAGS_LONGCTX) { 1508104478Ssam struct ubsec_pktctx_long *ctxl; 1509104478Ssam 1510104478Ssam ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr + 1511104478Ssam offsetof(struct ubsec_dmachunk, d_ctx)); 1512162969Sjhb 1513104478Ssam /* transform small context into long context */ 1514104478Ssam ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long)); 1515104478Ssam ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC); 1516104478Ssam ctxl->pc_flags = ctx.pc_flags; 1517104478Ssam ctxl->pc_offset = ctx.pc_offset; 1518104478Ssam for (i = 0; i < 6; i++) 1519104478Ssam ctxl->pc_deskey[i] = ctx.pc_deskey[i]; 1520104478Ssam for (i = 0; i < 5; i++) 1521104478Ssam ctxl->pc_hminner[i] = ctx.pc_hminner[i]; 1522104478Ssam for (i = 0; i < 5; i++) 1523162969Sjhb ctxl->pc_hmouter[i] = ctx.pc_hmouter[i]; 1524104478Ssam ctxl->pc_iv[0] = ctx.pc_iv[0]; 1525104478Ssam ctxl->pc_iv[1] = ctx.pc_iv[1]; 1526104478Ssam } else 1527104478Ssam bcopy(&ctx, dmap->d_alloc.dma_vaddr + 1528104478Ssam offsetof(struct ubsec_dmachunk, d_ctx), 1529104478Ssam sizeof(struct ubsec_pktctx)); 1530104478Ssam 1531115747Ssam mtx_lock(&sc->sc_mcr1lock); 1532104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next); 1533104478Ssam sc->sc_nqueue++; 1534104478Ssam ubsecstats.hst_ipackets++; 1535104478Ssam ubsecstats.hst_ibytes += dmap->d_alloc.dma_size; 1536111416Ssam if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= UBS_MAX_AGGR) 1537104478Ssam ubsec_feed(sc); 1538115747Ssam mtx_unlock(&sc->sc_mcr1lock); 1539104478Ssam return (0); 1540104478Ssam 1541104478Ssamerrout: 1542104478Ssam if (q != NULL) { 1543104478Ssam if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 1544104478Ssam m_freem(q->q_dst_m); 1545104478Ssam 1546104478Ssam if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1547104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1548104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1549104478Ssam } 1550104478Ssam if (q->q_src_map != NULL) { 1551104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1552104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1553104478Ssam } 1554158828Spjd } 1555158828Spjd if (q != NULL || err == ERESTART) { 1556115747Ssam mtx_lock(&sc->sc_freeqlock); 1557158828Spjd if (q != NULL) 1558158828Spjd SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1559158828Spjd if (err == ERESTART) 1560158828Spjd sc->sc_needwakeup |= CRYPTO_SYMQ; 1561115747Ssam mtx_unlock(&sc->sc_freeqlock); 1562104478Ssam } 1563104478Ssam if (err != ERESTART) { 1564104478Ssam crp->crp_etype = err; 1565104478Ssam crypto_done(crp); 1566104478Ssam } 1567104478Ssam return (err); 1568104478Ssam} 1569104478Ssam 1570104478Ssamstatic void 1571104478Ssamubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q) 1572104478Ssam{ 1573104478Ssam struct cryptop *crp = (struct cryptop *)q->q_crp; 1574104478Ssam struct cryptodesc *crd; 1575104478Ssam struct ubsec_dma *dmap = q->q_dma; 1576104478Ssam 1577112099Ssam ubsecstats.hst_opackets++; 1578112099Ssam ubsecstats.hst_obytes += dmap->d_alloc.dma_size; 1579112099Ssam 1580108823Ssam ubsec_dma_sync(&dmap->d_alloc, 1581104478Ssam BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1582104478Ssam if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1583104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 1584104478Ssam BUS_DMASYNC_POSTREAD); 1585104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1586104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1587104478Ssam } 1588104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE); 1589104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1590104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1591104478Ssam 1592104478Ssam if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) { 1593104478Ssam m_freem(q->q_src_m); 1594104478Ssam crp->crp_buf = (caddr_t)q->q_dst_m; 1595104478Ssam } 1596104478Ssam 1597104478Ssam /* copy out IV for future use */ 1598104478Ssam if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) { 1599104478Ssam for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1600104478Ssam if (crd->crd_alg != CRYPTO_DES_CBC && 1601104478Ssam crd->crd_alg != CRYPTO_3DES_CBC) 1602104478Ssam continue; 1603159242Spjd crypto_copydata(crp->crp_flags, crp->crp_buf, 1604159242Spjd crd->crd_skip + crd->crd_len - 8, 8, 1605159242Spjd (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv); 1606104478Ssam break; 1607104478Ssam } 1608104478Ssam } 1609104478Ssam 1610104478Ssam for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1611104478Ssam if (crd->crd_alg != CRYPTO_MD5_HMAC && 1612104478Ssam crd->crd_alg != CRYPTO_SHA1_HMAC) 1613104478Ssam continue; 1614159242Spjd crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, 1615159242Spjd sc->sc_sessions[q->q_sesn].ses_mlen, 1616159242Spjd (caddr_t)dmap->d_dma->d_macbuf); 1617104478Ssam break; 1618104478Ssam } 1619115747Ssam mtx_lock(&sc->sc_freeqlock); 1620104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1621115747Ssam mtx_unlock(&sc->sc_freeqlock); 1622104478Ssam crypto_done(crp); 1623104478Ssam} 1624104478Ssam 1625104478Ssamstatic void 1626104478Ssamubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset) 1627104478Ssam{ 1628104478Ssam int i, j, dlen, slen; 1629104478Ssam caddr_t dptr, sptr; 1630104478Ssam 1631104478Ssam j = 0; 1632104478Ssam sptr = srcm->m_data; 1633104478Ssam slen = srcm->m_len; 1634104478Ssam dptr = dstm->m_data; 1635104478Ssam dlen = dstm->m_len; 1636104478Ssam 1637104478Ssam while (1) { 1638104478Ssam for (i = 0; i < min(slen, dlen); i++) { 1639104478Ssam if (j < hoffset || j >= toffset) 1640104478Ssam *dptr++ = *sptr++; 1641104478Ssam slen--; 1642104478Ssam dlen--; 1643104478Ssam j++; 1644104478Ssam } 1645104478Ssam if (slen == 0) { 1646104478Ssam srcm = srcm->m_next; 1647104478Ssam if (srcm == NULL) 1648104478Ssam return; 1649104478Ssam sptr = srcm->m_data; 1650104478Ssam slen = srcm->m_len; 1651104478Ssam } 1652104478Ssam if (dlen == 0) { 1653104478Ssam dstm = dstm->m_next; 1654104478Ssam if (dstm == NULL) 1655104478Ssam return; 1656104478Ssam dptr = dstm->m_data; 1657104478Ssam dlen = dstm->m_len; 1658104478Ssam } 1659104478Ssam } 1660104478Ssam} 1661104478Ssam 1662104478Ssam/* 1663104478Ssam * feed the key generator, must be called at splimp() or higher. 1664104478Ssam */ 1665104478Ssamstatic int 1666104478Ssamubsec_feed2(struct ubsec_softc *sc) 1667104478Ssam{ 1668104478Ssam struct ubsec_q2 *q; 1669104478Ssam 1670104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) { 1671104478Ssam if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL) 1672104478Ssam break; 1673104478Ssam q = SIMPLEQ_FIRST(&sc->sc_queue2); 1674104478Ssam 1675108823Ssam ubsec_dma_sync(&q->q_mcr, 1676104478Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1677108823Ssam ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_PREWRITE); 1678104478Ssam 1679104478Ssam WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr); 1680163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q_next); 1681104478Ssam --sc->sc_nqueue2; 1682104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next); 1683104478Ssam } 1684104478Ssam return (0); 1685104478Ssam} 1686104478Ssam 1687104478Ssam/* 1688104478Ssam * Callback for handling random numbers 1689104478Ssam */ 1690104478Ssamstatic void 1691104478Ssamubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q) 1692104478Ssam{ 1693104478Ssam struct cryptkop *krp; 1694104478Ssam struct ubsec_ctx_keyop *ctx; 1695104478Ssam 1696104478Ssam ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr; 1697108823Ssam ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_POSTWRITE); 1698104478Ssam 1699104478Ssam switch (q->q_type) { 1700104478Ssam#ifndef UBSEC_NO_RNG 1701104478Ssam case UBS_CTXOP_RNGBYPASS: { 1702104478Ssam struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q; 1703104478Ssam 1704108823Ssam ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_POSTREAD); 1705112124Ssam (*sc->sc_harvest)(sc->sc_rndtest, 1706112124Ssam rng->rng_buf.dma_vaddr, 1707112124Ssam UBSEC_RNG_BUFSIZ*sizeof (u_int32_t)); 1708104478Ssam rng->rng_used = 0; 1709104478Ssam callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1710104478Ssam break; 1711104478Ssam } 1712104478Ssam#endif 1713104478Ssam case UBS_CTXOP_MODEXP: { 1714104478Ssam struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 1715104478Ssam u_int rlen, clen; 1716104478Ssam 1717104478Ssam krp = me->me_krp; 1718104478Ssam rlen = (me->me_modbits + 7) / 8; 1719104630Ssam clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8; 1720104478Ssam 1721108823Ssam ubsec_dma_sync(&me->me_M, BUS_DMASYNC_POSTWRITE); 1722108823Ssam ubsec_dma_sync(&me->me_E, BUS_DMASYNC_POSTWRITE); 1723108823Ssam ubsec_dma_sync(&me->me_C, BUS_DMASYNC_POSTREAD); 1724108823Ssam ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_POSTWRITE); 1725104478Ssam 1726104478Ssam if (clen < rlen) 1727104478Ssam krp->krp_status = E2BIG; 1728104630Ssam else { 1729104630Ssam if (sc->sc_flags & UBS_FLAGS_HWNORM) { 1730104630Ssam bzero(krp->krp_param[krp->krp_iparams].crp_p, 1731104630Ssam (krp->krp_param[krp->krp_iparams].crp_nbits 1732104630Ssam + 7) / 8); 1733104630Ssam bcopy(me->me_C.dma_vaddr, 1734104630Ssam krp->krp_param[krp->krp_iparams].crp_p, 1735104630Ssam (me->me_modbits + 7) / 8); 1736104630Ssam } else 1737104630Ssam ubsec_kshift_l(me->me_shiftbits, 1738104630Ssam me->me_C.dma_vaddr, me->me_normbits, 1739104630Ssam krp->krp_param[krp->krp_iparams].crp_p, 1740104630Ssam krp->krp_param[krp->krp_iparams].crp_nbits); 1741104630Ssam } 1742104478Ssam 1743104478Ssam crypto_kdone(krp); 1744104478Ssam 1745104478Ssam /* bzero all potentially sensitive data */ 1746104478Ssam bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 1747104478Ssam bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 1748104478Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 1749104478Ssam bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 1750104478Ssam 1751104478Ssam /* Can't free here, so put us on the free list. */ 1752104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next); 1753104478Ssam break; 1754104478Ssam } 1755104478Ssam case UBS_CTXOP_RSAPRIV: { 1756104478Ssam struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 1757104478Ssam u_int len; 1758104478Ssam 1759104478Ssam krp = rp->rpr_krp; 1760108823Ssam ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_POSTWRITE); 1761108823Ssam ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_POSTREAD); 1762104478Ssam 1763104478Ssam len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8; 1764104478Ssam bcopy(rp->rpr_msgout.dma_vaddr, 1765104478Ssam krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len); 1766104478Ssam 1767104478Ssam crypto_kdone(krp); 1768104478Ssam 1769104478Ssam bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 1770104478Ssam bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 1771104478Ssam bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size); 1772104478Ssam 1773104478Ssam /* Can't free here, so put us on the free list. */ 1774104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next); 1775104478Ssam break; 1776104478Ssam } 1777104478Ssam default: 1778104478Ssam device_printf(sc->sc_dev, "unknown ctx op: %x\n", 1779104478Ssam letoh16(ctx->ctx_op)); 1780104478Ssam break; 1781104478Ssam } 1782104478Ssam} 1783104478Ssam 1784104478Ssam#ifndef UBSEC_NO_RNG 1785104478Ssamstatic void 1786104478Ssamubsec_rng(void *vsc) 1787104478Ssam{ 1788104478Ssam struct ubsec_softc *sc = vsc; 1789104478Ssam struct ubsec_q2_rng *rng = &sc->sc_rng; 1790104478Ssam struct ubsec_mcr *mcr; 1791104478Ssam struct ubsec_ctx_rngbypass *ctx; 1792104478Ssam 1793115747Ssam mtx_lock(&sc->sc_mcr2lock); 1794104478Ssam if (rng->rng_used) { 1795115747Ssam mtx_unlock(&sc->sc_mcr2lock); 1796104478Ssam return; 1797104478Ssam } 1798104478Ssam sc->sc_nqueue2++; 1799104478Ssam if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE) 1800104478Ssam goto out; 1801104478Ssam 1802104478Ssam mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr; 1803104478Ssam ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr; 1804104478Ssam 1805104478Ssam mcr->mcr_pkts = htole16(1); 1806104478Ssam mcr->mcr_flags = 0; 1807104478Ssam mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr); 1808104478Ssam mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0; 1809104478Ssam mcr->mcr_ipktbuf.pb_len = 0; 1810104478Ssam mcr->mcr_reserved = mcr->mcr_pktlen = 0; 1811104478Ssam mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr); 1812104478Ssam mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) & 1813104478Ssam UBS_PKTBUF_LEN); 1814104478Ssam mcr->mcr_opktbuf.pb_next = 0; 1815104478Ssam 1816104478Ssam ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass)); 1817104478Ssam ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS); 1818104478Ssam rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS; 1819104478Ssam 1820108823Ssam ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_PREREAD); 1821104478Ssam 1822104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next); 1823104478Ssam rng->rng_used = 1; 1824104478Ssam ubsec_feed2(sc); 1825104478Ssam ubsecstats.hst_rng++; 1826115747Ssam mtx_unlock(&sc->sc_mcr2lock); 1827104478Ssam 1828104478Ssam return; 1829104478Ssam 1830104478Ssamout: 1831104478Ssam /* 1832104478Ssam * Something weird happened, generate our own call back. 1833104478Ssam */ 1834104478Ssam sc->sc_nqueue2--; 1835115747Ssam mtx_unlock(&sc->sc_mcr2lock); 1836104478Ssam callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1837104478Ssam} 1838104478Ssam#endif /* UBSEC_NO_RNG */ 1839104478Ssam 1840104478Ssamstatic void 1841104478Ssamubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 1842104478Ssam{ 1843104478Ssam bus_addr_t *paddr = (bus_addr_t*) arg; 1844104478Ssam *paddr = segs->ds_addr; 1845104478Ssam} 1846104478Ssam 1847104478Ssamstatic int 1848104478Ssamubsec_dma_malloc( 1849104478Ssam struct ubsec_softc *sc, 1850104478Ssam bus_size_t size, 1851104478Ssam struct ubsec_dma_alloc *dma, 1852104478Ssam int mapflags 1853104478Ssam) 1854104478Ssam{ 1855104478Ssam int r; 1856104478Ssam 1857108823Ssam /* XXX could specify sc_dmat as parent but that just adds overhead */ 1858232874Sscottl r = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ 1859108823Ssam 1, 0, /* alignment, bounds */ 1860108823Ssam BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 1861108823Ssam BUS_SPACE_MAXADDR, /* highaddr */ 1862108823Ssam NULL, NULL, /* filter, filterarg */ 1863108823Ssam size, /* maxsize */ 1864108823Ssam 1, /* nsegments */ 1865108823Ssam size, /* maxsegsize */ 1866108823Ssam BUS_DMA_ALLOCNOW, /* flags */ 1867117126Sscottl NULL, NULL, /* lockfunc, lockarg */ 1868108823Ssam &dma->dma_tag); 1869108823Ssam if (r != 0) { 1870108823Ssam device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1871108823Ssam "bus_dma_tag_create failed; error %u\n", r); 1872104478Ssam goto fail_0; 1873108823Ssam } 1874104478Ssam 1875108823Ssam r = bus_dmamap_create(dma->dma_tag, BUS_DMA_NOWAIT, &dma->dma_map); 1876108823Ssam if (r != 0) { 1877108823Ssam device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1878108823Ssam "bus_dmamap_create failed; error %u\n", r); 1879104478Ssam goto fail_1; 1880108823Ssam } 1881104478Ssam 1882108823Ssam r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 1883108823Ssam BUS_DMA_NOWAIT, &dma->dma_map); 1884108823Ssam if (r != 0) { 1885108823Ssam device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1886205844Simp "bus_dmammem_alloc failed; size %ju, error %u\n", 1887205844Simp (intmax_t)size, r); 1888108823Ssam goto fail_2; 1889108823Ssam } 1890108823Ssam 1891108823Ssam r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 1892104478Ssam size, 1893104478Ssam ubsec_dmamap_cb, 1894104478Ssam &dma->dma_paddr, 1895104478Ssam mapflags | BUS_DMA_NOWAIT); 1896108823Ssam if (r != 0) { 1897108823Ssam device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1898108823Ssam "bus_dmamap_load failed; error %u\n", r); 1899108823Ssam goto fail_3; 1900108823Ssam } 1901104478Ssam 1902104478Ssam dma->dma_size = size; 1903104478Ssam return (0); 1904104478Ssam 1905108823Ssamfail_3: 1906108823Ssam bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1907104478Ssamfail_2: 1908108823Ssam bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1909104478Ssamfail_1: 1910108823Ssam bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1911108823Ssam bus_dma_tag_destroy(dma->dma_tag); 1912104478Ssamfail_0: 1913104478Ssam dma->dma_map = NULL; 1914108823Ssam dma->dma_tag = NULL; 1915104478Ssam return (r); 1916104478Ssam} 1917104478Ssam 1918104478Ssamstatic void 1919104478Ssamubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma) 1920104478Ssam{ 1921108823Ssam bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1922108823Ssam bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1923108823Ssam bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1924108823Ssam bus_dma_tag_destroy(dma->dma_tag); 1925104478Ssam} 1926104478Ssam 1927104478Ssam/* 1928104478Ssam * Resets the board. Values in the regesters are left as is 1929104478Ssam * from the reset (i.e. initial values are assigned elsewhere). 1930104478Ssam */ 1931104478Ssamstatic void 1932104478Ssamubsec_reset_board(struct ubsec_softc *sc) 1933104478Ssam{ 1934104478Ssam volatile u_int32_t ctrl; 1935104478Ssam 1936104478Ssam ctrl = READ_REG(sc, BS_CTRL); 1937104478Ssam ctrl |= BS_CTRL_RESET; 1938104478Ssam WRITE_REG(sc, BS_CTRL, ctrl); 1939104478Ssam 1940104478Ssam /* 1941104478Ssam * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us 1942104478Ssam */ 1943104478Ssam DELAY(10); 1944104478Ssam} 1945104478Ssam 1946104478Ssam/* 1947104478Ssam * Init Broadcom registers 1948104478Ssam */ 1949104478Ssamstatic void 1950104478Ssamubsec_init_board(struct ubsec_softc *sc) 1951104478Ssam{ 1952104630Ssam u_int32_t ctrl; 1953104630Ssam 1954104630Ssam ctrl = READ_REG(sc, BS_CTRL); 1955104630Ssam ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64); 1956104630Ssam ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT; 1957104630Ssam 1958104630Ssam if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) 1959104630Ssam ctrl |= BS_CTRL_MCR2INT; 1960104630Ssam else 1961104630Ssam ctrl &= ~BS_CTRL_MCR2INT; 1962104630Ssam 1963104630Ssam if (sc->sc_flags & UBS_FLAGS_HWNORM) 1964104630Ssam ctrl &= ~BS_CTRL_SWNORM; 1965104630Ssam 1966104630Ssam WRITE_REG(sc, BS_CTRL, ctrl); 1967104478Ssam} 1968104478Ssam 1969104478Ssam/* 1970104478Ssam * Init Broadcom PCI registers 1971104478Ssam */ 1972104478Ssamstatic void 1973104478Ssamubsec_init_pciregs(device_t dev) 1974104478Ssam{ 1975104478Ssam#if 0 1976104478Ssam u_int32_t misc; 1977104478Ssam 1978104478Ssam misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT); 1979104478Ssam misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT)) 1980104478Ssam | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT); 1981104478Ssam misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT)) 1982104478Ssam | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT); 1983104478Ssam pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc); 1984104478Ssam#endif 1985104478Ssam 1986104478Ssam /* 1987104478Ssam * This will set the cache line size to 1, this will 1988104478Ssam * force the BCM58xx chip just to do burst read/writes. 1989104478Ssam * Cache line read/writes are to slow 1990104478Ssam */ 1991104478Ssam pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1); 1992104478Ssam} 1993104478Ssam 1994104478Ssam/* 1995104478Ssam * Clean up after a chip crash. 1996104478Ssam * It is assumed that the caller in splimp() 1997104478Ssam */ 1998104478Ssamstatic void 1999104478Ssamubsec_cleanchip(struct ubsec_softc *sc) 2000104478Ssam{ 2001104478Ssam struct ubsec_q *q; 2002104478Ssam 2003104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 2004104478Ssam q = SIMPLEQ_FIRST(&sc->sc_qchip); 2005163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next); 2006104478Ssam ubsec_free_q(sc, q); 2007104478Ssam } 2008108471Ssam sc->sc_nqchip = 0; 2009104478Ssam} 2010104478Ssam 2011104478Ssam/* 2012108823Ssam * free a ubsec_q 2013108823Ssam * It is assumed that the caller is within splimp(). 2014104478Ssam */ 2015104478Ssamstatic int 2016104478Ssamubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q) 2017104478Ssam{ 2018104478Ssam struct ubsec_q *q2; 2019104478Ssam struct cryptop *crp; 2020104478Ssam int npkts; 2021104478Ssam int i; 2022104478Ssam 2023104478Ssam npkts = q->q_nstacked_mcrs; 2024104478Ssam 2025104478Ssam for (i = 0; i < npkts; i++) { 2026104478Ssam if(q->q_stacked_mcr[i]) { 2027104478Ssam q2 = q->q_stacked_mcr[i]; 2028104478Ssam 2029162969Sjhb if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m)) 2030104478Ssam m_freem(q2->q_dst_m); 2031104478Ssam 2032104478Ssam crp = (struct cryptop *)q2->q_crp; 2033162969Sjhb 2034104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next); 2035162969Sjhb 2036104478Ssam crp->crp_etype = EFAULT; 2037104478Ssam crypto_done(crp); 2038104478Ssam } else { 2039104478Ssam break; 2040104478Ssam } 2041104478Ssam } 2042104478Ssam 2043104478Ssam /* 2044104478Ssam * Free header MCR 2045104478Ssam */ 2046104478Ssam if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 2047104478Ssam m_freem(q->q_dst_m); 2048104478Ssam 2049104478Ssam crp = (struct cryptop *)q->q_crp; 2050162969Sjhb 2051104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 2052162969Sjhb 2053104478Ssam crp->crp_etype = EFAULT; 2054104478Ssam crypto_done(crp); 2055104478Ssam return(0); 2056104478Ssam} 2057104478Ssam 2058104478Ssam/* 2059104478Ssam * Routine to reset the chip and clean up. 2060104478Ssam * It is assumed that the caller is in splimp() 2061104478Ssam */ 2062104478Ssamstatic void 2063104478Ssamubsec_totalreset(struct ubsec_softc *sc) 2064104478Ssam{ 2065104478Ssam ubsec_reset_board(sc); 2066104478Ssam ubsec_init_board(sc); 2067104478Ssam ubsec_cleanchip(sc); 2068104478Ssam} 2069104478Ssam 2070104478Ssamstatic int 2071104478Ssamubsec_dmamap_aligned(struct ubsec_operand *op) 2072104478Ssam{ 2073104478Ssam int i; 2074104478Ssam 2075104478Ssam for (i = 0; i < op->nsegs; i++) { 2076104478Ssam if (op->segs[i].ds_addr & 3) 2077104478Ssam return (0); 2078104478Ssam if ((i != (op->nsegs - 1)) && 2079104478Ssam (op->segs[i].ds_len & 3)) 2080104478Ssam return (0); 2081104478Ssam } 2082104478Ssam return (1); 2083104478Ssam} 2084104478Ssam 2085104478Ssamstatic void 2086104478Ssamubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q) 2087104478Ssam{ 2088104478Ssam switch (q->q_type) { 2089104478Ssam case UBS_CTXOP_MODEXP: { 2090104478Ssam struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 2091104478Ssam 2092104478Ssam ubsec_dma_free(sc, &me->me_q.q_mcr); 2093104478Ssam ubsec_dma_free(sc, &me->me_q.q_ctx); 2094104478Ssam ubsec_dma_free(sc, &me->me_M); 2095104478Ssam ubsec_dma_free(sc, &me->me_E); 2096104478Ssam ubsec_dma_free(sc, &me->me_C); 2097104478Ssam ubsec_dma_free(sc, &me->me_epb); 2098104478Ssam free(me, M_DEVBUF); 2099104478Ssam break; 2100104478Ssam } 2101104478Ssam case UBS_CTXOP_RSAPRIV: { 2102104478Ssam struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 2103104478Ssam 2104104478Ssam ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2105104478Ssam ubsec_dma_free(sc, &rp->rpr_q.q_ctx); 2106104478Ssam ubsec_dma_free(sc, &rp->rpr_msgin); 2107104478Ssam ubsec_dma_free(sc, &rp->rpr_msgout); 2108104478Ssam free(rp, M_DEVBUF); 2109104478Ssam break; 2110104478Ssam } 2111104478Ssam default: 2112104478Ssam device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type); 2113104478Ssam break; 2114104478Ssam } 2115104478Ssam} 2116104478Ssam 2117104478Ssamstatic int 2118167755Ssamubsec_kprocess(device_t dev, struct cryptkop *krp, int hint) 2119104478Ssam{ 2120167755Ssam struct ubsec_softc *sc = device_get_softc(dev); 2121104630Ssam int r; 2122104478Ssam 2123104478Ssam if (krp == NULL || krp->krp_callback == NULL) 2124104478Ssam return (EINVAL); 2125104478Ssam 2126104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) { 2127104478Ssam struct ubsec_q2 *q; 2128104478Ssam 2129104478Ssam q = SIMPLEQ_FIRST(&sc->sc_q2free); 2130163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q_next); 2131104478Ssam ubsec_kfree(sc, q); 2132104478Ssam } 2133104478Ssam 2134104478Ssam switch (krp->krp_op) { 2135104478Ssam case CRK_MOD_EXP: 2136104630Ssam if (sc->sc_flags & UBS_FLAGS_HWNORM) 2137104630Ssam r = ubsec_kprocess_modexp_hw(sc, krp, hint); 2138104630Ssam else 2139104630Ssam r = ubsec_kprocess_modexp_sw(sc, krp, hint); 2140104630Ssam break; 2141104478Ssam case CRK_MOD_EXP_CRT: 2142104478Ssam return (ubsec_kprocess_rsapriv(sc, krp, hint)); 2143104478Ssam default: 2144104478Ssam device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n", 2145104478Ssam krp->krp_op); 2146104478Ssam krp->krp_status = EOPNOTSUPP; 2147104478Ssam crypto_kdone(krp); 2148104478Ssam return (0); 2149104478Ssam } 2150104630Ssam return (0); /* silence compiler */ 2151104478Ssam} 2152104478Ssam 2153104478Ssam/* 2154104630Ssam * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization) 2155104478Ssam */ 2156104478Ssamstatic int 2157104630Ssamubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2158104478Ssam{ 2159104478Ssam struct ubsec_q2_modexp *me; 2160104478Ssam struct ubsec_mcr *mcr; 2161104478Ssam struct ubsec_ctx_modexp *ctx; 2162104478Ssam struct ubsec_pktbuf *epb; 2163104478Ssam int err = 0; 2164104478Ssam u_int nbits, normbits, mbits, shiftbits, ebits; 2165104478Ssam 2166104478Ssam me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2167104478Ssam if (me == NULL) { 2168104478Ssam err = ENOMEM; 2169104478Ssam goto errout; 2170104478Ssam } 2171104478Ssam bzero(me, sizeof *me); 2172104478Ssam me->me_krp = krp; 2173104478Ssam me->me_q.q_type = UBS_CTXOP_MODEXP; 2174104478Ssam 2175104478Ssam nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2176104478Ssam if (nbits <= 512) 2177104478Ssam normbits = 512; 2178104478Ssam else if (nbits <= 768) 2179104478Ssam normbits = 768; 2180104478Ssam else if (nbits <= 1024) 2181104478Ssam normbits = 1024; 2182104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2183104478Ssam normbits = 1536; 2184104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2185104478Ssam normbits = 2048; 2186104478Ssam else { 2187104478Ssam err = E2BIG; 2188104478Ssam goto errout; 2189104478Ssam } 2190104478Ssam 2191104630Ssam shiftbits = normbits - nbits; 2192104478Ssam 2193104630Ssam me->me_modbits = nbits; 2194104478Ssam me->me_shiftbits = shiftbits; 2195104630Ssam me->me_normbits = normbits; 2196104478Ssam 2197104478Ssam /* Sanity check: result bits must be >= true modulus bits. */ 2198104630Ssam if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2199104478Ssam err = ERANGE; 2200104478Ssam goto errout; 2201104478Ssam } 2202104478Ssam 2203104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2204104478Ssam &me->me_q.q_mcr, 0)) { 2205104478Ssam err = ENOMEM; 2206104478Ssam goto errout; 2207104478Ssam } 2208104478Ssam mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2209104478Ssam 2210104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2211104478Ssam &me->me_q.q_ctx, 0)) { 2212104478Ssam err = ENOMEM; 2213104478Ssam goto errout; 2214104478Ssam } 2215104478Ssam 2216104478Ssam mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2217104478Ssam if (mbits > nbits) { 2218104478Ssam err = E2BIG; 2219104478Ssam goto errout; 2220104478Ssam } 2221104478Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2222104478Ssam err = ENOMEM; 2223104478Ssam goto errout; 2224104478Ssam } 2225104478Ssam ubsec_kshift_r(shiftbits, 2226104478Ssam krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits, 2227104478Ssam me->me_M.dma_vaddr, normbits); 2228104478Ssam 2229104478Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2230104478Ssam err = ENOMEM; 2231104478Ssam goto errout; 2232104478Ssam } 2233104478Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2234104478Ssam 2235104478Ssam ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2236104478Ssam if (ebits > nbits) { 2237104478Ssam err = E2BIG; 2238104478Ssam goto errout; 2239104478Ssam } 2240104478Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2241104478Ssam err = ENOMEM; 2242104478Ssam goto errout; 2243104478Ssam } 2244104478Ssam ubsec_kshift_r(shiftbits, 2245104478Ssam krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits, 2246104478Ssam me->me_E.dma_vaddr, normbits); 2247104478Ssam 2248104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2249104478Ssam &me->me_epb, 0)) { 2250104478Ssam err = ENOMEM; 2251104478Ssam goto errout; 2252104478Ssam } 2253104478Ssam epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2254104478Ssam epb->pb_addr = htole32(me->me_E.dma_paddr); 2255104478Ssam epb->pb_next = 0; 2256104478Ssam epb->pb_len = htole32(normbits / 8); 2257104478Ssam 2258104478Ssam#ifdef UBSEC_DEBUG 2259104478Ssam if (ubsec_debug) { 2260104478Ssam printf("Epb "); 2261104478Ssam ubsec_dump_pb(epb); 2262104478Ssam } 2263104478Ssam#endif 2264104478Ssam 2265104478Ssam mcr->mcr_pkts = htole16(1); 2266104478Ssam mcr->mcr_flags = 0; 2267104478Ssam mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2268104478Ssam mcr->mcr_reserved = 0; 2269104478Ssam mcr->mcr_pktlen = 0; 2270104478Ssam 2271104478Ssam mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2272104478Ssam mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2273104478Ssam mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2274104478Ssam 2275104478Ssam mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2276104478Ssam mcr->mcr_opktbuf.pb_next = 0; 2277104478Ssam mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2278104478Ssam 2279104478Ssam#ifdef DIAGNOSTIC 2280104478Ssam /* Misaligned output buffer will hang the chip. */ 2281104478Ssam if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2282104478Ssam panic("%s: modexp invalid addr 0x%x\n", 2283104478Ssam device_get_nameunit(sc->sc_dev), 2284104478Ssam letoh32(mcr->mcr_opktbuf.pb_addr)); 2285104478Ssam if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2286104478Ssam panic("%s: modexp invalid len 0x%x\n", 2287104478Ssam device_get_nameunit(sc->sc_dev), 2288104478Ssam letoh32(mcr->mcr_opktbuf.pb_len)); 2289104478Ssam#endif 2290104478Ssam 2291104478Ssam ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2292104478Ssam bzero(ctx, sizeof(*ctx)); 2293104478Ssam ubsec_kshift_r(shiftbits, 2294104478Ssam krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits, 2295104478Ssam ctx->me_N, normbits); 2296104478Ssam ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2297104478Ssam ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2298104630Ssam ctx->me_E_len = htole16(nbits); 2299104630Ssam ctx->me_N_len = htole16(nbits); 2300104478Ssam 2301104478Ssam#ifdef UBSEC_DEBUG 2302104478Ssam if (ubsec_debug) { 2303104478Ssam ubsec_dump_mcr(mcr); 2304104478Ssam ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2305104478Ssam } 2306104478Ssam#endif 2307104478Ssam 2308104478Ssam /* 2309104478Ssam * ubsec_feed2 will sync mcr and ctx, we just need to sync 2310104478Ssam * everything else. 2311104478Ssam */ 2312108823Ssam ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE); 2313108823Ssam ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE); 2314108823Ssam ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD); 2315108823Ssam ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE); 2316104478Ssam 2317104478Ssam /* Enqueue and we're done... */ 2318115747Ssam mtx_lock(&sc->sc_mcr2lock); 2319104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2320104478Ssam ubsec_feed2(sc); 2321104478Ssam ubsecstats.hst_modexp++; 2322115747Ssam mtx_unlock(&sc->sc_mcr2lock); 2323104478Ssam 2324104478Ssam return (0); 2325104478Ssam 2326104478Ssamerrout: 2327104478Ssam if (me != NULL) { 2328104478Ssam if (me->me_q.q_mcr.dma_map != NULL) 2329104478Ssam ubsec_dma_free(sc, &me->me_q.q_mcr); 2330104478Ssam if (me->me_q.q_ctx.dma_map != NULL) { 2331104478Ssam bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2332104478Ssam ubsec_dma_free(sc, &me->me_q.q_ctx); 2333104478Ssam } 2334104478Ssam if (me->me_M.dma_map != NULL) { 2335104478Ssam bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2336104478Ssam ubsec_dma_free(sc, &me->me_M); 2337104478Ssam } 2338104478Ssam if (me->me_E.dma_map != NULL) { 2339104478Ssam bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2340104478Ssam ubsec_dma_free(sc, &me->me_E); 2341104478Ssam } 2342104478Ssam if (me->me_C.dma_map != NULL) { 2343104478Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2344104478Ssam ubsec_dma_free(sc, &me->me_C); 2345104478Ssam } 2346104478Ssam if (me->me_epb.dma_map != NULL) 2347104478Ssam ubsec_dma_free(sc, &me->me_epb); 2348104478Ssam free(me, M_DEVBUF); 2349104478Ssam } 2350104478Ssam krp->krp_status = err; 2351104478Ssam crypto_kdone(krp); 2352104478Ssam return (0); 2353104478Ssam} 2354104478Ssam 2355104630Ssam/* 2356104630Ssam * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization) 2357104630Ssam */ 2358105215Sphkstatic int 2359104630Ssamubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2360104630Ssam{ 2361104630Ssam struct ubsec_q2_modexp *me; 2362104630Ssam struct ubsec_mcr *mcr; 2363104630Ssam struct ubsec_ctx_modexp *ctx; 2364104630Ssam struct ubsec_pktbuf *epb; 2365104630Ssam int err = 0; 2366104630Ssam u_int nbits, normbits, mbits, shiftbits, ebits; 2367104630Ssam 2368104630Ssam me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2369104630Ssam if (me == NULL) { 2370104630Ssam err = ENOMEM; 2371104630Ssam goto errout; 2372104630Ssam } 2373104630Ssam bzero(me, sizeof *me); 2374104630Ssam me->me_krp = krp; 2375104630Ssam me->me_q.q_type = UBS_CTXOP_MODEXP; 2376104630Ssam 2377104630Ssam nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2378104630Ssam if (nbits <= 512) 2379104630Ssam normbits = 512; 2380104630Ssam else if (nbits <= 768) 2381104630Ssam normbits = 768; 2382104630Ssam else if (nbits <= 1024) 2383104630Ssam normbits = 1024; 2384104630Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2385104630Ssam normbits = 1536; 2386104630Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2387104630Ssam normbits = 2048; 2388104630Ssam else { 2389104630Ssam err = E2BIG; 2390104630Ssam goto errout; 2391104630Ssam } 2392104630Ssam 2393104630Ssam shiftbits = normbits - nbits; 2394104630Ssam 2395104630Ssam /* XXX ??? */ 2396104630Ssam me->me_modbits = nbits; 2397104630Ssam me->me_shiftbits = shiftbits; 2398104630Ssam me->me_normbits = normbits; 2399104630Ssam 2400104630Ssam /* Sanity check: result bits must be >= true modulus bits. */ 2401104630Ssam if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2402104630Ssam err = ERANGE; 2403104630Ssam goto errout; 2404104630Ssam } 2405104630Ssam 2406104630Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2407104630Ssam &me->me_q.q_mcr, 0)) { 2408104630Ssam err = ENOMEM; 2409104630Ssam goto errout; 2410104630Ssam } 2411104630Ssam mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2412104630Ssam 2413104630Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2414104630Ssam &me->me_q.q_ctx, 0)) { 2415104630Ssam err = ENOMEM; 2416104630Ssam goto errout; 2417104630Ssam } 2418104630Ssam 2419104630Ssam mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2420104630Ssam if (mbits > nbits) { 2421104630Ssam err = E2BIG; 2422104630Ssam goto errout; 2423104630Ssam } 2424104630Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2425104630Ssam err = ENOMEM; 2426104630Ssam goto errout; 2427104630Ssam } 2428104630Ssam bzero(me->me_M.dma_vaddr, normbits / 8); 2429104630Ssam bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p, 2430104630Ssam me->me_M.dma_vaddr, (mbits + 7) / 8); 2431104630Ssam 2432104630Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2433104630Ssam err = ENOMEM; 2434104630Ssam goto errout; 2435104630Ssam } 2436104630Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2437104630Ssam 2438104630Ssam ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2439104630Ssam if (ebits > nbits) { 2440104630Ssam err = E2BIG; 2441104630Ssam goto errout; 2442104630Ssam } 2443104630Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2444104630Ssam err = ENOMEM; 2445104630Ssam goto errout; 2446104630Ssam } 2447104630Ssam bzero(me->me_E.dma_vaddr, normbits / 8); 2448104630Ssam bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p, 2449104630Ssam me->me_E.dma_vaddr, (ebits + 7) / 8); 2450104630Ssam 2451104630Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2452104630Ssam &me->me_epb, 0)) { 2453104630Ssam err = ENOMEM; 2454104630Ssam goto errout; 2455104630Ssam } 2456104630Ssam epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2457104630Ssam epb->pb_addr = htole32(me->me_E.dma_paddr); 2458104630Ssam epb->pb_next = 0; 2459104630Ssam epb->pb_len = htole32((ebits + 7) / 8); 2460104630Ssam 2461104630Ssam#ifdef UBSEC_DEBUG 2462108823Ssam if (ubsec_debug) { 2463108823Ssam printf("Epb "); 2464108823Ssam ubsec_dump_pb(epb); 2465108823Ssam } 2466104630Ssam#endif 2467104630Ssam 2468104630Ssam mcr->mcr_pkts = htole16(1); 2469104630Ssam mcr->mcr_flags = 0; 2470104630Ssam mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2471104630Ssam mcr->mcr_reserved = 0; 2472104630Ssam mcr->mcr_pktlen = 0; 2473104630Ssam 2474104630Ssam mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2475104630Ssam mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2476104630Ssam mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2477104630Ssam 2478104630Ssam mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2479104630Ssam mcr->mcr_opktbuf.pb_next = 0; 2480104630Ssam mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2481104630Ssam 2482104630Ssam#ifdef DIAGNOSTIC 2483104630Ssam /* Misaligned output buffer will hang the chip. */ 2484104630Ssam if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2485104630Ssam panic("%s: modexp invalid addr 0x%x\n", 2486104630Ssam device_get_nameunit(sc->sc_dev), 2487104630Ssam letoh32(mcr->mcr_opktbuf.pb_addr)); 2488104630Ssam if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2489104630Ssam panic("%s: modexp invalid len 0x%x\n", 2490104630Ssam device_get_nameunit(sc->sc_dev), 2491104630Ssam letoh32(mcr->mcr_opktbuf.pb_len)); 2492104630Ssam#endif 2493104630Ssam 2494104630Ssam ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2495104630Ssam bzero(ctx, sizeof(*ctx)); 2496104630Ssam bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N, 2497104630Ssam (nbits + 7) / 8); 2498104630Ssam ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2499104630Ssam ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2500104630Ssam ctx->me_E_len = htole16(ebits); 2501104630Ssam ctx->me_N_len = htole16(nbits); 2502104630Ssam 2503104630Ssam#ifdef UBSEC_DEBUG 2504104630Ssam if (ubsec_debug) { 2505104630Ssam ubsec_dump_mcr(mcr); 2506104630Ssam ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2507104630Ssam } 2508104630Ssam#endif 2509104630Ssam 2510104630Ssam /* 2511104630Ssam * ubsec_feed2 will sync mcr and ctx, we just need to sync 2512104630Ssam * everything else. 2513104630Ssam */ 2514108823Ssam ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE); 2515108823Ssam ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE); 2516108823Ssam ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD); 2517108823Ssam ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE); 2518104630Ssam 2519104630Ssam /* Enqueue and we're done... */ 2520115747Ssam mtx_lock(&sc->sc_mcr2lock); 2521104630Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2522104630Ssam ubsec_feed2(sc); 2523115747Ssam mtx_unlock(&sc->sc_mcr2lock); 2524104630Ssam 2525104630Ssam return (0); 2526104630Ssam 2527104630Ssamerrout: 2528104630Ssam if (me != NULL) { 2529104630Ssam if (me->me_q.q_mcr.dma_map != NULL) 2530104630Ssam ubsec_dma_free(sc, &me->me_q.q_mcr); 2531104630Ssam if (me->me_q.q_ctx.dma_map != NULL) { 2532104630Ssam bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2533104630Ssam ubsec_dma_free(sc, &me->me_q.q_ctx); 2534104630Ssam } 2535104630Ssam if (me->me_M.dma_map != NULL) { 2536104630Ssam bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2537104630Ssam ubsec_dma_free(sc, &me->me_M); 2538104630Ssam } 2539104630Ssam if (me->me_E.dma_map != NULL) { 2540104630Ssam bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2541104630Ssam ubsec_dma_free(sc, &me->me_E); 2542104630Ssam } 2543104630Ssam if (me->me_C.dma_map != NULL) { 2544104630Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2545104630Ssam ubsec_dma_free(sc, &me->me_C); 2546104630Ssam } 2547104630Ssam if (me->me_epb.dma_map != NULL) 2548104630Ssam ubsec_dma_free(sc, &me->me_epb); 2549104630Ssam free(me, M_DEVBUF); 2550104630Ssam } 2551104630Ssam krp->krp_status = err; 2552104630Ssam crypto_kdone(krp); 2553104630Ssam return (0); 2554104630Ssam} 2555104630Ssam 2556104478Ssamstatic int 2557104478Ssamubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2558104478Ssam{ 2559104478Ssam struct ubsec_q2_rsapriv *rp = NULL; 2560104478Ssam struct ubsec_mcr *mcr; 2561104478Ssam struct ubsec_ctx_rsapriv *ctx; 2562104478Ssam int err = 0; 2563104478Ssam u_int padlen, msglen; 2564104478Ssam 2565104478Ssam msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]); 2566104478Ssam padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]); 2567104478Ssam if (msglen > padlen) 2568104478Ssam padlen = msglen; 2569104478Ssam 2570104478Ssam if (padlen <= 256) 2571104478Ssam padlen = 256; 2572104478Ssam else if (padlen <= 384) 2573104478Ssam padlen = 384; 2574104478Ssam else if (padlen <= 512) 2575104478Ssam padlen = 512; 2576104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768) 2577104478Ssam padlen = 768; 2578104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024) 2579104478Ssam padlen = 1024; 2580104478Ssam else { 2581104478Ssam err = E2BIG; 2582104478Ssam goto errout; 2583104478Ssam } 2584104478Ssam 2585104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) { 2586104478Ssam err = E2BIG; 2587104478Ssam goto errout; 2588104478Ssam } 2589104478Ssam 2590104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) { 2591104478Ssam err = E2BIG; 2592104478Ssam goto errout; 2593104478Ssam } 2594104478Ssam 2595104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) { 2596104478Ssam err = E2BIG; 2597104478Ssam goto errout; 2598104478Ssam } 2599104478Ssam 2600104478Ssam rp = (struct ubsec_q2_rsapriv *)malloc(sizeof *rp, M_DEVBUF, M_NOWAIT); 2601104478Ssam if (rp == NULL) 2602104478Ssam return (ENOMEM); 2603104478Ssam bzero(rp, sizeof *rp); 2604104478Ssam rp->rpr_krp = krp; 2605104478Ssam rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV; 2606104478Ssam 2607104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2608104478Ssam &rp->rpr_q.q_mcr, 0)) { 2609104478Ssam err = ENOMEM; 2610104478Ssam goto errout; 2611104478Ssam } 2612104478Ssam mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr; 2613104478Ssam 2614104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv), 2615104478Ssam &rp->rpr_q.q_ctx, 0)) { 2616104478Ssam err = ENOMEM; 2617104478Ssam goto errout; 2618104478Ssam } 2619104478Ssam ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr; 2620104478Ssam bzero(ctx, sizeof *ctx); 2621104478Ssam 2622104478Ssam /* Copy in p */ 2623104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p, 2624104478Ssam &ctx->rpr_buf[0 * (padlen / 8)], 2625104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8); 2626104478Ssam 2627104478Ssam /* Copy in q */ 2628104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p, 2629104478Ssam &ctx->rpr_buf[1 * (padlen / 8)], 2630104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8); 2631104478Ssam 2632104478Ssam /* Copy in dp */ 2633104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p, 2634104478Ssam &ctx->rpr_buf[2 * (padlen / 8)], 2635104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8); 2636104478Ssam 2637104478Ssam /* Copy in dq */ 2638104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p, 2639104478Ssam &ctx->rpr_buf[3 * (padlen / 8)], 2640104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8); 2641104478Ssam 2642104478Ssam /* Copy in pinv */ 2643104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p, 2644104478Ssam &ctx->rpr_buf[4 * (padlen / 8)], 2645104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8); 2646104478Ssam 2647104478Ssam msglen = padlen * 2; 2648104478Ssam 2649104478Ssam /* Copy in input message (aligned buffer/length). */ 2650104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) { 2651104478Ssam /* Is this likely? */ 2652104478Ssam err = E2BIG; 2653104478Ssam goto errout; 2654104478Ssam } 2655104478Ssam if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) { 2656104478Ssam err = ENOMEM; 2657104478Ssam goto errout; 2658104478Ssam } 2659104478Ssam bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8); 2660104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p, 2661104478Ssam rp->rpr_msgin.dma_vaddr, 2662104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8); 2663104478Ssam 2664104478Ssam /* Prepare space for output message (aligned buffer/length). */ 2665104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) { 2666104478Ssam /* Is this likely? */ 2667104478Ssam err = E2BIG; 2668104478Ssam goto errout; 2669104478Ssam } 2670104478Ssam if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) { 2671104478Ssam err = ENOMEM; 2672104478Ssam goto errout; 2673104478Ssam } 2674104478Ssam bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8); 2675104478Ssam 2676104478Ssam mcr->mcr_pkts = htole16(1); 2677104478Ssam mcr->mcr_flags = 0; 2678104478Ssam mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr); 2679104478Ssam mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr); 2680104478Ssam mcr->mcr_ipktbuf.pb_next = 0; 2681104478Ssam mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size); 2682104478Ssam mcr->mcr_reserved = 0; 2683104478Ssam mcr->mcr_pktlen = htole16(msglen); 2684104478Ssam mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr); 2685104478Ssam mcr->mcr_opktbuf.pb_next = 0; 2686104478Ssam mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size); 2687104478Ssam 2688104478Ssam#ifdef DIAGNOSTIC 2689104478Ssam if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) { 2690106579Sjhb panic("%s: rsapriv: invalid msgin %x(0x%jx)", 2691104478Ssam device_get_nameunit(sc->sc_dev), 2692106579Sjhb rp->rpr_msgin.dma_paddr, (uintmax_t)rp->rpr_msgin.dma_size); 2693104478Ssam } 2694104478Ssam if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) { 2695106579Sjhb panic("%s: rsapriv: invalid msgout %x(0x%jx)", 2696104478Ssam device_get_nameunit(sc->sc_dev), 2697106579Sjhb rp->rpr_msgout.dma_paddr, (uintmax_t)rp->rpr_msgout.dma_size); 2698104478Ssam } 2699104478Ssam#endif 2700104478Ssam 2701104478Ssam ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8)); 2702104478Ssam ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV); 2703104478Ssam ctx->rpr_q_len = htole16(padlen); 2704104478Ssam ctx->rpr_p_len = htole16(padlen); 2705104478Ssam 2706104478Ssam /* 2707104478Ssam * ubsec_feed2 will sync mcr and ctx, we just need to sync 2708104478Ssam * everything else. 2709104478Ssam */ 2710108823Ssam ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_PREWRITE); 2711108823Ssam ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_PREREAD); 2712104478Ssam 2713104478Ssam /* Enqueue and we're done... */ 2714115747Ssam mtx_lock(&sc->sc_mcr2lock); 2715104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next); 2716104478Ssam ubsec_feed2(sc); 2717104478Ssam ubsecstats.hst_modexpcrt++; 2718115747Ssam mtx_unlock(&sc->sc_mcr2lock); 2719104478Ssam return (0); 2720104478Ssam 2721104478Ssamerrout: 2722104478Ssam if (rp != NULL) { 2723104478Ssam if (rp->rpr_q.q_mcr.dma_map != NULL) 2724104478Ssam ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2725104478Ssam if (rp->rpr_msgin.dma_map != NULL) { 2726104478Ssam bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 2727104478Ssam ubsec_dma_free(sc, &rp->rpr_msgin); 2728104478Ssam } 2729104478Ssam if (rp->rpr_msgout.dma_map != NULL) { 2730104478Ssam bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 2731104478Ssam ubsec_dma_free(sc, &rp->rpr_msgout); 2732104478Ssam } 2733104478Ssam free(rp, M_DEVBUF); 2734104478Ssam } 2735104478Ssam krp->krp_status = err; 2736104478Ssam crypto_kdone(krp); 2737104478Ssam return (0); 2738104478Ssam} 2739104478Ssam 2740104478Ssam#ifdef UBSEC_DEBUG 2741104478Ssamstatic void 2742104478Ssamubsec_dump_pb(volatile struct ubsec_pktbuf *pb) 2743104478Ssam{ 2744104478Ssam printf("addr 0x%x (0x%x) next 0x%x\n", 2745104478Ssam pb->pb_addr, pb->pb_len, pb->pb_next); 2746104478Ssam} 2747104478Ssam 2748104478Ssamstatic void 2749104478Ssamubsec_dump_ctx2(struct ubsec_ctx_keyop *c) 2750104478Ssam{ 2751104478Ssam printf("CTX (0x%x):\n", c->ctx_len); 2752104478Ssam switch (letoh16(c->ctx_op)) { 2753104478Ssam case UBS_CTXOP_RNGBYPASS: 2754104478Ssam case UBS_CTXOP_RNGSHA1: 2755104478Ssam break; 2756104478Ssam case UBS_CTXOP_MODEXP: 2757104478Ssam { 2758104478Ssam struct ubsec_ctx_modexp *cx = (void *)c; 2759104478Ssam int i, len; 2760104478Ssam 2761104478Ssam printf(" Elen %u, Nlen %u\n", 2762104478Ssam letoh16(cx->me_E_len), letoh16(cx->me_N_len)); 2763104478Ssam len = (cx->me_N_len + 7)/8; 2764104478Ssam for (i = 0; i < len; i++) 2765104478Ssam printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]); 2766104478Ssam printf("\n"); 2767104478Ssam break; 2768104478Ssam } 2769104478Ssam default: 2770104478Ssam printf("unknown context: %x\n", c->ctx_op); 2771104478Ssam } 2772104478Ssam printf("END CTX\n"); 2773104478Ssam} 2774104478Ssam 2775104478Ssamstatic void 2776104478Ssamubsec_dump_mcr(struct ubsec_mcr *mcr) 2777104478Ssam{ 2778104478Ssam volatile struct ubsec_mcr_add *ma; 2779104478Ssam int i; 2780104478Ssam 2781104478Ssam printf("MCR:\n"); 2782104478Ssam printf(" pkts: %u, flags 0x%x\n", 2783104478Ssam letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags)); 2784104478Ssam ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp; 2785104478Ssam for (i = 0; i < letoh16(mcr->mcr_pkts); i++) { 2786104478Ssam printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i, 2787104478Ssam letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen), 2788104478Ssam letoh16(ma->mcr_reserved)); 2789104478Ssam printf(" %d: ipkt ", i); 2790104478Ssam ubsec_dump_pb(&ma->mcr_ipktbuf); 2791104478Ssam printf(" %d: opkt ", i); 2792104478Ssam ubsec_dump_pb(&ma->mcr_opktbuf); 2793104478Ssam ma++; 2794104478Ssam } 2795104478Ssam printf("END MCR\n"); 2796104478Ssam} 2797104478Ssam#endif /* UBSEC_DEBUG */ 2798104478Ssam 2799104478Ssam/* 2800104478Ssam * Return the number of significant bits of a big number. 2801104478Ssam */ 2802104478Ssamstatic int 2803104478Ssamubsec_ksigbits(struct crparam *cr) 2804104478Ssam{ 2805104478Ssam u_int plen = (cr->crp_nbits + 7) / 8; 2806104478Ssam int i, sig = plen * 8; 2807104478Ssam u_int8_t c, *p = cr->crp_p; 2808104478Ssam 2809104478Ssam for (i = plen - 1; i >= 0; i--) { 2810104478Ssam c = p[i]; 2811104478Ssam if (c != 0) { 2812104478Ssam while ((c & 0x80) == 0) { 2813104478Ssam sig--; 2814104478Ssam c <<= 1; 2815104478Ssam } 2816104478Ssam break; 2817104478Ssam } 2818104478Ssam sig -= 8; 2819104478Ssam } 2820104478Ssam return (sig); 2821104478Ssam} 2822104478Ssam 2823104478Ssamstatic void 2824104478Ssamubsec_kshift_r( 2825104478Ssam u_int shiftbits, 2826104478Ssam u_int8_t *src, u_int srcbits, 2827104478Ssam u_int8_t *dst, u_int dstbits) 2828104478Ssam{ 2829104478Ssam u_int slen, dlen; 2830104478Ssam int i, si, di, n; 2831104478Ssam 2832104478Ssam slen = (srcbits + 7) / 8; 2833104478Ssam dlen = (dstbits + 7) / 8; 2834104478Ssam 2835104478Ssam for (i = 0; i < slen; i++) 2836104478Ssam dst[i] = src[i]; 2837104478Ssam for (i = 0; i < dlen - slen; i++) 2838104478Ssam dst[slen + i] = 0; 2839104478Ssam 2840104478Ssam n = shiftbits / 8; 2841104478Ssam if (n != 0) { 2842104478Ssam si = dlen - n - 1; 2843104478Ssam di = dlen - 1; 2844104478Ssam while (si >= 0) 2845104478Ssam dst[di--] = dst[si--]; 2846104478Ssam while (di >= 0) 2847104478Ssam dst[di--] = 0; 2848104478Ssam } 2849104478Ssam 2850104478Ssam n = shiftbits % 8; 2851104478Ssam if (n != 0) { 2852104478Ssam for (i = dlen - 1; i > 0; i--) 2853104478Ssam dst[i] = (dst[i] << n) | 2854104478Ssam (dst[i - 1] >> (8 - n)); 2855104478Ssam dst[0] = dst[0] << n; 2856104478Ssam } 2857104478Ssam} 2858104478Ssam 2859104478Ssamstatic void 2860104478Ssamubsec_kshift_l( 2861104478Ssam u_int shiftbits, 2862104478Ssam u_int8_t *src, u_int srcbits, 2863104478Ssam u_int8_t *dst, u_int dstbits) 2864104478Ssam{ 2865104478Ssam int slen, dlen, i, n; 2866104478Ssam 2867104478Ssam slen = (srcbits + 7) / 8; 2868104478Ssam dlen = (dstbits + 7) / 8; 2869104478Ssam 2870104478Ssam n = shiftbits / 8; 2871104478Ssam for (i = 0; i < slen; i++) 2872104478Ssam dst[i] = src[i + n]; 2873104478Ssam for (i = 0; i < dlen - slen; i++) 2874104478Ssam dst[slen + i] = 0; 2875104478Ssam 2876104478Ssam n = shiftbits % 8; 2877104478Ssam if (n != 0) { 2878104478Ssam for (i = 0; i < (dlen - 1); i++) 2879104478Ssam dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n)); 2880104478Ssam dst[dlen - 1] = dst[dlen - 1] >> n; 2881104478Ssam } 2882104478Ssam} 2883