1104630Ssam/* $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $ */ 2104478Ssam 3139749Simp/*- 4104478Ssam * Copyright (c) 2000 Jason L. Wright (jason@thought.net) 5104478Ssam * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org) 6104478Ssam * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com) 7162969Sjhb * 8104478Ssam * All rights reserved. 9104478Ssam * 10104478Ssam * Redistribution and use in source and binary forms, with or without 11104478Ssam * modification, are permitted provided that the following conditions 12104478Ssam * are met: 13104478Ssam * 1. Redistributions of source code must retain the above copyright 14104478Ssam * notice, this list of conditions and the following disclaimer. 15104478Ssam * 2. Redistributions in binary form must reproduce the above copyright 16104478Ssam * notice, this list of conditions and the following disclaimer in the 17104478Ssam * documentation and/or other materials provided with the distribution. 18104478Ssam * 3. All advertising materials mentioning features or use of this software 19104478Ssam * must display the following acknowledgement: 20104478Ssam * This product includes software developed by Jason L. Wright 21104478Ssam * 4. The name of the author may not be used to endorse or promote products 22104478Ssam * derived from this software without specific prior written permission. 23104478Ssam * 24104478Ssam * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 25104478Ssam * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 26104478Ssam * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 27104478Ssam * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 28104478Ssam * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 29104478Ssam * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 30104478Ssam * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31104478Ssam * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 32104478Ssam * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 33104478Ssam * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34104478Ssam * POSSIBILITY OF SUCH DAMAGE. 35104478Ssam * 36104478Ssam * Effort sponsored in part by the Defense Advanced Research Projects 37104478Ssam * Agency (DARPA) and Air Force Research Laboratory, Air Force 38104478Ssam * Materiel Command, USAF, under agreement number F30602-01-2-0537. 39104478Ssam */ 40104478Ssam 41119418Sobrien#include <sys/cdefs.h> 42119418Sobrien__FBSDID("$FreeBSD: stable/11/sys/dev/ubsec/ubsec.c 361044 2020-05-14 18:59:34Z jhb $"); 43119418Sobrien 44104478Ssam/* 45104478Ssam * uBsec 5[56]01, 58xx hardware crypto accelerator 46104478Ssam */ 47104478Ssam 48112124Ssam#include "opt_ubsec.h" 49112124Ssam 50104478Ssam#include <sys/param.h> 51104478Ssam#include <sys/systm.h> 52104478Ssam#include <sys/proc.h> 53104478Ssam#include <sys/errno.h> 54104478Ssam#include <sys/malloc.h> 55104478Ssam#include <sys/kernel.h> 56129879Sphk#include <sys/module.h> 57104478Ssam#include <sys/mbuf.h> 58104478Ssam#include <sys/lock.h> 59104478Ssam#include <sys/mutex.h> 60104478Ssam#include <sys/sysctl.h> 61104478Ssam#include <sys/endian.h> 62104478Ssam 63104478Ssam#include <vm/vm.h> 64104478Ssam#include <vm/pmap.h> 65104478Ssam 66104478Ssam#include <machine/bus.h> 67104478Ssam#include <machine/resource.h> 68104478Ssam#include <sys/bus.h> 69104478Ssam#include <sys/rman.h> 70104478Ssam 71104478Ssam#include <crypto/sha1.h> 72104478Ssam#include <opencrypto/cryptodev.h> 73104478Ssam#include <opencrypto/cryptosoft.h> 74104478Ssam#include <sys/md5.h> 75104478Ssam#include <sys/random.h> 76167755Ssam#include <sys/kobj.h> 77104478Ssam 78167755Ssam#include "cryptodev_if.h" 79167755Ssam 80119287Simp#include <dev/pci/pcivar.h> 81119287Simp#include <dev/pci/pcireg.h> 82104478Ssam 83104478Ssam/* grr, #defines for gratuitous incompatibility in queue.h */ 84104478Ssam#define SIMPLEQ_HEAD STAILQ_HEAD 85104478Ssam#define SIMPLEQ_ENTRY STAILQ_ENTRY 86104478Ssam#define SIMPLEQ_INIT STAILQ_INIT 87104478Ssam#define SIMPLEQ_INSERT_TAIL STAILQ_INSERT_TAIL 88104478Ssam#define SIMPLEQ_EMPTY STAILQ_EMPTY 89104478Ssam#define SIMPLEQ_FIRST STAILQ_FIRST 90163648Sru#define SIMPLEQ_REMOVE_HEAD STAILQ_REMOVE_HEAD 91111416Ssam#define SIMPLEQ_FOREACH STAILQ_FOREACH 92104478Ssam/* ditto for endian.h */ 93104478Ssam#define letoh16(x) le16toh(x) 94104478Ssam#define letoh32(x) le32toh(x) 95104478Ssam 96112124Ssam#ifdef UBSEC_RNDTEST 97112124Ssam#include <dev/rndtest/rndtest.h> 98112124Ssam#endif 99104478Ssam#include <dev/ubsec/ubsecreg.h> 100104478Ssam#include <dev/ubsec/ubsecvar.h> 101104478Ssam 102104478Ssam/* 103104478Ssam * Prototypes and count for the pci_device structure 104104478Ssam */ 105104478Ssamstatic int ubsec_probe(device_t); 106104478Ssamstatic int ubsec_attach(device_t); 107104478Ssamstatic int ubsec_detach(device_t); 108104478Ssamstatic int ubsec_suspend(device_t); 109104478Ssamstatic int ubsec_resume(device_t); 110194023Savgstatic int ubsec_shutdown(device_t); 111104478Ssam 112167755Ssamstatic int ubsec_newsession(device_t, u_int32_t *, struct cryptoini *); 113167755Ssamstatic int ubsec_freesession(device_t, u_int64_t); 114167755Ssamstatic int ubsec_process(device_t, struct cryptop *, int); 115167755Ssamstatic int ubsec_kprocess(device_t, struct cryptkop *, int); 116167755Ssam 117104478Ssamstatic device_method_t ubsec_methods[] = { 118104478Ssam /* Device interface */ 119104478Ssam DEVMETHOD(device_probe, ubsec_probe), 120104478Ssam DEVMETHOD(device_attach, ubsec_attach), 121104478Ssam DEVMETHOD(device_detach, ubsec_detach), 122104478Ssam DEVMETHOD(device_suspend, ubsec_suspend), 123104478Ssam DEVMETHOD(device_resume, ubsec_resume), 124104478Ssam DEVMETHOD(device_shutdown, ubsec_shutdown), 125104478Ssam 126167755Ssam /* crypto device methods */ 127167755Ssam DEVMETHOD(cryptodev_newsession, ubsec_newsession), 128167755Ssam DEVMETHOD(cryptodev_freesession,ubsec_freesession), 129167755Ssam DEVMETHOD(cryptodev_process, ubsec_process), 130167755Ssam DEVMETHOD(cryptodev_kprocess, ubsec_kprocess), 131167755Ssam 132227843Smarius DEVMETHOD_END 133104478Ssam}; 134104478Ssamstatic driver_t ubsec_driver = { 135104478Ssam "ubsec", 136104478Ssam ubsec_methods, 137104478Ssam sizeof (struct ubsec_softc) 138104478Ssam}; 139104478Ssamstatic devclass_t ubsec_devclass; 140104478Ssam 141104478SsamDRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0); 142105251SmarkmMODULE_DEPEND(ubsec, crypto, 1, 1, 1); 143112124Ssam#ifdef UBSEC_RNDTEST 144112124SsamMODULE_DEPEND(ubsec, rndtest, 1, 1, 1); 145112124Ssam#endif 146104478Ssam 147104478Ssamstatic void ubsec_intr(void *); 148104478Ssamstatic void ubsec_callback(struct ubsec_softc *, struct ubsec_q *); 149111416Ssamstatic void ubsec_feed(struct ubsec_softc *); 150104478Ssamstatic void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int); 151104478Ssamstatic void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *); 152104478Ssamstatic int ubsec_feed2(struct ubsec_softc *); 153104478Ssamstatic void ubsec_rng(void *); 154104478Ssamstatic int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t, 155104478Ssam struct ubsec_dma_alloc *, int); 156108823Ssam#define ubsec_dma_sync(_dma, _flags) \ 157108823Ssam bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags)) 158104478Ssamstatic void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *); 159104478Ssamstatic int ubsec_dmamap_aligned(struct ubsec_operand *op); 160104478Ssam 161104478Ssamstatic void ubsec_reset_board(struct ubsec_softc *sc); 162104478Ssamstatic void ubsec_init_board(struct ubsec_softc *sc); 163104478Ssamstatic void ubsec_init_pciregs(device_t dev); 164104478Ssamstatic void ubsec_totalreset(struct ubsec_softc *sc); 165104478Ssam 166104478Ssamstatic int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q); 167104478Ssam 168104630Ssamstatic int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int); 169104630Ssamstatic int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int); 170104478Ssamstatic int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int); 171104478Ssamstatic void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *); 172104478Ssamstatic int ubsec_ksigbits(struct crparam *); 173104478Ssamstatic void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 174104478Ssamstatic void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 175104478Ssam 176227309Sedstatic SYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0, 177227309Sed "Broadcom driver parameters"); 178109595Ssam 179104478Ssam#ifdef UBSEC_DEBUG 180104478Ssamstatic void ubsec_dump_pb(volatile struct ubsec_pktbuf *); 181104478Ssamstatic void ubsec_dump_mcr(struct ubsec_mcr *); 182104478Ssamstatic void ubsec_dump_ctx2(struct ubsec_ctx_keyop *); 183104478Ssam 184104478Ssamstatic int ubsec_debug = 0; 185109595SsamSYSCTL_INT(_hw_ubsec, OID_AUTO, debug, CTLFLAG_RW, &ubsec_debug, 186109595Ssam 0, "control debugging msgs"); 187104478Ssam#endif 188104478Ssam 189104478Ssam#define READ_REG(sc,r) \ 190104478Ssam bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r)) 191104478Ssam 192104478Ssam#define WRITE_REG(sc,reg,val) \ 193104478Ssam bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val) 194104478Ssam 195104478Ssam#define SWAP32(x) (x) = htole32(ntohl((x))) 196104478Ssam#define HTOLE32(x) (x) = htole32(x) 197104478Ssam 198104478Ssamstruct ubsec_stats ubsecstats; 199109595SsamSYSCTL_STRUCT(_hw_ubsec, OID_AUTO, stats, CTLFLAG_RD, &ubsecstats, 200109595Ssam ubsec_stats, "driver statistics"); 201104478Ssam 202105215Sphkstatic int 203104478Ssamubsec_probe(device_t dev) 204104478Ssam{ 205114105Ssam if (pci_get_vendor(dev) == PCI_VENDOR_SUN && 206114105Ssam (pci_get_device(dev) == PCI_PRODUCT_SUN_5821 || 207114105Ssam pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K)) 208142880Simp return (BUS_PROBE_DEFAULT); 209104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 210104478Ssam (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 || 211104478Ssam pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)) 212142880Simp return (BUS_PROBE_DEFAULT); 213104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 214111646Ssam (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5801 || 215111646Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 || 216111646Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 || 217104478Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 || 218104478Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 219110522Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 || 220191894Sphilip pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 || 221191894Sphilip pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5825 222110522Ssam )) 223142880Simp return (BUS_PROBE_DEFAULT); 224104478Ssam return (ENXIO); 225104478Ssam} 226104478Ssam 227104478Ssamstatic const char* 228104478Ssamubsec_partname(struct ubsec_softc *sc) 229104478Ssam{ 230104478Ssam /* XXX sprintf numbers when not decoded */ 231104478Ssam switch (pci_get_vendor(sc->sc_dev)) { 232104478Ssam case PCI_VENDOR_BROADCOM: 233104478Ssam switch (pci_get_device(sc->sc_dev)) { 234111646Ssam case PCI_PRODUCT_BROADCOM_5801: return "Broadcom 5801"; 235111646Ssam case PCI_PRODUCT_BROADCOM_5802: return "Broadcom 5802"; 236104478Ssam case PCI_PRODUCT_BROADCOM_5805: return "Broadcom 5805"; 237104478Ssam case PCI_PRODUCT_BROADCOM_5820: return "Broadcom 5820"; 238104478Ssam case PCI_PRODUCT_BROADCOM_5821: return "Broadcom 5821"; 239104478Ssam case PCI_PRODUCT_BROADCOM_5822: return "Broadcom 5822"; 240110522Ssam case PCI_PRODUCT_BROADCOM_5823: return "Broadcom 5823"; 241191894Sphilip case PCI_PRODUCT_BROADCOM_5825: return "Broadcom 5825"; 242104478Ssam } 243104478Ssam return "Broadcom unknown-part"; 244104478Ssam case PCI_VENDOR_BLUESTEEL: 245104478Ssam switch (pci_get_device(sc->sc_dev)) { 246104478Ssam case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601"; 247104478Ssam } 248104478Ssam return "Bluesteel unknown-part"; 249114105Ssam case PCI_VENDOR_SUN: 250114105Ssam switch (pci_get_device(sc->sc_dev)) { 251114105Ssam case PCI_PRODUCT_SUN_5821: return "Sun Crypto 5821"; 252114105Ssam case PCI_PRODUCT_SUN_SCA1K: return "Sun Crypto 1K"; 253114105Ssam } 254114105Ssam return "Sun unknown-part"; 255104478Ssam } 256104478Ssam return "Unknown-vendor unknown-part"; 257104478Ssam} 258104478Ssam 259112124Ssamstatic void 260112124Ssamdefault_harvest(struct rndtest_state *rsp, void *buf, u_int count) 261112124Ssam{ 262284959Smarkm /* MarkM: FIX!! Check that this does not swamp the harvester! */ 263284959Smarkm random_harvest_queue(buf, count, count*NBBY/2, RANDOM_PURE_UBSEC); 264112124Ssam} 265112124Ssam 266104478Ssamstatic int 267104478Ssamubsec_attach(device_t dev) 268104478Ssam{ 269104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 270104478Ssam struct ubsec_dma *dmap; 271254263Sscottl u_int32_t i; 272104478Ssam int rid; 273104478Ssam 274104478Ssam bzero(sc, sizeof (*sc)); 275104478Ssam sc->sc_dev = dev; 276104478Ssam 277104478Ssam SIMPLEQ_INIT(&sc->sc_queue); 278104478Ssam SIMPLEQ_INIT(&sc->sc_qchip); 279104478Ssam SIMPLEQ_INIT(&sc->sc_queue2); 280104478Ssam SIMPLEQ_INIT(&sc->sc_qchip2); 281104478Ssam SIMPLEQ_INIT(&sc->sc_q2free); 282104478Ssam 283104478Ssam /* XXX handle power management */ 284104478Ssam 285104478Ssam sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR; 286104478Ssam 287104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 288104478Ssam pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601) 289104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 290104478Ssam 291104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 292111646Ssam (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 || 293111646Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805)) 294104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 295104478Ssam 296104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 297104478Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820) 298104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 299104478Ssam UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 300104478Ssam 301114105Ssam if ((pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 302114105Ssam (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 303114105Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 || 304191894Sphilip pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 || 305191894Sphilip pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5825)) || 306114105Ssam (pci_get_vendor(dev) == PCI_VENDOR_SUN && 307114105Ssam (pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K || 308114105Ssam pci_get_device(dev) == PCI_PRODUCT_SUN_5821))) { 309104478Ssam /* NB: the 5821/5822 defines some additional status bits */ 310104478Ssam sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY | 311104478Ssam BS_STAT_MCR2_ALLEMPTY; 312104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 313104478Ssam UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 314104478Ssam } 315162969Sjhb 316254263Sscottl pci_enable_busmaster(dev); 317104478Ssam 318162969Sjhb /* 319104478Ssam * Setup memory-mapping of PCI registers. 320104478Ssam */ 321104478Ssam rid = BS_BAR; 322127135Snjl sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 323127135Snjl RF_ACTIVE); 324104478Ssam if (sc->sc_sr == NULL) { 325104478Ssam device_printf(dev, "cannot map register space\n"); 326104478Ssam goto bad; 327104478Ssam } 328104478Ssam sc->sc_st = rman_get_bustag(sc->sc_sr); 329104478Ssam sc->sc_sh = rman_get_bushandle(sc->sc_sr); 330104478Ssam 331104478Ssam /* 332104478Ssam * Arrange interrupt line. 333104478Ssam */ 334104478Ssam rid = 0; 335127135Snjl sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 336127135Snjl RF_SHAREABLE|RF_ACTIVE); 337104478Ssam if (sc->sc_irq == NULL) { 338104478Ssam device_printf(dev, "could not map interrupt\n"); 339108823Ssam goto bad1; 340104478Ssam } 341104478Ssam /* 342104478Ssam * NB: Network code assumes we are blocked with splimp() 343104478Ssam * so make sure the IRQ is mapped appropriately. 344104478Ssam */ 345115747Ssam if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE, 346166901Spiso NULL, ubsec_intr, sc, &sc->sc_ih)) { 347104478Ssam device_printf(dev, "could not establish interrupt\n"); 348108823Ssam goto bad2; 349104478Ssam } 350104478Ssam 351167755Ssam sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); 352104478Ssam if (sc->sc_cid < 0) { 353104478Ssam device_printf(dev, "could not get crypto driver id\n"); 354108823Ssam goto bad3; 355104478Ssam } 356104478Ssam 357104478Ssam /* 358104478Ssam * Setup DMA descriptor area. 359104478Ssam */ 360232874Sscottl if (bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 361104478Ssam 1, 0, /* alignment, bounds */ 362104478Ssam BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 363104478Ssam BUS_SPACE_MAXADDR, /* highaddr */ 364104478Ssam NULL, NULL, /* filter, filterarg */ 365108823Ssam 0x3ffff, /* maxsize */ 366104478Ssam UBS_MAX_SCATTER, /* nsegments */ 367108823Ssam 0xffff, /* maxsegsize */ 368104478Ssam BUS_DMA_ALLOCNOW, /* flags */ 369117126Sscottl NULL, NULL, /* lockfunc, lockarg */ 370104478Ssam &sc->sc_dmat)) { 371104478Ssam device_printf(dev, "cannot allocate DMA tag\n"); 372108823Ssam goto bad4; 373104478Ssam } 374104478Ssam SIMPLEQ_INIT(&sc->sc_freequeue); 375104478Ssam dmap = sc->sc_dmaa; 376104478Ssam for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) { 377104478Ssam struct ubsec_q *q; 378104478Ssam 379104478Ssam q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q), 380104478Ssam M_DEVBUF, M_NOWAIT); 381104478Ssam if (q == NULL) { 382104478Ssam device_printf(dev, "cannot allocate queue buffers\n"); 383104478Ssam break; 384104478Ssam } 385104478Ssam 386104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk), 387104478Ssam &dmap->d_alloc, 0)) { 388104478Ssam device_printf(dev, "cannot allocate dma buffers\n"); 389104478Ssam free(q, M_DEVBUF); 390104478Ssam break; 391104478Ssam } 392104478Ssam dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr; 393104478Ssam 394104478Ssam q->q_dma = dmap; 395104478Ssam sc->sc_queuea[i] = q; 396104478Ssam 397104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 398104478Ssam } 399115747Ssam mtx_init(&sc->sc_mcr1lock, device_get_nameunit(dev), 400115747Ssam "mcr1 operations", MTX_DEF); 401115747Ssam mtx_init(&sc->sc_freeqlock, device_get_nameunit(dev), 402115747Ssam "mcr1 free q", MTX_DEF); 403104478Ssam 404104478Ssam device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc)); 405104478Ssam 406167755Ssam crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); 407167755Ssam crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); 408167755Ssam crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 409167755Ssam crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 410104478Ssam 411104478Ssam /* 412104478Ssam * Reset Broadcom chip 413104478Ssam */ 414104478Ssam ubsec_reset_board(sc); 415104478Ssam 416104478Ssam /* 417104478Ssam * Init Broadcom specific PCI settings 418104478Ssam */ 419104478Ssam ubsec_init_pciregs(dev); 420104478Ssam 421104478Ssam /* 422104478Ssam * Init Broadcom chip 423104478Ssam */ 424104478Ssam ubsec_init_board(sc); 425104478Ssam 426104478Ssam#ifndef UBSEC_NO_RNG 427104478Ssam if (sc->sc_flags & UBS_FLAGS_RNG) { 428104478Ssam sc->sc_statmask |= BS_STAT_MCR2_DONE; 429112124Ssam#ifdef UBSEC_RNDTEST 430112124Ssam sc->sc_rndtest = rndtest_attach(dev); 431112124Ssam if (sc->sc_rndtest) 432112124Ssam sc->sc_harvest = rndtest_harvest; 433112124Ssam else 434112124Ssam sc->sc_harvest = default_harvest; 435112124Ssam#else 436112124Ssam sc->sc_harvest = default_harvest; 437112124Ssam#endif 438104478Ssam 439104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 440104478Ssam &sc->sc_rng.rng_q.q_mcr, 0)) 441104478Ssam goto skip_rng; 442104478Ssam 443104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass), 444104478Ssam &sc->sc_rng.rng_q.q_ctx, 0)) { 445104478Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 446104478Ssam goto skip_rng; 447104478Ssam } 448104478Ssam 449104478Ssam if (ubsec_dma_malloc(sc, sizeof(u_int32_t) * 450104478Ssam UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) { 451104478Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 452104478Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 453104478Ssam goto skip_rng; 454104478Ssam } 455104478Ssam 456104478Ssam if (hz >= 100) 457104478Ssam sc->sc_rnghz = hz / 100; 458104478Ssam else 459104478Ssam sc->sc_rnghz = 1; 460283291Sjkim callout_init(&sc->sc_rngto, 1); 461104478Ssam callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 462104478Ssamskip_rng: 463104478Ssam ; 464104478Ssam } 465104478Ssam#endif /* UBSEC_NO_RNG */ 466115747Ssam mtx_init(&sc->sc_mcr2lock, device_get_nameunit(dev), 467115747Ssam "mcr2 operations", MTX_DEF); 468104478Ssam 469104478Ssam if (sc->sc_flags & UBS_FLAGS_KEY) { 470104478Ssam sc->sc_statmask |= BS_STAT_MCR2_DONE; 471104478Ssam 472167755Ssam crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0); 473104630Ssam#if 0 474167755Ssam crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0); 475104630Ssam#endif 476104478Ssam } 477361044Sjhb gone_in_dev(dev, 13, "Does not support modern crypto algorithms"); 478104478Ssam return (0); 479108823Ssambad4: 480108823Ssam crypto_unregister_all(sc->sc_cid); 481108823Ssambad3: 482108823Ssam bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 483108823Ssambad2: 484108823Ssam bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 485108823Ssambad1: 486108823Ssam bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 487104478Ssambad: 488104478Ssam return (ENXIO); 489104478Ssam} 490104478Ssam 491104478Ssam/* 492104478Ssam * Detach a device that successfully probed. 493104478Ssam */ 494104478Ssamstatic int 495104478Ssamubsec_detach(device_t dev) 496104478Ssam{ 497104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 498104478Ssam 499108823Ssam /* XXX wait/abort active ops */ 500108823Ssam 501115747Ssam /* disable interrupts */ 502115747Ssam WRITE_REG(sc, BS_CTRL, READ_REG(sc, BS_CTRL) &~ 503115747Ssam (BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR)); 504104478Ssam 505104478Ssam callout_stop(&sc->sc_rngto); 506104478Ssam 507104478Ssam crypto_unregister_all(sc->sc_cid); 508104478Ssam 509112124Ssam#ifdef UBSEC_RNDTEST 510112124Ssam if (sc->sc_rndtest) 511112124Ssam rndtest_detach(sc->sc_rndtest); 512112124Ssam#endif 513112124Ssam 514108823Ssam while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 515108823Ssam struct ubsec_q *q; 516108823Ssam 517108823Ssam q = SIMPLEQ_FIRST(&sc->sc_freequeue); 518163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next); 519108823Ssam ubsec_dma_free(sc, &q->q_dma->d_alloc); 520108823Ssam free(q, M_DEVBUF); 521108823Ssam } 522115747Ssam mtx_destroy(&sc->sc_mcr1lock); 523159224Spjd mtx_destroy(&sc->sc_freeqlock); 524108823Ssam#ifndef UBSEC_NO_RNG 525108823Ssam if (sc->sc_flags & UBS_FLAGS_RNG) { 526108823Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 527108823Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 528108823Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_buf); 529108823Ssam } 530108823Ssam#endif /* UBSEC_NO_RNG */ 531115747Ssam mtx_destroy(&sc->sc_mcr2lock); 532108823Ssam 533104478Ssam bus_generic_detach(dev); 534104478Ssam bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 535104478Ssam bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 536104478Ssam 537104478Ssam bus_dma_tag_destroy(sc->sc_dmat); 538104478Ssam bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 539104478Ssam 540104478Ssam return (0); 541104478Ssam} 542104478Ssam 543104478Ssam/* 544104478Ssam * Stop all chip i/o so that the kernel's probe routines don't 545104478Ssam * get confused by errant DMAs when rebooting. 546104478Ssam */ 547194023Savgstatic int 548104478Ssamubsec_shutdown(device_t dev) 549104478Ssam{ 550104478Ssam#ifdef notyet 551104478Ssam ubsec_stop(device_get_softc(dev)); 552104478Ssam#endif 553194023Savg return (0); 554104478Ssam} 555104478Ssam 556104478Ssam/* 557104478Ssam * Device suspend routine. 558104478Ssam */ 559104478Ssamstatic int 560104478Ssamubsec_suspend(device_t dev) 561104478Ssam{ 562104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 563104478Ssam 564104478Ssam#ifdef notyet 565104478Ssam /* XXX stop the device and save PCI settings */ 566104478Ssam#endif 567104478Ssam sc->sc_suspended = 1; 568104478Ssam 569104478Ssam return (0); 570104478Ssam} 571104478Ssam 572104478Ssamstatic int 573104478Ssamubsec_resume(device_t dev) 574104478Ssam{ 575104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 576104478Ssam 577104478Ssam#ifdef notyet 578104478Ssam /* XXX retore PCI settings and start the device */ 579104478Ssam#endif 580104478Ssam sc->sc_suspended = 0; 581104478Ssam return (0); 582104478Ssam} 583104478Ssam 584104478Ssam/* 585104478Ssam * UBSEC Interrupt routine 586104478Ssam */ 587104478Ssamstatic void 588104478Ssamubsec_intr(void *arg) 589104478Ssam{ 590104478Ssam struct ubsec_softc *sc = arg; 591104478Ssam volatile u_int32_t stat; 592104478Ssam struct ubsec_q *q; 593104478Ssam struct ubsec_dma *dmap; 594104478Ssam int npkts = 0, i; 595104478Ssam 596104478Ssam stat = READ_REG(sc, BS_STAT); 597104478Ssam stat &= sc->sc_statmask; 598115747Ssam if (stat == 0) 599104478Ssam return; 600104478Ssam 601104478Ssam WRITE_REG(sc, BS_STAT, stat); /* IACK */ 602104478Ssam 603104478Ssam /* 604104478Ssam * Check to see if we have any packets waiting for us 605104478Ssam */ 606104478Ssam if ((stat & BS_STAT_MCR1_DONE)) { 607115747Ssam mtx_lock(&sc->sc_mcr1lock); 608104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 609104478Ssam q = SIMPLEQ_FIRST(&sc->sc_qchip); 610104478Ssam dmap = q->q_dma; 611104478Ssam 612104478Ssam if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0) 613104478Ssam break; 614104478Ssam 615163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next); 616104478Ssam 617104478Ssam npkts = q->q_nstacked_mcrs; 618108471Ssam sc->sc_nqchip -= 1+npkts; 619104478Ssam /* 620104478Ssam * search for further sc_qchip ubsec_q's that share 621104478Ssam * the same MCR, and complete them too, they must be 622104478Ssam * at the top. 623104478Ssam */ 624104478Ssam for (i = 0; i < npkts; i++) { 625104478Ssam if(q->q_stacked_mcr[i]) { 626104478Ssam ubsec_callback(sc, q->q_stacked_mcr[i]); 627104478Ssam } else { 628104478Ssam break; 629104478Ssam } 630104478Ssam } 631104478Ssam ubsec_callback(sc, q); 632104478Ssam } 633104478Ssam /* 634104478Ssam * Don't send any more packet to chip if there has been 635104478Ssam * a DMAERR. 636104478Ssam */ 637104478Ssam if (!(stat & BS_STAT_DMAERR)) 638104478Ssam ubsec_feed(sc); 639115747Ssam mtx_unlock(&sc->sc_mcr1lock); 640104478Ssam } 641104478Ssam 642104478Ssam /* 643104478Ssam * Check to see if we have any key setups/rng's waiting for us 644104478Ssam */ 645104478Ssam if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) && 646104478Ssam (stat & BS_STAT_MCR2_DONE)) { 647104478Ssam struct ubsec_q2 *q2; 648104478Ssam struct ubsec_mcr *mcr; 649104478Ssam 650115747Ssam mtx_lock(&sc->sc_mcr2lock); 651104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) { 652104478Ssam q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 653104478Ssam 654108823Ssam ubsec_dma_sync(&q2->q_mcr, 655104478Ssam BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 656104478Ssam 657104478Ssam mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 658104478Ssam if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) { 659108823Ssam ubsec_dma_sync(&q2->q_mcr, 660104478Ssam BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 661104478Ssam break; 662104478Ssam } 663163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q_next); 664104478Ssam ubsec_callback2(sc, q2); 665104478Ssam /* 666104478Ssam * Don't send any more packet to chip if there has been 667104478Ssam * a DMAERR. 668104478Ssam */ 669104478Ssam if (!(stat & BS_STAT_DMAERR)) 670104478Ssam ubsec_feed2(sc); 671104478Ssam } 672115747Ssam mtx_unlock(&sc->sc_mcr2lock); 673104478Ssam } 674104478Ssam 675104478Ssam /* 676104478Ssam * Check to see if we got any DMA Error 677104478Ssam */ 678104478Ssam if (stat & BS_STAT_DMAERR) { 679104478Ssam#ifdef UBSEC_DEBUG 680104478Ssam if (ubsec_debug) { 681104478Ssam volatile u_int32_t a = READ_REG(sc, BS_ERR); 682104478Ssam 683104478Ssam printf("dmaerr %s@%08x\n", 684104478Ssam (a & BS_ERR_READ) ? "read" : "write", 685104478Ssam a & BS_ERR_ADDR); 686104478Ssam } 687104478Ssam#endif /* UBSEC_DEBUG */ 688104478Ssam ubsecstats.hst_dmaerr++; 689115747Ssam mtx_lock(&sc->sc_mcr1lock); 690104478Ssam ubsec_totalreset(sc); 691104478Ssam ubsec_feed(sc); 692115747Ssam mtx_unlock(&sc->sc_mcr1lock); 693104478Ssam } 694104478Ssam 695104478Ssam if (sc->sc_needwakeup) { /* XXX check high watermark */ 696158828Spjd int wakeup; 697158828Spjd 698158828Spjd mtx_lock(&sc->sc_freeqlock); 699158828Spjd wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ); 700104478Ssam#ifdef UBSEC_DEBUG 701104478Ssam if (ubsec_debug) 702104478Ssam device_printf(sc->sc_dev, "wakeup crypto (%x)\n", 703104478Ssam sc->sc_needwakeup); 704104478Ssam#endif /* UBSEC_DEBUG */ 705104478Ssam sc->sc_needwakeup &= ~wakeup; 706158828Spjd mtx_unlock(&sc->sc_freeqlock); 707104478Ssam crypto_unblock(sc->sc_cid, wakeup); 708104478Ssam } 709104478Ssam} 710104478Ssam 711104478Ssam/* 712104478Ssam * ubsec_feed() - aggregate and post requests to chip 713104478Ssam */ 714111416Ssamstatic void 715104478Ssamubsec_feed(struct ubsec_softc *sc) 716104478Ssam{ 717104478Ssam struct ubsec_q *q, *q2; 718104478Ssam int npkts, i; 719104478Ssam void *v; 720104478Ssam u_int32_t stat; 721104478Ssam 722108471Ssam /* 723108471Ssam * Decide how many ops to combine in a single MCR. We cannot 724108471Ssam * aggregate more than UBS_MAX_AGGR because this is the number 725111416Ssam * of slots defined in the data structure. Note that 726111416Ssam * aggregation only happens if ops are marked batch'able. 727108471Ssam * Aggregating ops reduces the number of interrupts to the host 728108471Ssam * but also (potentially) increases the latency for processing 729108471Ssam * completed ops as we only get an interrupt when all aggregated 730108471Ssam * ops have completed. 731108471Ssam */ 732111416Ssam if (sc->sc_nqueue == 0) 733111416Ssam return; 734111416Ssam if (sc->sc_nqueue > 1) { 735111416Ssam npkts = 0; 736111416Ssam SIMPLEQ_FOREACH(q, &sc->sc_queue, q_next) { 737111416Ssam npkts++; 738111416Ssam if ((q->q_crp->crp_flags & CRYPTO_F_BATCH) == 0) 739111416Ssam break; 740111416Ssam } 741111416Ssam } else 742111416Ssam npkts = 1; 743111416Ssam /* 744111416Ssam * Check device status before going any further. 745111416Ssam */ 746104478Ssam if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 747108471Ssam if (stat & BS_STAT_DMAERR) { 748104478Ssam ubsec_totalreset(sc); 749104478Ssam ubsecstats.hst_dmaerr++; 750108471Ssam } else 751108471Ssam ubsecstats.hst_mcr1full++; 752111416Ssam return; 753104478Ssam } 754111416Ssam if (sc->sc_nqueue > ubsecstats.hst_maxqueue) 755111416Ssam ubsecstats.hst_maxqueue = sc->sc_nqueue; 756111416Ssam if (npkts > UBS_MAX_AGGR) 757111416Ssam npkts = UBS_MAX_AGGR; 758111416Ssam if (npkts < 2) /* special case 1 op */ 759111416Ssam goto feed1; 760104478Ssam 761111416Ssam ubsecstats.hst_totbatch += npkts-1; 762104478Ssam#ifdef UBSEC_DEBUG 763104478Ssam if (ubsec_debug) 764104478Ssam printf("merging %d records\n", npkts); 765104478Ssam#endif /* UBSEC_DEBUG */ 766104478Ssam 767104478Ssam q = SIMPLEQ_FIRST(&sc->sc_queue); 768163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next); 769104478Ssam --sc->sc_nqueue; 770104478Ssam 771104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE); 772104478Ssam if (q->q_dst_map != NULL) 773104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD); 774104478Ssam 775104478Ssam q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */ 776104478Ssam 777104478Ssam for (i = 0; i < q->q_nstacked_mcrs; i++) { 778104478Ssam q2 = SIMPLEQ_FIRST(&sc->sc_queue); 779104478Ssam bus_dmamap_sync(sc->sc_dmat, q2->q_src_map, 780104478Ssam BUS_DMASYNC_PREWRITE); 781104478Ssam if (q2->q_dst_map != NULL) 782104478Ssam bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map, 783104478Ssam BUS_DMASYNC_PREREAD); 784163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next); 785104478Ssam --sc->sc_nqueue; 786104478Ssam 787104478Ssam v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) - 788104478Ssam sizeof(struct ubsec_mcr_add)); 789104478Ssam bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add)); 790104478Ssam q->q_stacked_mcr[i] = q2; 791104478Ssam } 792104478Ssam q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts); 793104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 794108471Ssam sc->sc_nqchip += npkts; 795108471Ssam if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 796108471Ssam ubsecstats.hst_maxqchip = sc->sc_nqchip; 797108823Ssam ubsec_dma_sync(&q->q_dma->d_alloc, 798104478Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 799104478Ssam WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 800104478Ssam offsetof(struct ubsec_dmachunk, d_mcr)); 801111416Ssam return; 802104478Ssamfeed1: 803111416Ssam q = SIMPLEQ_FIRST(&sc->sc_queue); 804104478Ssam 805111416Ssam bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE); 806111416Ssam if (q->q_dst_map != NULL) 807111416Ssam bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD); 808111416Ssam ubsec_dma_sync(&q->q_dma->d_alloc, 809111416Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 810104478Ssam 811111416Ssam WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 812111416Ssam offsetof(struct ubsec_dmachunk, d_mcr)); 813104478Ssam#ifdef UBSEC_DEBUG 814111416Ssam if (ubsec_debug) 815111416Ssam printf("feed1: q->chip %p %08x stat %08x\n", 816111416Ssam q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr), 817111416Ssam stat); 818104478Ssam#endif /* UBSEC_DEBUG */ 819163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next); 820111416Ssam --sc->sc_nqueue; 821111416Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 822111416Ssam sc->sc_nqchip++; 823108471Ssam if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 824108471Ssam ubsecstats.hst_maxqchip = sc->sc_nqchip; 825111416Ssam return; 826104478Ssam} 827104478Ssam 828159225Spjdstatic void 829159225Spjdubsec_setup_enckey(struct ubsec_session *ses, int algo, caddr_t key) 830159225Spjd{ 831159225Spjd 832159225Spjd /* Go ahead and compute key in ubsec's byte order */ 833159225Spjd if (algo == CRYPTO_DES_CBC) { 834159225Spjd bcopy(key, &ses->ses_deskey[0], 8); 835159225Spjd bcopy(key, &ses->ses_deskey[2], 8); 836159225Spjd bcopy(key, &ses->ses_deskey[4], 8); 837159225Spjd } else 838159225Spjd bcopy(key, ses->ses_deskey, 24); 839159225Spjd 840159225Spjd SWAP32(ses->ses_deskey[0]); 841159225Spjd SWAP32(ses->ses_deskey[1]); 842159225Spjd SWAP32(ses->ses_deskey[2]); 843159225Spjd SWAP32(ses->ses_deskey[3]); 844159225Spjd SWAP32(ses->ses_deskey[4]); 845159225Spjd SWAP32(ses->ses_deskey[5]); 846159225Spjd} 847159225Spjd 848159225Spjdstatic void 849159225Spjdubsec_setup_mackey(struct ubsec_session *ses, int algo, caddr_t key, int klen) 850159225Spjd{ 851159225Spjd MD5_CTX md5ctx; 852159225Spjd SHA1_CTX sha1ctx; 853159225Spjd int i; 854159225Spjd 855159225Spjd for (i = 0; i < klen; i++) 856159225Spjd key[i] ^= HMAC_IPAD_VAL; 857159225Spjd 858159225Spjd if (algo == CRYPTO_MD5_HMAC) { 859159225Spjd MD5Init(&md5ctx); 860159225Spjd MD5Update(&md5ctx, key, klen); 861159232Spjd MD5Update(&md5ctx, hmac_ipad_buffer, MD5_HMAC_BLOCK_LEN - klen); 862159225Spjd bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state)); 863159225Spjd } else { 864159225Spjd SHA1Init(&sha1ctx); 865159225Spjd SHA1Update(&sha1ctx, key, klen); 866159232Spjd SHA1Update(&sha1ctx, hmac_ipad_buffer, 867159232Spjd SHA1_HMAC_BLOCK_LEN - klen); 868159225Spjd bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32)); 869159225Spjd } 870159225Spjd 871159225Spjd for (i = 0; i < klen; i++) 872159225Spjd key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 873159225Spjd 874159225Spjd if (algo == CRYPTO_MD5_HMAC) { 875159225Spjd MD5Init(&md5ctx); 876159225Spjd MD5Update(&md5ctx, key, klen); 877159232Spjd MD5Update(&md5ctx, hmac_opad_buffer, MD5_HMAC_BLOCK_LEN - klen); 878159225Spjd bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state)); 879159225Spjd } else { 880159225Spjd SHA1Init(&sha1ctx); 881159225Spjd SHA1Update(&sha1ctx, key, klen); 882159232Spjd SHA1Update(&sha1ctx, hmac_opad_buffer, 883159232Spjd SHA1_HMAC_BLOCK_LEN - klen); 884159225Spjd bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32)); 885159225Spjd } 886159225Spjd 887159225Spjd for (i = 0; i < klen; i++) 888159225Spjd key[i] ^= HMAC_OPAD_VAL; 889159225Spjd} 890159225Spjd 891104478Ssam/* 892104478Ssam * Allocate a new 'session' and return an encoded session id. 'sidp' 893104478Ssam * contains our registration id, and should contain an encoded session 894104478Ssam * id on successful allocation. 895104478Ssam */ 896104478Ssamstatic int 897167755Ssamubsec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri) 898104478Ssam{ 899167755Ssam struct ubsec_softc *sc = device_get_softc(dev); 900104478Ssam struct cryptoini *c, *encini = NULL, *macini = NULL; 901104478Ssam struct ubsec_session *ses = NULL; 902159225Spjd int sesn; 903104478Ssam 904104478Ssam if (sidp == NULL || cri == NULL || sc == NULL) 905104478Ssam return (EINVAL); 906104478Ssam 907104478Ssam for (c = cri; c != NULL; c = c->cri_next) { 908104478Ssam if (c->cri_alg == CRYPTO_MD5_HMAC || 909104478Ssam c->cri_alg == CRYPTO_SHA1_HMAC) { 910104478Ssam if (macini) 911104478Ssam return (EINVAL); 912104478Ssam macini = c; 913104478Ssam } else if (c->cri_alg == CRYPTO_DES_CBC || 914104478Ssam c->cri_alg == CRYPTO_3DES_CBC) { 915104478Ssam if (encini) 916104478Ssam return (EINVAL); 917104478Ssam encini = c; 918104478Ssam } else 919104478Ssam return (EINVAL); 920104478Ssam } 921104478Ssam if (encini == NULL && macini == NULL) 922104478Ssam return (EINVAL); 923104478Ssam 924104478Ssam if (sc->sc_sessions == NULL) { 925104478Ssam ses = sc->sc_sessions = (struct ubsec_session *)malloc( 926104478Ssam sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 927104478Ssam if (ses == NULL) 928104478Ssam return (ENOMEM); 929104478Ssam sesn = 0; 930104478Ssam sc->sc_nsessions = 1; 931104478Ssam } else { 932104478Ssam for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 933104478Ssam if (sc->sc_sessions[sesn].ses_used == 0) { 934104478Ssam ses = &sc->sc_sessions[sesn]; 935104478Ssam break; 936104478Ssam } 937104478Ssam } 938104478Ssam 939104478Ssam if (ses == NULL) { 940104478Ssam sesn = sc->sc_nsessions; 941104478Ssam ses = (struct ubsec_session *)malloc((sesn + 1) * 942104478Ssam sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 943104478Ssam if (ses == NULL) 944104478Ssam return (ENOMEM); 945104478Ssam bcopy(sc->sc_sessions, ses, sesn * 946104478Ssam sizeof(struct ubsec_session)); 947104478Ssam bzero(sc->sc_sessions, sesn * 948104478Ssam sizeof(struct ubsec_session)); 949104478Ssam free(sc->sc_sessions, M_DEVBUF); 950104478Ssam sc->sc_sessions = ses; 951104478Ssam ses = &sc->sc_sessions[sesn]; 952104478Ssam sc->sc_nsessions++; 953104478Ssam } 954104478Ssam } 955104478Ssam bzero(ses, sizeof(struct ubsec_session)); 956104478Ssam ses->ses_used = 1; 957115747Ssam 958104478Ssam if (encini) { 959104478Ssam /* get an IV, network byte order */ 960104478Ssam /* XXX may read fewer than requested */ 961104478Ssam read_random(ses->ses_iv, sizeof(ses->ses_iv)); 962104478Ssam 963159225Spjd if (encini->cri_key != NULL) { 964159225Spjd ubsec_setup_enckey(ses, encini->cri_alg, 965159225Spjd encini->cri_key); 966159225Spjd } 967104478Ssam } 968104478Ssam 969104478Ssam if (macini) { 970158705Spjd ses->ses_mlen = macini->cri_mlen; 971158705Spjd if (ses->ses_mlen == 0) { 972158705Spjd if (macini->cri_alg == CRYPTO_MD5_HMAC) 973159233Spjd ses->ses_mlen = MD5_HASH_LEN; 974158705Spjd else 975159233Spjd ses->ses_mlen = SHA1_HASH_LEN; 976158705Spjd } 977158705Spjd 978159225Spjd if (macini->cri_key != NULL) { 979159225Spjd ubsec_setup_mackey(ses, macini->cri_alg, 980159225Spjd macini->cri_key, macini->cri_klen / 8); 981104478Ssam } 982104478Ssam } 983104478Ssam 984104478Ssam *sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn); 985104478Ssam return (0); 986104478Ssam} 987104478Ssam 988104478Ssam/* 989104478Ssam * Deallocate a session. 990104478Ssam */ 991104478Ssamstatic int 992167755Ssamubsec_freesession(device_t dev, u_int64_t tid) 993104478Ssam{ 994167755Ssam struct ubsec_softc *sc = device_get_softc(dev); 995115747Ssam int session, ret; 996116924Ssam u_int32_t sid = CRYPTO_SESID2LID(tid); 997104478Ssam 998104478Ssam if (sc == NULL) 999104478Ssam return (EINVAL); 1000104478Ssam 1001104478Ssam session = UBSEC_SESSION(sid); 1002115747Ssam if (session < sc->sc_nsessions) { 1003115747Ssam bzero(&sc->sc_sessions[session], 1004115747Ssam sizeof(sc->sc_sessions[session])); 1005115747Ssam ret = 0; 1006115747Ssam } else 1007115747Ssam ret = EINVAL; 1008104478Ssam 1009115747Ssam return (ret); 1010104478Ssam} 1011104478Ssam 1012104478Ssamstatic void 1013104478Ssamubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error) 1014104478Ssam{ 1015104478Ssam struct ubsec_operand *op = arg; 1016104478Ssam 1017104478Ssam KASSERT(nsegs <= UBS_MAX_SCATTER, 1018104478Ssam ("Too many DMA segments returned when mapping operand")); 1019104478Ssam#ifdef UBSEC_DEBUG 1020104478Ssam if (ubsec_debug) 1021159341Spjd printf("ubsec_op_cb: mapsize %u nsegs %d error %d\n", 1022159341Spjd (u_int) mapsize, nsegs, error); 1023104478Ssam#endif 1024159341Spjd if (error != 0) 1025159341Spjd return; 1026104478Ssam op->mapsize = mapsize; 1027104478Ssam op->nsegs = nsegs; 1028104478Ssam bcopy(seg, op->segs, nsegs * sizeof (seg[0])); 1029104478Ssam} 1030104478Ssam 1031104478Ssamstatic int 1032167755Ssamubsec_process(device_t dev, struct cryptop *crp, int hint) 1033104478Ssam{ 1034167755Ssam struct ubsec_softc *sc = device_get_softc(dev); 1035104478Ssam struct ubsec_q *q = NULL; 1036104478Ssam int err = 0, i, j, nicealign; 1037104478Ssam struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 1038104478Ssam int encoffset = 0, macoffset = 0, cpskip, cpoffset; 1039104478Ssam int sskip, dskip, stheend, dtheend; 1040104478Ssam int16_t coffset; 1041104478Ssam struct ubsec_session *ses; 1042104478Ssam struct ubsec_pktctx ctx; 1043104478Ssam struct ubsec_dma *dmap = NULL; 1044104478Ssam 1045104478Ssam if (crp == NULL || crp->crp_callback == NULL || sc == NULL) { 1046104478Ssam ubsecstats.hst_invalid++; 1047104478Ssam return (EINVAL); 1048104478Ssam } 1049104478Ssam if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) { 1050108471Ssam ubsecstats.hst_badsession++; 1051104478Ssam return (EINVAL); 1052104478Ssam } 1053104478Ssam 1054115747Ssam mtx_lock(&sc->sc_freeqlock); 1055104478Ssam if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 1056104478Ssam ubsecstats.hst_queuefull++; 1057104478Ssam sc->sc_needwakeup |= CRYPTO_SYMQ; 1058115747Ssam mtx_unlock(&sc->sc_freeqlock); 1059104478Ssam return (ERESTART); 1060104478Ssam } 1061104478Ssam q = SIMPLEQ_FIRST(&sc->sc_freequeue); 1062163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next); 1063115747Ssam mtx_unlock(&sc->sc_freeqlock); 1064104478Ssam 1065104478Ssam dmap = q->q_dma; /* Save dma pointer */ 1066104478Ssam bzero(q, sizeof(struct ubsec_q)); 1067104478Ssam bzero(&ctx, sizeof(ctx)); 1068104478Ssam 1069104478Ssam q->q_sesn = UBSEC_SESSION(crp->crp_sid); 1070104478Ssam q->q_dma = dmap; 1071104478Ssam ses = &sc->sc_sessions[q->q_sesn]; 1072104478Ssam 1073104478Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) { 1074104478Ssam q->q_src_m = (struct mbuf *)crp->crp_buf; 1075104478Ssam q->q_dst_m = (struct mbuf *)crp->crp_buf; 1076104478Ssam } else if (crp->crp_flags & CRYPTO_F_IOV) { 1077104478Ssam q->q_src_io = (struct uio *)crp->crp_buf; 1078104478Ssam q->q_dst_io = (struct uio *)crp->crp_buf; 1079104478Ssam } else { 1080108471Ssam ubsecstats.hst_badflags++; 1081104478Ssam err = EINVAL; 1082104478Ssam goto errout; /* XXX we don't handle contiguous blocks! */ 1083104478Ssam } 1084104478Ssam 1085104478Ssam bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr)); 1086104478Ssam 1087104478Ssam dmap->d_dma->d_mcr.mcr_pkts = htole16(1); 1088104478Ssam dmap->d_dma->d_mcr.mcr_flags = 0; 1089104478Ssam q->q_crp = crp; 1090104478Ssam 1091104478Ssam crd1 = crp->crp_desc; 1092104478Ssam if (crd1 == NULL) { 1093108471Ssam ubsecstats.hst_nodesc++; 1094104478Ssam err = EINVAL; 1095104478Ssam goto errout; 1096104478Ssam } 1097104478Ssam crd2 = crd1->crd_next; 1098104478Ssam 1099104478Ssam if (crd2 == NULL) { 1100104478Ssam if (crd1->crd_alg == CRYPTO_MD5_HMAC || 1101104478Ssam crd1->crd_alg == CRYPTO_SHA1_HMAC) { 1102104478Ssam maccrd = crd1; 1103104478Ssam enccrd = NULL; 1104104478Ssam } else if (crd1->crd_alg == CRYPTO_DES_CBC || 1105104478Ssam crd1->crd_alg == CRYPTO_3DES_CBC) { 1106104478Ssam maccrd = NULL; 1107104478Ssam enccrd = crd1; 1108104478Ssam } else { 1109108471Ssam ubsecstats.hst_badalg++; 1110104478Ssam err = EINVAL; 1111104478Ssam goto errout; 1112104478Ssam } 1113104478Ssam } else { 1114104478Ssam if ((crd1->crd_alg == CRYPTO_MD5_HMAC || 1115104478Ssam crd1->crd_alg == CRYPTO_SHA1_HMAC) && 1116104478Ssam (crd2->crd_alg == CRYPTO_DES_CBC || 1117104478Ssam crd2->crd_alg == CRYPTO_3DES_CBC) && 1118104478Ssam ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 1119104478Ssam maccrd = crd1; 1120104478Ssam enccrd = crd2; 1121104478Ssam } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 1122104478Ssam crd1->crd_alg == CRYPTO_3DES_CBC) && 1123104478Ssam (crd2->crd_alg == CRYPTO_MD5_HMAC || 1124104478Ssam crd2->crd_alg == CRYPTO_SHA1_HMAC) && 1125104478Ssam (crd1->crd_flags & CRD_F_ENCRYPT)) { 1126104478Ssam enccrd = crd1; 1127104478Ssam maccrd = crd2; 1128104478Ssam } else { 1129104478Ssam /* 1130104478Ssam * We cannot order the ubsec as requested 1131104478Ssam */ 1132108471Ssam ubsecstats.hst_badalg++; 1133104478Ssam err = EINVAL; 1134104478Ssam goto errout; 1135104478Ssam } 1136104478Ssam } 1137104478Ssam 1138104478Ssam if (enccrd) { 1139159225Spjd if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) { 1140159225Spjd ubsec_setup_enckey(ses, enccrd->crd_alg, 1141159225Spjd enccrd->crd_key); 1142159225Spjd } 1143159225Spjd 1144104478Ssam encoffset = enccrd->crd_skip; 1145104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES); 1146104478Ssam 1147104478Ssam if (enccrd->crd_flags & CRD_F_ENCRYPT) { 1148104478Ssam q->q_flags |= UBSEC_QFLAGS_COPYOUTIV; 1149104478Ssam 1150104478Ssam if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1151104478Ssam bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1152104478Ssam else { 1153104478Ssam ctx.pc_iv[0] = ses->ses_iv[0]; 1154104478Ssam ctx.pc_iv[1] = ses->ses_iv[1]; 1155104478Ssam } 1156104478Ssam 1157104478Ssam if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { 1158159242Spjd crypto_copyback(crp->crp_flags, crp->crp_buf, 1159159242Spjd enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv); 1160104478Ssam } 1161104478Ssam } else { 1162104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND); 1163104478Ssam 1164104478Ssam if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1165104478Ssam bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1166159242Spjd else { 1167159242Spjd crypto_copydata(crp->crp_flags, crp->crp_buf, 1168159242Spjd enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv); 1169159242Spjd } 1170104478Ssam } 1171104478Ssam 1172104478Ssam ctx.pc_deskey[0] = ses->ses_deskey[0]; 1173104478Ssam ctx.pc_deskey[1] = ses->ses_deskey[1]; 1174104478Ssam ctx.pc_deskey[2] = ses->ses_deskey[2]; 1175104478Ssam ctx.pc_deskey[3] = ses->ses_deskey[3]; 1176104478Ssam ctx.pc_deskey[4] = ses->ses_deskey[4]; 1177104478Ssam ctx.pc_deskey[5] = ses->ses_deskey[5]; 1178104478Ssam SWAP32(ctx.pc_iv[0]); 1179104478Ssam SWAP32(ctx.pc_iv[1]); 1180104478Ssam } 1181104478Ssam 1182104478Ssam if (maccrd) { 1183159225Spjd if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) { 1184159225Spjd ubsec_setup_mackey(ses, maccrd->crd_alg, 1185159225Spjd maccrd->crd_key, maccrd->crd_klen / 8); 1186159225Spjd } 1187159225Spjd 1188104478Ssam macoffset = maccrd->crd_skip; 1189104478Ssam 1190104478Ssam if (maccrd->crd_alg == CRYPTO_MD5_HMAC) 1191104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5); 1192104478Ssam else 1193104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1); 1194104478Ssam 1195104478Ssam for (i = 0; i < 5; i++) { 1196104478Ssam ctx.pc_hminner[i] = ses->ses_hminner[i]; 1197104478Ssam ctx.pc_hmouter[i] = ses->ses_hmouter[i]; 1198104478Ssam 1199104478Ssam HTOLE32(ctx.pc_hminner[i]); 1200104478Ssam HTOLE32(ctx.pc_hmouter[i]); 1201104478Ssam } 1202104478Ssam } 1203104478Ssam 1204104478Ssam if (enccrd && maccrd) { 1205104478Ssam /* 1206104478Ssam * ubsec cannot handle packets where the end of encryption 1207104478Ssam * and authentication are not the same, or where the 1208104478Ssam * encrypted part begins before the authenticated part. 1209104478Ssam */ 1210104478Ssam if ((encoffset + enccrd->crd_len) != 1211104478Ssam (macoffset + maccrd->crd_len)) { 1212104478Ssam ubsecstats.hst_lenmismatch++; 1213104478Ssam err = EINVAL; 1214104478Ssam goto errout; 1215104478Ssam } 1216104478Ssam if (enccrd->crd_skip < maccrd->crd_skip) { 1217104478Ssam ubsecstats.hst_skipmismatch++; 1218104478Ssam err = EINVAL; 1219104478Ssam goto errout; 1220104478Ssam } 1221104478Ssam sskip = maccrd->crd_skip; 1222104478Ssam cpskip = dskip = enccrd->crd_skip; 1223104478Ssam stheend = maccrd->crd_len; 1224104478Ssam dtheend = enccrd->crd_len; 1225104478Ssam coffset = enccrd->crd_skip - maccrd->crd_skip; 1226104478Ssam cpoffset = cpskip + dtheend; 1227104478Ssam#ifdef UBSEC_DEBUG 1228104478Ssam if (ubsec_debug) { 1229104478Ssam printf("mac: skip %d, len %d, inject %d\n", 1230104478Ssam maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject); 1231104478Ssam printf("enc: skip %d, len %d, inject %d\n", 1232104478Ssam enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject); 1233104478Ssam printf("src: skip %d, len %d\n", sskip, stheend); 1234104478Ssam printf("dst: skip %d, len %d\n", dskip, dtheend); 1235104478Ssam printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n", 1236104478Ssam coffset, stheend, cpskip, cpoffset); 1237104478Ssam } 1238104478Ssam#endif 1239104478Ssam } else { 1240104478Ssam cpskip = dskip = sskip = macoffset + encoffset; 1241104478Ssam dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len; 1242104478Ssam cpoffset = cpskip + dtheend; 1243104478Ssam coffset = 0; 1244104478Ssam } 1245104478Ssam ctx.pc_offset = htole16(coffset >> 2); 1246104478Ssam 1247104478Ssam if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) { 1248104478Ssam ubsecstats.hst_nomap++; 1249104478Ssam err = ENOMEM; 1250104478Ssam goto errout; 1251104478Ssam } 1252104478Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) { 1253104478Ssam if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map, 1254104478Ssam q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1255104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1256104478Ssam q->q_src_map = NULL; 1257104478Ssam ubsecstats.hst_noload++; 1258104478Ssam err = ENOMEM; 1259104478Ssam goto errout; 1260104478Ssam } 1261104478Ssam } else if (crp->crp_flags & CRYPTO_F_IOV) { 1262104478Ssam if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map, 1263104478Ssam q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1264104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1265104478Ssam q->q_src_map = NULL; 1266104478Ssam ubsecstats.hst_noload++; 1267104478Ssam err = ENOMEM; 1268104478Ssam goto errout; 1269104478Ssam } 1270104478Ssam } 1271104478Ssam nicealign = ubsec_dmamap_aligned(&q->q_src); 1272104478Ssam 1273104478Ssam dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend); 1274104478Ssam 1275104478Ssam#ifdef UBSEC_DEBUG 1276104478Ssam if (ubsec_debug) 1277104478Ssam printf("src skip: %d nicealign: %u\n", sskip, nicealign); 1278104478Ssam#endif 1279104478Ssam for (i = j = 0; i < q->q_src_nsegs; i++) { 1280104478Ssam struct ubsec_pktbuf *pb; 1281104478Ssam bus_size_t packl = q->q_src_segs[i].ds_len; 1282104478Ssam bus_addr_t packp = q->q_src_segs[i].ds_addr; 1283104478Ssam 1284104478Ssam if (sskip >= packl) { 1285104478Ssam sskip -= packl; 1286104478Ssam continue; 1287104478Ssam } 1288104478Ssam 1289104478Ssam packl -= sskip; 1290104478Ssam packp += sskip; 1291104478Ssam sskip = 0; 1292104478Ssam 1293104478Ssam if (packl > 0xfffc) { 1294104478Ssam err = EIO; 1295104478Ssam goto errout; 1296104478Ssam } 1297104478Ssam 1298104478Ssam if (j == 0) 1299104478Ssam pb = &dmap->d_dma->d_mcr.mcr_ipktbuf; 1300104478Ssam else 1301104478Ssam pb = &dmap->d_dma->d_sbuf[j - 1]; 1302104478Ssam 1303104478Ssam pb->pb_addr = htole32(packp); 1304104478Ssam 1305104478Ssam if (stheend) { 1306104478Ssam if (packl > stheend) { 1307104478Ssam pb->pb_len = htole32(stheend); 1308104478Ssam stheend = 0; 1309104478Ssam } else { 1310104478Ssam pb->pb_len = htole32(packl); 1311104478Ssam stheend -= packl; 1312104478Ssam } 1313104478Ssam } else 1314104478Ssam pb->pb_len = htole32(packl); 1315104478Ssam 1316104478Ssam if ((i + 1) == q->q_src_nsegs) 1317104478Ssam pb->pb_next = 0; 1318104478Ssam else 1319104478Ssam pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1320104478Ssam offsetof(struct ubsec_dmachunk, d_sbuf[j])); 1321104478Ssam j++; 1322104478Ssam } 1323104478Ssam 1324104478Ssam if (enccrd == NULL && maccrd != NULL) { 1325104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0; 1326104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0; 1327104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr + 1328104478Ssam offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1329104478Ssam#ifdef UBSEC_DEBUG 1330104478Ssam if (ubsec_debug) 1331104478Ssam printf("opkt: %x %x %x\n", 1332104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr, 1333104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_len, 1334104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_next); 1335104478Ssam#endif 1336104478Ssam } else { 1337104478Ssam if (crp->crp_flags & CRYPTO_F_IOV) { 1338104478Ssam if (!nicealign) { 1339104478Ssam ubsecstats.hst_iovmisaligned++; 1340104478Ssam err = EINVAL; 1341104478Ssam goto errout; 1342104478Ssam } 1343104478Ssam if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, 1344104478Ssam &q->q_dst_map)) { 1345104478Ssam ubsecstats.hst_nomap++; 1346104478Ssam err = ENOMEM; 1347104478Ssam goto errout; 1348104478Ssam } 1349104478Ssam if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map, 1350104478Ssam q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) { 1351104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1352104478Ssam q->q_dst_map = NULL; 1353104478Ssam ubsecstats.hst_noload++; 1354104478Ssam err = ENOMEM; 1355104478Ssam goto errout; 1356104478Ssam } 1357104478Ssam } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1358104478Ssam if (nicealign) { 1359104478Ssam q->q_dst = q->q_src; 1360104478Ssam } else { 1361104478Ssam int totlen, len; 1362104478Ssam struct mbuf *m, *top, **mp; 1363104478Ssam 1364104478Ssam ubsecstats.hst_unaligned++; 1365104478Ssam totlen = q->q_src_mapsize; 1366160931Sjhb if (totlen >= MINCLSIZE) { 1367243857Sglebius m = m_getcl(M_NOWAIT, MT_DATA, 1368160931Sjhb q->q_src_m->m_flags & M_PKTHDR); 1369160931Sjhb len = MCLBYTES; 1370160931Sjhb } else if (q->q_src_m->m_flags & M_PKTHDR) { 1371243857Sglebius m = m_gethdr(M_NOWAIT, MT_DATA); 1372104478Ssam len = MHLEN; 1373104478Ssam } else { 1374243857Sglebius m = m_get(M_NOWAIT, MT_DATA); 1375104478Ssam len = MLEN; 1376104478Ssam } 1377160931Sjhb if (m && q->q_src_m->m_flags & M_PKTHDR && 1378243857Sglebius !m_dup_pkthdr(m, q->q_src_m, M_NOWAIT)) { 1379160931Sjhb m_free(m); 1380160931Sjhb m = NULL; 1381160931Sjhb } 1382104478Ssam if (m == NULL) { 1383104478Ssam ubsecstats.hst_nombuf++; 1384104478Ssam err = sc->sc_nqueue ? ERESTART : ENOMEM; 1385104478Ssam goto errout; 1386104478Ssam } 1387160931Sjhb m->m_len = len = min(totlen, len); 1388160931Sjhb totlen -= len; 1389160931Sjhb top = m; 1390104478Ssam mp = ⊤ 1391104478Ssam 1392104478Ssam while (totlen > 0) { 1393160931Sjhb if (totlen >= MINCLSIZE) { 1394243857Sglebius m = m_getcl(M_NOWAIT, 1395160931Sjhb MT_DATA, 0); 1396160931Sjhb len = MCLBYTES; 1397160931Sjhb } else { 1398243857Sglebius m = m_get(M_NOWAIT, MT_DATA); 1399104478Ssam len = MLEN; 1400104478Ssam } 1401160931Sjhb if (m == NULL) { 1402160931Sjhb m_freem(top); 1403160931Sjhb ubsecstats.hst_nombuf++; 1404160931Sjhb err = sc->sc_nqueue ? ERESTART : ENOMEM; 1405160931Sjhb goto errout; 1406104478Ssam } 1407104478Ssam m->m_len = len = min(totlen, len); 1408104478Ssam totlen -= len; 1409104478Ssam *mp = m; 1410104478Ssam mp = &m->m_next; 1411104478Ssam } 1412104478Ssam q->q_dst_m = top; 1413104478Ssam ubsec_mcopy(q->q_src_m, q->q_dst_m, 1414104478Ssam cpskip, cpoffset); 1415162969Sjhb if (bus_dmamap_create(sc->sc_dmat, 1416104478Ssam BUS_DMA_NOWAIT, &q->q_dst_map) != 0) { 1417104478Ssam ubsecstats.hst_nomap++; 1418104478Ssam err = ENOMEM; 1419104478Ssam goto errout; 1420104478Ssam } 1421104478Ssam if (bus_dmamap_load_mbuf(sc->sc_dmat, 1422104478Ssam q->q_dst_map, q->q_dst_m, 1423104478Ssam ubsec_op_cb, &q->q_dst, 1424104478Ssam BUS_DMA_NOWAIT) != 0) { 1425104478Ssam bus_dmamap_destroy(sc->sc_dmat, 1426104478Ssam q->q_dst_map); 1427104478Ssam q->q_dst_map = NULL; 1428104478Ssam ubsecstats.hst_noload++; 1429104478Ssam err = ENOMEM; 1430104478Ssam goto errout; 1431104478Ssam } 1432104478Ssam } 1433104478Ssam } else { 1434108471Ssam ubsecstats.hst_badflags++; 1435104478Ssam err = EINVAL; 1436104478Ssam goto errout; 1437104478Ssam } 1438104478Ssam 1439104478Ssam#ifdef UBSEC_DEBUG 1440104478Ssam if (ubsec_debug) 1441104478Ssam printf("dst skip: %d\n", dskip); 1442104478Ssam#endif 1443104478Ssam for (i = j = 0; i < q->q_dst_nsegs; i++) { 1444104478Ssam struct ubsec_pktbuf *pb; 1445104478Ssam bus_size_t packl = q->q_dst_segs[i].ds_len; 1446104478Ssam bus_addr_t packp = q->q_dst_segs[i].ds_addr; 1447104478Ssam 1448104478Ssam if (dskip >= packl) { 1449104478Ssam dskip -= packl; 1450104478Ssam continue; 1451104478Ssam } 1452104478Ssam 1453104478Ssam packl -= dskip; 1454104478Ssam packp += dskip; 1455104478Ssam dskip = 0; 1456104478Ssam 1457104478Ssam if (packl > 0xfffc) { 1458104478Ssam err = EIO; 1459104478Ssam goto errout; 1460104478Ssam } 1461104478Ssam 1462104478Ssam if (j == 0) 1463104478Ssam pb = &dmap->d_dma->d_mcr.mcr_opktbuf; 1464104478Ssam else 1465104478Ssam pb = &dmap->d_dma->d_dbuf[j - 1]; 1466104478Ssam 1467104478Ssam pb->pb_addr = htole32(packp); 1468104478Ssam 1469104478Ssam if (dtheend) { 1470104478Ssam if (packl > dtheend) { 1471104478Ssam pb->pb_len = htole32(dtheend); 1472104478Ssam dtheend = 0; 1473104478Ssam } else { 1474104478Ssam pb->pb_len = htole32(packl); 1475104478Ssam dtheend -= packl; 1476104478Ssam } 1477104478Ssam } else 1478104478Ssam pb->pb_len = htole32(packl); 1479104478Ssam 1480104478Ssam if ((i + 1) == q->q_dst_nsegs) { 1481104478Ssam if (maccrd) 1482104478Ssam pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1483104478Ssam offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1484104478Ssam else 1485104478Ssam pb->pb_next = 0; 1486104478Ssam } else 1487104478Ssam pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1488104478Ssam offsetof(struct ubsec_dmachunk, d_dbuf[j])); 1489104478Ssam j++; 1490104478Ssam } 1491104478Ssam } 1492104478Ssam 1493104478Ssam dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr + 1494104478Ssam offsetof(struct ubsec_dmachunk, d_ctx)); 1495104478Ssam 1496104478Ssam if (sc->sc_flags & UBS_FLAGS_LONGCTX) { 1497104478Ssam struct ubsec_pktctx_long *ctxl; 1498104478Ssam 1499104478Ssam ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr + 1500104478Ssam offsetof(struct ubsec_dmachunk, d_ctx)); 1501162969Sjhb 1502104478Ssam /* transform small context into long context */ 1503104478Ssam ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long)); 1504104478Ssam ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC); 1505104478Ssam ctxl->pc_flags = ctx.pc_flags; 1506104478Ssam ctxl->pc_offset = ctx.pc_offset; 1507104478Ssam for (i = 0; i < 6; i++) 1508104478Ssam ctxl->pc_deskey[i] = ctx.pc_deskey[i]; 1509104478Ssam for (i = 0; i < 5; i++) 1510104478Ssam ctxl->pc_hminner[i] = ctx.pc_hminner[i]; 1511104478Ssam for (i = 0; i < 5; i++) 1512162969Sjhb ctxl->pc_hmouter[i] = ctx.pc_hmouter[i]; 1513104478Ssam ctxl->pc_iv[0] = ctx.pc_iv[0]; 1514104478Ssam ctxl->pc_iv[1] = ctx.pc_iv[1]; 1515104478Ssam } else 1516104478Ssam bcopy(&ctx, dmap->d_alloc.dma_vaddr + 1517104478Ssam offsetof(struct ubsec_dmachunk, d_ctx), 1518104478Ssam sizeof(struct ubsec_pktctx)); 1519104478Ssam 1520115747Ssam mtx_lock(&sc->sc_mcr1lock); 1521104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next); 1522104478Ssam sc->sc_nqueue++; 1523104478Ssam ubsecstats.hst_ipackets++; 1524104478Ssam ubsecstats.hst_ibytes += dmap->d_alloc.dma_size; 1525111416Ssam if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= UBS_MAX_AGGR) 1526104478Ssam ubsec_feed(sc); 1527115747Ssam mtx_unlock(&sc->sc_mcr1lock); 1528104478Ssam return (0); 1529104478Ssam 1530104478Ssamerrout: 1531104478Ssam if (q != NULL) { 1532104478Ssam if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 1533104478Ssam m_freem(q->q_dst_m); 1534104478Ssam 1535104478Ssam if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1536104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1537104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1538104478Ssam } 1539104478Ssam if (q->q_src_map != NULL) { 1540104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1541104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1542104478Ssam } 1543158828Spjd } 1544158828Spjd if (q != NULL || err == ERESTART) { 1545115747Ssam mtx_lock(&sc->sc_freeqlock); 1546158828Spjd if (q != NULL) 1547158828Spjd SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1548158828Spjd if (err == ERESTART) 1549158828Spjd sc->sc_needwakeup |= CRYPTO_SYMQ; 1550115747Ssam mtx_unlock(&sc->sc_freeqlock); 1551104478Ssam } 1552104478Ssam if (err != ERESTART) { 1553104478Ssam crp->crp_etype = err; 1554104478Ssam crypto_done(crp); 1555104478Ssam } 1556104478Ssam return (err); 1557104478Ssam} 1558104478Ssam 1559104478Ssamstatic void 1560104478Ssamubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q) 1561104478Ssam{ 1562104478Ssam struct cryptop *crp = (struct cryptop *)q->q_crp; 1563104478Ssam struct cryptodesc *crd; 1564104478Ssam struct ubsec_dma *dmap = q->q_dma; 1565104478Ssam 1566112099Ssam ubsecstats.hst_opackets++; 1567112099Ssam ubsecstats.hst_obytes += dmap->d_alloc.dma_size; 1568112099Ssam 1569108823Ssam ubsec_dma_sync(&dmap->d_alloc, 1570104478Ssam BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1571104478Ssam if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1572104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 1573104478Ssam BUS_DMASYNC_POSTREAD); 1574104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1575104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1576104478Ssam } 1577104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE); 1578104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1579104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1580104478Ssam 1581104478Ssam if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) { 1582104478Ssam m_freem(q->q_src_m); 1583104478Ssam crp->crp_buf = (caddr_t)q->q_dst_m; 1584104478Ssam } 1585104478Ssam 1586104478Ssam /* copy out IV for future use */ 1587104478Ssam if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) { 1588104478Ssam for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1589104478Ssam if (crd->crd_alg != CRYPTO_DES_CBC && 1590104478Ssam crd->crd_alg != CRYPTO_3DES_CBC) 1591104478Ssam continue; 1592159242Spjd crypto_copydata(crp->crp_flags, crp->crp_buf, 1593159242Spjd crd->crd_skip + crd->crd_len - 8, 8, 1594159242Spjd (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv); 1595104478Ssam break; 1596104478Ssam } 1597104478Ssam } 1598104478Ssam 1599104478Ssam for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1600104478Ssam if (crd->crd_alg != CRYPTO_MD5_HMAC && 1601104478Ssam crd->crd_alg != CRYPTO_SHA1_HMAC) 1602104478Ssam continue; 1603159242Spjd crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, 1604159242Spjd sc->sc_sessions[q->q_sesn].ses_mlen, 1605159242Spjd (caddr_t)dmap->d_dma->d_macbuf); 1606104478Ssam break; 1607104478Ssam } 1608115747Ssam mtx_lock(&sc->sc_freeqlock); 1609104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1610115747Ssam mtx_unlock(&sc->sc_freeqlock); 1611104478Ssam crypto_done(crp); 1612104478Ssam} 1613104478Ssam 1614104478Ssamstatic void 1615104478Ssamubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset) 1616104478Ssam{ 1617104478Ssam int i, j, dlen, slen; 1618104478Ssam caddr_t dptr, sptr; 1619104478Ssam 1620104478Ssam j = 0; 1621104478Ssam sptr = srcm->m_data; 1622104478Ssam slen = srcm->m_len; 1623104478Ssam dptr = dstm->m_data; 1624104478Ssam dlen = dstm->m_len; 1625104478Ssam 1626104478Ssam while (1) { 1627104478Ssam for (i = 0; i < min(slen, dlen); i++) { 1628104478Ssam if (j < hoffset || j >= toffset) 1629104478Ssam *dptr++ = *sptr++; 1630104478Ssam slen--; 1631104478Ssam dlen--; 1632104478Ssam j++; 1633104478Ssam } 1634104478Ssam if (slen == 0) { 1635104478Ssam srcm = srcm->m_next; 1636104478Ssam if (srcm == NULL) 1637104478Ssam return; 1638104478Ssam sptr = srcm->m_data; 1639104478Ssam slen = srcm->m_len; 1640104478Ssam } 1641104478Ssam if (dlen == 0) { 1642104478Ssam dstm = dstm->m_next; 1643104478Ssam if (dstm == NULL) 1644104478Ssam return; 1645104478Ssam dptr = dstm->m_data; 1646104478Ssam dlen = dstm->m_len; 1647104478Ssam } 1648104478Ssam } 1649104478Ssam} 1650104478Ssam 1651104478Ssam/* 1652104478Ssam * feed the key generator, must be called at splimp() or higher. 1653104478Ssam */ 1654104478Ssamstatic int 1655104478Ssamubsec_feed2(struct ubsec_softc *sc) 1656104478Ssam{ 1657104478Ssam struct ubsec_q2 *q; 1658104478Ssam 1659104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) { 1660104478Ssam if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL) 1661104478Ssam break; 1662104478Ssam q = SIMPLEQ_FIRST(&sc->sc_queue2); 1663104478Ssam 1664108823Ssam ubsec_dma_sync(&q->q_mcr, 1665104478Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1666108823Ssam ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_PREWRITE); 1667104478Ssam 1668104478Ssam WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr); 1669163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q_next); 1670104478Ssam --sc->sc_nqueue2; 1671104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next); 1672104478Ssam } 1673104478Ssam return (0); 1674104478Ssam} 1675104478Ssam 1676104478Ssam/* 1677104478Ssam * Callback for handling random numbers 1678104478Ssam */ 1679104478Ssamstatic void 1680104478Ssamubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q) 1681104478Ssam{ 1682104478Ssam struct cryptkop *krp; 1683104478Ssam struct ubsec_ctx_keyop *ctx; 1684104478Ssam 1685104478Ssam ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr; 1686108823Ssam ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_POSTWRITE); 1687104478Ssam 1688104478Ssam switch (q->q_type) { 1689104478Ssam#ifndef UBSEC_NO_RNG 1690104478Ssam case UBS_CTXOP_RNGBYPASS: { 1691104478Ssam struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q; 1692104478Ssam 1693108823Ssam ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_POSTREAD); 1694112124Ssam (*sc->sc_harvest)(sc->sc_rndtest, 1695112124Ssam rng->rng_buf.dma_vaddr, 1696112124Ssam UBSEC_RNG_BUFSIZ*sizeof (u_int32_t)); 1697104478Ssam rng->rng_used = 0; 1698104478Ssam callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1699104478Ssam break; 1700104478Ssam } 1701104478Ssam#endif 1702104478Ssam case UBS_CTXOP_MODEXP: { 1703104478Ssam struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 1704104478Ssam u_int rlen, clen; 1705104478Ssam 1706104478Ssam krp = me->me_krp; 1707104478Ssam rlen = (me->me_modbits + 7) / 8; 1708104630Ssam clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8; 1709104478Ssam 1710108823Ssam ubsec_dma_sync(&me->me_M, BUS_DMASYNC_POSTWRITE); 1711108823Ssam ubsec_dma_sync(&me->me_E, BUS_DMASYNC_POSTWRITE); 1712108823Ssam ubsec_dma_sync(&me->me_C, BUS_DMASYNC_POSTREAD); 1713108823Ssam ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_POSTWRITE); 1714104478Ssam 1715104478Ssam if (clen < rlen) 1716104478Ssam krp->krp_status = E2BIG; 1717104630Ssam else { 1718104630Ssam if (sc->sc_flags & UBS_FLAGS_HWNORM) { 1719104630Ssam bzero(krp->krp_param[krp->krp_iparams].crp_p, 1720104630Ssam (krp->krp_param[krp->krp_iparams].crp_nbits 1721104630Ssam + 7) / 8); 1722104630Ssam bcopy(me->me_C.dma_vaddr, 1723104630Ssam krp->krp_param[krp->krp_iparams].crp_p, 1724104630Ssam (me->me_modbits + 7) / 8); 1725104630Ssam } else 1726104630Ssam ubsec_kshift_l(me->me_shiftbits, 1727104630Ssam me->me_C.dma_vaddr, me->me_normbits, 1728104630Ssam krp->krp_param[krp->krp_iparams].crp_p, 1729104630Ssam krp->krp_param[krp->krp_iparams].crp_nbits); 1730104630Ssam } 1731104478Ssam 1732104478Ssam crypto_kdone(krp); 1733104478Ssam 1734104478Ssam /* bzero all potentially sensitive data */ 1735104478Ssam bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 1736104478Ssam bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 1737104478Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 1738104478Ssam bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 1739104478Ssam 1740104478Ssam /* Can't free here, so put us on the free list. */ 1741104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next); 1742104478Ssam break; 1743104478Ssam } 1744104478Ssam case UBS_CTXOP_RSAPRIV: { 1745104478Ssam struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 1746104478Ssam u_int len; 1747104478Ssam 1748104478Ssam krp = rp->rpr_krp; 1749108823Ssam ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_POSTWRITE); 1750108823Ssam ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_POSTREAD); 1751104478Ssam 1752104478Ssam len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8; 1753104478Ssam bcopy(rp->rpr_msgout.dma_vaddr, 1754104478Ssam krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len); 1755104478Ssam 1756104478Ssam crypto_kdone(krp); 1757104478Ssam 1758104478Ssam bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 1759104478Ssam bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 1760104478Ssam bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size); 1761104478Ssam 1762104478Ssam /* Can't free here, so put us on the free list. */ 1763104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next); 1764104478Ssam break; 1765104478Ssam } 1766104478Ssam default: 1767104478Ssam device_printf(sc->sc_dev, "unknown ctx op: %x\n", 1768104478Ssam letoh16(ctx->ctx_op)); 1769104478Ssam break; 1770104478Ssam } 1771104478Ssam} 1772104478Ssam 1773104478Ssam#ifndef UBSEC_NO_RNG 1774104478Ssamstatic void 1775104478Ssamubsec_rng(void *vsc) 1776104478Ssam{ 1777104478Ssam struct ubsec_softc *sc = vsc; 1778104478Ssam struct ubsec_q2_rng *rng = &sc->sc_rng; 1779104478Ssam struct ubsec_mcr *mcr; 1780104478Ssam struct ubsec_ctx_rngbypass *ctx; 1781104478Ssam 1782115747Ssam mtx_lock(&sc->sc_mcr2lock); 1783104478Ssam if (rng->rng_used) { 1784115747Ssam mtx_unlock(&sc->sc_mcr2lock); 1785104478Ssam return; 1786104478Ssam } 1787104478Ssam sc->sc_nqueue2++; 1788104478Ssam if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE) 1789104478Ssam goto out; 1790104478Ssam 1791104478Ssam mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr; 1792104478Ssam ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr; 1793104478Ssam 1794104478Ssam mcr->mcr_pkts = htole16(1); 1795104478Ssam mcr->mcr_flags = 0; 1796104478Ssam mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr); 1797104478Ssam mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0; 1798104478Ssam mcr->mcr_ipktbuf.pb_len = 0; 1799104478Ssam mcr->mcr_reserved = mcr->mcr_pktlen = 0; 1800104478Ssam mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr); 1801104478Ssam mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) & 1802104478Ssam UBS_PKTBUF_LEN); 1803104478Ssam mcr->mcr_opktbuf.pb_next = 0; 1804104478Ssam 1805104478Ssam ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass)); 1806104478Ssam ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS); 1807104478Ssam rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS; 1808104478Ssam 1809108823Ssam ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_PREREAD); 1810104478Ssam 1811104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next); 1812104478Ssam rng->rng_used = 1; 1813104478Ssam ubsec_feed2(sc); 1814104478Ssam ubsecstats.hst_rng++; 1815115747Ssam mtx_unlock(&sc->sc_mcr2lock); 1816104478Ssam 1817104478Ssam return; 1818104478Ssam 1819104478Ssamout: 1820104478Ssam /* 1821104478Ssam * Something weird happened, generate our own call back. 1822104478Ssam */ 1823104478Ssam sc->sc_nqueue2--; 1824115747Ssam mtx_unlock(&sc->sc_mcr2lock); 1825104478Ssam callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1826104478Ssam} 1827104478Ssam#endif /* UBSEC_NO_RNG */ 1828104478Ssam 1829104478Ssamstatic void 1830104478Ssamubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 1831104478Ssam{ 1832104478Ssam bus_addr_t *paddr = (bus_addr_t*) arg; 1833104478Ssam *paddr = segs->ds_addr; 1834104478Ssam} 1835104478Ssam 1836104478Ssamstatic int 1837104478Ssamubsec_dma_malloc( 1838104478Ssam struct ubsec_softc *sc, 1839104478Ssam bus_size_t size, 1840104478Ssam struct ubsec_dma_alloc *dma, 1841104478Ssam int mapflags 1842104478Ssam) 1843104478Ssam{ 1844104478Ssam int r; 1845104478Ssam 1846108823Ssam /* XXX could specify sc_dmat as parent but that just adds overhead */ 1847232874Sscottl r = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ 1848108823Ssam 1, 0, /* alignment, bounds */ 1849108823Ssam BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 1850108823Ssam BUS_SPACE_MAXADDR, /* highaddr */ 1851108823Ssam NULL, NULL, /* filter, filterarg */ 1852108823Ssam size, /* maxsize */ 1853108823Ssam 1, /* nsegments */ 1854108823Ssam size, /* maxsegsize */ 1855108823Ssam BUS_DMA_ALLOCNOW, /* flags */ 1856117126Sscottl NULL, NULL, /* lockfunc, lockarg */ 1857108823Ssam &dma->dma_tag); 1858108823Ssam if (r != 0) { 1859108823Ssam device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1860108823Ssam "bus_dma_tag_create failed; error %u\n", r); 1861104478Ssam goto fail_1; 1862108823Ssam } 1863104478Ssam 1864108823Ssam r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 1865108823Ssam BUS_DMA_NOWAIT, &dma->dma_map); 1866108823Ssam if (r != 0) { 1867108823Ssam device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1868205844Simp "bus_dmammem_alloc failed; size %ju, error %u\n", 1869205844Simp (intmax_t)size, r); 1870108823Ssam goto fail_2; 1871108823Ssam } 1872108823Ssam 1873108823Ssam r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 1874104478Ssam size, 1875104478Ssam ubsec_dmamap_cb, 1876104478Ssam &dma->dma_paddr, 1877104478Ssam mapflags | BUS_DMA_NOWAIT); 1878108823Ssam if (r != 0) { 1879108823Ssam device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1880108823Ssam "bus_dmamap_load failed; error %u\n", r); 1881108823Ssam goto fail_3; 1882108823Ssam } 1883104478Ssam 1884104478Ssam dma->dma_size = size; 1885104478Ssam return (0); 1886104478Ssam 1887108823Ssamfail_3: 1888108823Ssam bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1889104478Ssamfail_2: 1890108823Ssam bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1891104478Ssamfail_1: 1892108823Ssam bus_dma_tag_destroy(dma->dma_tag); 1893108823Ssam dma->dma_tag = NULL; 1894104478Ssam return (r); 1895104478Ssam} 1896104478Ssam 1897104478Ssamstatic void 1898104478Ssamubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma) 1899104478Ssam{ 1900108823Ssam bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1901108823Ssam bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1902108823Ssam bus_dma_tag_destroy(dma->dma_tag); 1903104478Ssam} 1904104478Ssam 1905104478Ssam/* 1906104478Ssam * Resets the board. Values in the regesters are left as is 1907104478Ssam * from the reset (i.e. initial values are assigned elsewhere). 1908104478Ssam */ 1909104478Ssamstatic void 1910104478Ssamubsec_reset_board(struct ubsec_softc *sc) 1911104478Ssam{ 1912104478Ssam volatile u_int32_t ctrl; 1913104478Ssam 1914104478Ssam ctrl = READ_REG(sc, BS_CTRL); 1915104478Ssam ctrl |= BS_CTRL_RESET; 1916104478Ssam WRITE_REG(sc, BS_CTRL, ctrl); 1917104478Ssam 1918104478Ssam /* 1919104478Ssam * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us 1920104478Ssam */ 1921104478Ssam DELAY(10); 1922104478Ssam} 1923104478Ssam 1924104478Ssam/* 1925104478Ssam * Init Broadcom registers 1926104478Ssam */ 1927104478Ssamstatic void 1928104478Ssamubsec_init_board(struct ubsec_softc *sc) 1929104478Ssam{ 1930104630Ssam u_int32_t ctrl; 1931104630Ssam 1932104630Ssam ctrl = READ_REG(sc, BS_CTRL); 1933104630Ssam ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64); 1934104630Ssam ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT; 1935104630Ssam 1936104630Ssam if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) 1937104630Ssam ctrl |= BS_CTRL_MCR2INT; 1938104630Ssam else 1939104630Ssam ctrl &= ~BS_CTRL_MCR2INT; 1940104630Ssam 1941104630Ssam if (sc->sc_flags & UBS_FLAGS_HWNORM) 1942104630Ssam ctrl &= ~BS_CTRL_SWNORM; 1943104630Ssam 1944104630Ssam WRITE_REG(sc, BS_CTRL, ctrl); 1945104478Ssam} 1946104478Ssam 1947104478Ssam/* 1948104478Ssam * Init Broadcom PCI registers 1949104478Ssam */ 1950104478Ssamstatic void 1951104478Ssamubsec_init_pciregs(device_t dev) 1952104478Ssam{ 1953104478Ssam#if 0 1954104478Ssam u_int32_t misc; 1955104478Ssam 1956104478Ssam misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT); 1957104478Ssam misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT)) 1958104478Ssam | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT); 1959104478Ssam misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT)) 1960104478Ssam | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT); 1961104478Ssam pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc); 1962104478Ssam#endif 1963104478Ssam 1964104478Ssam /* 1965104478Ssam * This will set the cache line size to 1, this will 1966104478Ssam * force the BCM58xx chip just to do burst read/writes. 1967104478Ssam * Cache line read/writes are to slow 1968104478Ssam */ 1969104478Ssam pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1); 1970104478Ssam} 1971104478Ssam 1972104478Ssam/* 1973104478Ssam * Clean up after a chip crash. 1974104478Ssam * It is assumed that the caller in splimp() 1975104478Ssam */ 1976104478Ssamstatic void 1977104478Ssamubsec_cleanchip(struct ubsec_softc *sc) 1978104478Ssam{ 1979104478Ssam struct ubsec_q *q; 1980104478Ssam 1981104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 1982104478Ssam q = SIMPLEQ_FIRST(&sc->sc_qchip); 1983163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next); 1984104478Ssam ubsec_free_q(sc, q); 1985104478Ssam } 1986108471Ssam sc->sc_nqchip = 0; 1987104478Ssam} 1988104478Ssam 1989104478Ssam/* 1990108823Ssam * free a ubsec_q 1991108823Ssam * It is assumed that the caller is within splimp(). 1992104478Ssam */ 1993104478Ssamstatic int 1994104478Ssamubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q) 1995104478Ssam{ 1996104478Ssam struct ubsec_q *q2; 1997104478Ssam struct cryptop *crp; 1998104478Ssam int npkts; 1999104478Ssam int i; 2000104478Ssam 2001104478Ssam npkts = q->q_nstacked_mcrs; 2002104478Ssam 2003104478Ssam for (i = 0; i < npkts; i++) { 2004104478Ssam if(q->q_stacked_mcr[i]) { 2005104478Ssam q2 = q->q_stacked_mcr[i]; 2006104478Ssam 2007162969Sjhb if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m)) 2008104478Ssam m_freem(q2->q_dst_m); 2009104478Ssam 2010104478Ssam crp = (struct cryptop *)q2->q_crp; 2011162969Sjhb 2012104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next); 2013162969Sjhb 2014104478Ssam crp->crp_etype = EFAULT; 2015104478Ssam crypto_done(crp); 2016104478Ssam } else { 2017104478Ssam break; 2018104478Ssam } 2019104478Ssam } 2020104478Ssam 2021104478Ssam /* 2022104478Ssam * Free header MCR 2023104478Ssam */ 2024104478Ssam if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 2025104478Ssam m_freem(q->q_dst_m); 2026104478Ssam 2027104478Ssam crp = (struct cryptop *)q->q_crp; 2028162969Sjhb 2029104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 2030162969Sjhb 2031104478Ssam crp->crp_etype = EFAULT; 2032104478Ssam crypto_done(crp); 2033104478Ssam return(0); 2034104478Ssam} 2035104478Ssam 2036104478Ssam/* 2037104478Ssam * Routine to reset the chip and clean up. 2038104478Ssam * It is assumed that the caller is in splimp() 2039104478Ssam */ 2040104478Ssamstatic void 2041104478Ssamubsec_totalreset(struct ubsec_softc *sc) 2042104478Ssam{ 2043104478Ssam ubsec_reset_board(sc); 2044104478Ssam ubsec_init_board(sc); 2045104478Ssam ubsec_cleanchip(sc); 2046104478Ssam} 2047104478Ssam 2048104478Ssamstatic int 2049104478Ssamubsec_dmamap_aligned(struct ubsec_operand *op) 2050104478Ssam{ 2051104478Ssam int i; 2052104478Ssam 2053104478Ssam for (i = 0; i < op->nsegs; i++) { 2054104478Ssam if (op->segs[i].ds_addr & 3) 2055104478Ssam return (0); 2056104478Ssam if ((i != (op->nsegs - 1)) && 2057104478Ssam (op->segs[i].ds_len & 3)) 2058104478Ssam return (0); 2059104478Ssam } 2060104478Ssam return (1); 2061104478Ssam} 2062104478Ssam 2063104478Ssamstatic void 2064104478Ssamubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q) 2065104478Ssam{ 2066104478Ssam switch (q->q_type) { 2067104478Ssam case UBS_CTXOP_MODEXP: { 2068104478Ssam struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 2069104478Ssam 2070104478Ssam ubsec_dma_free(sc, &me->me_q.q_mcr); 2071104478Ssam ubsec_dma_free(sc, &me->me_q.q_ctx); 2072104478Ssam ubsec_dma_free(sc, &me->me_M); 2073104478Ssam ubsec_dma_free(sc, &me->me_E); 2074104478Ssam ubsec_dma_free(sc, &me->me_C); 2075104478Ssam ubsec_dma_free(sc, &me->me_epb); 2076104478Ssam free(me, M_DEVBUF); 2077104478Ssam break; 2078104478Ssam } 2079104478Ssam case UBS_CTXOP_RSAPRIV: { 2080104478Ssam struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 2081104478Ssam 2082104478Ssam ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2083104478Ssam ubsec_dma_free(sc, &rp->rpr_q.q_ctx); 2084104478Ssam ubsec_dma_free(sc, &rp->rpr_msgin); 2085104478Ssam ubsec_dma_free(sc, &rp->rpr_msgout); 2086104478Ssam free(rp, M_DEVBUF); 2087104478Ssam break; 2088104478Ssam } 2089104478Ssam default: 2090104478Ssam device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type); 2091104478Ssam break; 2092104478Ssam } 2093104478Ssam} 2094104478Ssam 2095104478Ssamstatic int 2096167755Ssamubsec_kprocess(device_t dev, struct cryptkop *krp, int hint) 2097104478Ssam{ 2098167755Ssam struct ubsec_softc *sc = device_get_softc(dev); 2099104630Ssam int r; 2100104478Ssam 2101104478Ssam if (krp == NULL || krp->krp_callback == NULL) 2102104478Ssam return (EINVAL); 2103104478Ssam 2104104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) { 2105104478Ssam struct ubsec_q2 *q; 2106104478Ssam 2107104478Ssam q = SIMPLEQ_FIRST(&sc->sc_q2free); 2108163648Sru SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q_next); 2109104478Ssam ubsec_kfree(sc, q); 2110104478Ssam } 2111104478Ssam 2112104478Ssam switch (krp->krp_op) { 2113104478Ssam case CRK_MOD_EXP: 2114104630Ssam if (sc->sc_flags & UBS_FLAGS_HWNORM) 2115104630Ssam r = ubsec_kprocess_modexp_hw(sc, krp, hint); 2116104630Ssam else 2117104630Ssam r = ubsec_kprocess_modexp_sw(sc, krp, hint); 2118104630Ssam break; 2119104478Ssam case CRK_MOD_EXP_CRT: 2120104478Ssam return (ubsec_kprocess_rsapriv(sc, krp, hint)); 2121104478Ssam default: 2122104478Ssam device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n", 2123104478Ssam krp->krp_op); 2124104478Ssam krp->krp_status = EOPNOTSUPP; 2125104478Ssam crypto_kdone(krp); 2126104478Ssam return (0); 2127104478Ssam } 2128104630Ssam return (0); /* silence compiler */ 2129104478Ssam} 2130104478Ssam 2131104478Ssam/* 2132104630Ssam * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization) 2133104478Ssam */ 2134104478Ssamstatic int 2135104630Ssamubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2136104478Ssam{ 2137104478Ssam struct ubsec_q2_modexp *me; 2138104478Ssam struct ubsec_mcr *mcr; 2139104478Ssam struct ubsec_ctx_modexp *ctx; 2140104478Ssam struct ubsec_pktbuf *epb; 2141104478Ssam int err = 0; 2142104478Ssam u_int nbits, normbits, mbits, shiftbits, ebits; 2143104478Ssam 2144104478Ssam me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2145104478Ssam if (me == NULL) { 2146104478Ssam err = ENOMEM; 2147104478Ssam goto errout; 2148104478Ssam } 2149104478Ssam bzero(me, sizeof *me); 2150104478Ssam me->me_krp = krp; 2151104478Ssam me->me_q.q_type = UBS_CTXOP_MODEXP; 2152104478Ssam 2153104478Ssam nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2154104478Ssam if (nbits <= 512) 2155104478Ssam normbits = 512; 2156104478Ssam else if (nbits <= 768) 2157104478Ssam normbits = 768; 2158104478Ssam else if (nbits <= 1024) 2159104478Ssam normbits = 1024; 2160104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2161104478Ssam normbits = 1536; 2162104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2163104478Ssam normbits = 2048; 2164104478Ssam else { 2165104478Ssam err = E2BIG; 2166104478Ssam goto errout; 2167104478Ssam } 2168104478Ssam 2169104630Ssam shiftbits = normbits - nbits; 2170104478Ssam 2171104630Ssam me->me_modbits = nbits; 2172104478Ssam me->me_shiftbits = shiftbits; 2173104630Ssam me->me_normbits = normbits; 2174104478Ssam 2175104478Ssam /* Sanity check: result bits must be >= true modulus bits. */ 2176104630Ssam if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2177104478Ssam err = ERANGE; 2178104478Ssam goto errout; 2179104478Ssam } 2180104478Ssam 2181104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2182104478Ssam &me->me_q.q_mcr, 0)) { 2183104478Ssam err = ENOMEM; 2184104478Ssam goto errout; 2185104478Ssam } 2186104478Ssam mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2187104478Ssam 2188104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2189104478Ssam &me->me_q.q_ctx, 0)) { 2190104478Ssam err = ENOMEM; 2191104478Ssam goto errout; 2192104478Ssam } 2193104478Ssam 2194104478Ssam mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2195104478Ssam if (mbits > nbits) { 2196104478Ssam err = E2BIG; 2197104478Ssam goto errout; 2198104478Ssam } 2199104478Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2200104478Ssam err = ENOMEM; 2201104478Ssam goto errout; 2202104478Ssam } 2203104478Ssam ubsec_kshift_r(shiftbits, 2204104478Ssam krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits, 2205104478Ssam me->me_M.dma_vaddr, normbits); 2206104478Ssam 2207104478Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2208104478Ssam err = ENOMEM; 2209104478Ssam goto errout; 2210104478Ssam } 2211104478Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2212104478Ssam 2213104478Ssam ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2214104478Ssam if (ebits > nbits) { 2215104478Ssam err = E2BIG; 2216104478Ssam goto errout; 2217104478Ssam } 2218104478Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2219104478Ssam err = ENOMEM; 2220104478Ssam goto errout; 2221104478Ssam } 2222104478Ssam ubsec_kshift_r(shiftbits, 2223104478Ssam krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits, 2224104478Ssam me->me_E.dma_vaddr, normbits); 2225104478Ssam 2226104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2227104478Ssam &me->me_epb, 0)) { 2228104478Ssam err = ENOMEM; 2229104478Ssam goto errout; 2230104478Ssam } 2231104478Ssam epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2232104478Ssam epb->pb_addr = htole32(me->me_E.dma_paddr); 2233104478Ssam epb->pb_next = 0; 2234104478Ssam epb->pb_len = htole32(normbits / 8); 2235104478Ssam 2236104478Ssam#ifdef UBSEC_DEBUG 2237104478Ssam if (ubsec_debug) { 2238104478Ssam printf("Epb "); 2239104478Ssam ubsec_dump_pb(epb); 2240104478Ssam } 2241104478Ssam#endif 2242104478Ssam 2243104478Ssam mcr->mcr_pkts = htole16(1); 2244104478Ssam mcr->mcr_flags = 0; 2245104478Ssam mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2246104478Ssam mcr->mcr_reserved = 0; 2247104478Ssam mcr->mcr_pktlen = 0; 2248104478Ssam 2249104478Ssam mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2250104478Ssam mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2251104478Ssam mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2252104478Ssam 2253104478Ssam mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2254104478Ssam mcr->mcr_opktbuf.pb_next = 0; 2255104478Ssam mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2256104478Ssam 2257104478Ssam#ifdef DIAGNOSTIC 2258104478Ssam /* Misaligned output buffer will hang the chip. */ 2259104478Ssam if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2260104478Ssam panic("%s: modexp invalid addr 0x%x\n", 2261104478Ssam device_get_nameunit(sc->sc_dev), 2262104478Ssam letoh32(mcr->mcr_opktbuf.pb_addr)); 2263104478Ssam if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2264104478Ssam panic("%s: modexp invalid len 0x%x\n", 2265104478Ssam device_get_nameunit(sc->sc_dev), 2266104478Ssam letoh32(mcr->mcr_opktbuf.pb_len)); 2267104478Ssam#endif 2268104478Ssam 2269104478Ssam ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2270104478Ssam bzero(ctx, sizeof(*ctx)); 2271104478Ssam ubsec_kshift_r(shiftbits, 2272104478Ssam krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits, 2273104478Ssam ctx->me_N, normbits); 2274104478Ssam ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2275104478Ssam ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2276104630Ssam ctx->me_E_len = htole16(nbits); 2277104630Ssam ctx->me_N_len = htole16(nbits); 2278104478Ssam 2279104478Ssam#ifdef UBSEC_DEBUG 2280104478Ssam if (ubsec_debug) { 2281104478Ssam ubsec_dump_mcr(mcr); 2282104478Ssam ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2283104478Ssam } 2284104478Ssam#endif 2285104478Ssam 2286104478Ssam /* 2287104478Ssam * ubsec_feed2 will sync mcr and ctx, we just need to sync 2288104478Ssam * everything else. 2289104478Ssam */ 2290108823Ssam ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE); 2291108823Ssam ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE); 2292108823Ssam ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD); 2293108823Ssam ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE); 2294104478Ssam 2295104478Ssam /* Enqueue and we're done... */ 2296115747Ssam mtx_lock(&sc->sc_mcr2lock); 2297104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2298104478Ssam ubsec_feed2(sc); 2299104478Ssam ubsecstats.hst_modexp++; 2300115747Ssam mtx_unlock(&sc->sc_mcr2lock); 2301104478Ssam 2302104478Ssam return (0); 2303104478Ssam 2304104478Ssamerrout: 2305104478Ssam if (me != NULL) { 2306267448Sjhb if (me->me_q.q_mcr.dma_tag != NULL) 2307104478Ssam ubsec_dma_free(sc, &me->me_q.q_mcr); 2308267448Sjhb if (me->me_q.q_ctx.dma_tag != NULL) { 2309104478Ssam bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2310104478Ssam ubsec_dma_free(sc, &me->me_q.q_ctx); 2311104478Ssam } 2312267448Sjhb if (me->me_M.dma_tag != NULL) { 2313104478Ssam bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2314104478Ssam ubsec_dma_free(sc, &me->me_M); 2315104478Ssam } 2316267448Sjhb if (me->me_E.dma_tag != NULL) { 2317104478Ssam bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2318104478Ssam ubsec_dma_free(sc, &me->me_E); 2319104478Ssam } 2320267448Sjhb if (me->me_C.dma_tag != NULL) { 2321104478Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2322104478Ssam ubsec_dma_free(sc, &me->me_C); 2323104478Ssam } 2324267448Sjhb if (me->me_epb.dma_tag != NULL) 2325104478Ssam ubsec_dma_free(sc, &me->me_epb); 2326104478Ssam free(me, M_DEVBUF); 2327104478Ssam } 2328104478Ssam krp->krp_status = err; 2329104478Ssam crypto_kdone(krp); 2330104478Ssam return (0); 2331104478Ssam} 2332104478Ssam 2333104630Ssam/* 2334104630Ssam * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization) 2335104630Ssam */ 2336105215Sphkstatic int 2337104630Ssamubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2338104630Ssam{ 2339104630Ssam struct ubsec_q2_modexp *me; 2340104630Ssam struct ubsec_mcr *mcr; 2341104630Ssam struct ubsec_ctx_modexp *ctx; 2342104630Ssam struct ubsec_pktbuf *epb; 2343104630Ssam int err = 0; 2344104630Ssam u_int nbits, normbits, mbits, shiftbits, ebits; 2345104630Ssam 2346104630Ssam me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2347104630Ssam if (me == NULL) { 2348104630Ssam err = ENOMEM; 2349104630Ssam goto errout; 2350104630Ssam } 2351104630Ssam bzero(me, sizeof *me); 2352104630Ssam me->me_krp = krp; 2353104630Ssam me->me_q.q_type = UBS_CTXOP_MODEXP; 2354104630Ssam 2355104630Ssam nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2356104630Ssam if (nbits <= 512) 2357104630Ssam normbits = 512; 2358104630Ssam else if (nbits <= 768) 2359104630Ssam normbits = 768; 2360104630Ssam else if (nbits <= 1024) 2361104630Ssam normbits = 1024; 2362104630Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2363104630Ssam normbits = 1536; 2364104630Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2365104630Ssam normbits = 2048; 2366104630Ssam else { 2367104630Ssam err = E2BIG; 2368104630Ssam goto errout; 2369104630Ssam } 2370104630Ssam 2371104630Ssam shiftbits = normbits - nbits; 2372104630Ssam 2373104630Ssam /* XXX ??? */ 2374104630Ssam me->me_modbits = nbits; 2375104630Ssam me->me_shiftbits = shiftbits; 2376104630Ssam me->me_normbits = normbits; 2377104630Ssam 2378104630Ssam /* Sanity check: result bits must be >= true modulus bits. */ 2379104630Ssam if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2380104630Ssam err = ERANGE; 2381104630Ssam goto errout; 2382104630Ssam } 2383104630Ssam 2384104630Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2385104630Ssam &me->me_q.q_mcr, 0)) { 2386104630Ssam err = ENOMEM; 2387104630Ssam goto errout; 2388104630Ssam } 2389104630Ssam mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2390104630Ssam 2391104630Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2392104630Ssam &me->me_q.q_ctx, 0)) { 2393104630Ssam err = ENOMEM; 2394104630Ssam goto errout; 2395104630Ssam } 2396104630Ssam 2397104630Ssam mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2398104630Ssam if (mbits > nbits) { 2399104630Ssam err = E2BIG; 2400104630Ssam goto errout; 2401104630Ssam } 2402104630Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2403104630Ssam err = ENOMEM; 2404104630Ssam goto errout; 2405104630Ssam } 2406104630Ssam bzero(me->me_M.dma_vaddr, normbits / 8); 2407104630Ssam bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p, 2408104630Ssam me->me_M.dma_vaddr, (mbits + 7) / 8); 2409104630Ssam 2410104630Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2411104630Ssam err = ENOMEM; 2412104630Ssam goto errout; 2413104630Ssam } 2414104630Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2415104630Ssam 2416104630Ssam ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2417104630Ssam if (ebits > nbits) { 2418104630Ssam err = E2BIG; 2419104630Ssam goto errout; 2420104630Ssam } 2421104630Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2422104630Ssam err = ENOMEM; 2423104630Ssam goto errout; 2424104630Ssam } 2425104630Ssam bzero(me->me_E.dma_vaddr, normbits / 8); 2426104630Ssam bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p, 2427104630Ssam me->me_E.dma_vaddr, (ebits + 7) / 8); 2428104630Ssam 2429104630Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2430104630Ssam &me->me_epb, 0)) { 2431104630Ssam err = ENOMEM; 2432104630Ssam goto errout; 2433104630Ssam } 2434104630Ssam epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2435104630Ssam epb->pb_addr = htole32(me->me_E.dma_paddr); 2436104630Ssam epb->pb_next = 0; 2437104630Ssam epb->pb_len = htole32((ebits + 7) / 8); 2438104630Ssam 2439104630Ssam#ifdef UBSEC_DEBUG 2440108823Ssam if (ubsec_debug) { 2441108823Ssam printf("Epb "); 2442108823Ssam ubsec_dump_pb(epb); 2443108823Ssam } 2444104630Ssam#endif 2445104630Ssam 2446104630Ssam mcr->mcr_pkts = htole16(1); 2447104630Ssam mcr->mcr_flags = 0; 2448104630Ssam mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2449104630Ssam mcr->mcr_reserved = 0; 2450104630Ssam mcr->mcr_pktlen = 0; 2451104630Ssam 2452104630Ssam mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2453104630Ssam mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2454104630Ssam mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2455104630Ssam 2456104630Ssam mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2457104630Ssam mcr->mcr_opktbuf.pb_next = 0; 2458104630Ssam mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2459104630Ssam 2460104630Ssam#ifdef DIAGNOSTIC 2461104630Ssam /* Misaligned output buffer will hang the chip. */ 2462104630Ssam if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2463104630Ssam panic("%s: modexp invalid addr 0x%x\n", 2464104630Ssam device_get_nameunit(sc->sc_dev), 2465104630Ssam letoh32(mcr->mcr_opktbuf.pb_addr)); 2466104630Ssam if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2467104630Ssam panic("%s: modexp invalid len 0x%x\n", 2468104630Ssam device_get_nameunit(sc->sc_dev), 2469104630Ssam letoh32(mcr->mcr_opktbuf.pb_len)); 2470104630Ssam#endif 2471104630Ssam 2472104630Ssam ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2473104630Ssam bzero(ctx, sizeof(*ctx)); 2474104630Ssam bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N, 2475104630Ssam (nbits + 7) / 8); 2476104630Ssam ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2477104630Ssam ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2478104630Ssam ctx->me_E_len = htole16(ebits); 2479104630Ssam ctx->me_N_len = htole16(nbits); 2480104630Ssam 2481104630Ssam#ifdef UBSEC_DEBUG 2482104630Ssam if (ubsec_debug) { 2483104630Ssam ubsec_dump_mcr(mcr); 2484104630Ssam ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2485104630Ssam } 2486104630Ssam#endif 2487104630Ssam 2488104630Ssam /* 2489104630Ssam * ubsec_feed2 will sync mcr and ctx, we just need to sync 2490104630Ssam * everything else. 2491104630Ssam */ 2492108823Ssam ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE); 2493108823Ssam ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE); 2494108823Ssam ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD); 2495108823Ssam ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE); 2496104630Ssam 2497104630Ssam /* Enqueue and we're done... */ 2498115747Ssam mtx_lock(&sc->sc_mcr2lock); 2499104630Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2500104630Ssam ubsec_feed2(sc); 2501115747Ssam mtx_unlock(&sc->sc_mcr2lock); 2502104630Ssam 2503104630Ssam return (0); 2504104630Ssam 2505104630Ssamerrout: 2506104630Ssam if (me != NULL) { 2507267448Sjhb if (me->me_q.q_mcr.dma_tag != NULL) 2508104630Ssam ubsec_dma_free(sc, &me->me_q.q_mcr); 2509267448Sjhb if (me->me_q.q_ctx.dma_tag != NULL) { 2510104630Ssam bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2511104630Ssam ubsec_dma_free(sc, &me->me_q.q_ctx); 2512104630Ssam } 2513267448Sjhb if (me->me_M.dma_tag != NULL) { 2514104630Ssam bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2515104630Ssam ubsec_dma_free(sc, &me->me_M); 2516104630Ssam } 2517267448Sjhb if (me->me_E.dma_tag != NULL) { 2518104630Ssam bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2519104630Ssam ubsec_dma_free(sc, &me->me_E); 2520104630Ssam } 2521267448Sjhb if (me->me_C.dma_tag != NULL) { 2522104630Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2523104630Ssam ubsec_dma_free(sc, &me->me_C); 2524104630Ssam } 2525267448Sjhb if (me->me_epb.dma_tag != NULL) 2526104630Ssam ubsec_dma_free(sc, &me->me_epb); 2527104630Ssam free(me, M_DEVBUF); 2528104630Ssam } 2529104630Ssam krp->krp_status = err; 2530104630Ssam crypto_kdone(krp); 2531104630Ssam return (0); 2532104630Ssam} 2533104630Ssam 2534104478Ssamstatic int 2535104478Ssamubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2536104478Ssam{ 2537104478Ssam struct ubsec_q2_rsapriv *rp = NULL; 2538104478Ssam struct ubsec_mcr *mcr; 2539104478Ssam struct ubsec_ctx_rsapriv *ctx; 2540104478Ssam int err = 0; 2541104478Ssam u_int padlen, msglen; 2542104478Ssam 2543104478Ssam msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]); 2544104478Ssam padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]); 2545104478Ssam if (msglen > padlen) 2546104478Ssam padlen = msglen; 2547104478Ssam 2548104478Ssam if (padlen <= 256) 2549104478Ssam padlen = 256; 2550104478Ssam else if (padlen <= 384) 2551104478Ssam padlen = 384; 2552104478Ssam else if (padlen <= 512) 2553104478Ssam padlen = 512; 2554104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768) 2555104478Ssam padlen = 768; 2556104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024) 2557104478Ssam padlen = 1024; 2558104478Ssam else { 2559104478Ssam err = E2BIG; 2560104478Ssam goto errout; 2561104478Ssam } 2562104478Ssam 2563104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) { 2564104478Ssam err = E2BIG; 2565104478Ssam goto errout; 2566104478Ssam } 2567104478Ssam 2568104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) { 2569104478Ssam err = E2BIG; 2570104478Ssam goto errout; 2571104478Ssam } 2572104478Ssam 2573104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) { 2574104478Ssam err = E2BIG; 2575104478Ssam goto errout; 2576104478Ssam } 2577104478Ssam 2578104478Ssam rp = (struct ubsec_q2_rsapriv *)malloc(sizeof *rp, M_DEVBUF, M_NOWAIT); 2579104478Ssam if (rp == NULL) 2580104478Ssam return (ENOMEM); 2581104478Ssam bzero(rp, sizeof *rp); 2582104478Ssam rp->rpr_krp = krp; 2583104478Ssam rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV; 2584104478Ssam 2585104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2586104478Ssam &rp->rpr_q.q_mcr, 0)) { 2587104478Ssam err = ENOMEM; 2588104478Ssam goto errout; 2589104478Ssam } 2590104478Ssam mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr; 2591104478Ssam 2592104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv), 2593104478Ssam &rp->rpr_q.q_ctx, 0)) { 2594104478Ssam err = ENOMEM; 2595104478Ssam goto errout; 2596104478Ssam } 2597104478Ssam ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr; 2598104478Ssam bzero(ctx, sizeof *ctx); 2599104478Ssam 2600104478Ssam /* Copy in p */ 2601104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p, 2602104478Ssam &ctx->rpr_buf[0 * (padlen / 8)], 2603104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8); 2604104478Ssam 2605104478Ssam /* Copy in q */ 2606104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p, 2607104478Ssam &ctx->rpr_buf[1 * (padlen / 8)], 2608104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8); 2609104478Ssam 2610104478Ssam /* Copy in dp */ 2611104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p, 2612104478Ssam &ctx->rpr_buf[2 * (padlen / 8)], 2613104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8); 2614104478Ssam 2615104478Ssam /* Copy in dq */ 2616104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p, 2617104478Ssam &ctx->rpr_buf[3 * (padlen / 8)], 2618104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8); 2619104478Ssam 2620104478Ssam /* Copy in pinv */ 2621104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p, 2622104478Ssam &ctx->rpr_buf[4 * (padlen / 8)], 2623104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8); 2624104478Ssam 2625104478Ssam msglen = padlen * 2; 2626104478Ssam 2627104478Ssam /* Copy in input message (aligned buffer/length). */ 2628104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) { 2629104478Ssam /* Is this likely? */ 2630104478Ssam err = E2BIG; 2631104478Ssam goto errout; 2632104478Ssam } 2633104478Ssam if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) { 2634104478Ssam err = ENOMEM; 2635104478Ssam goto errout; 2636104478Ssam } 2637104478Ssam bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8); 2638104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p, 2639104478Ssam rp->rpr_msgin.dma_vaddr, 2640104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8); 2641104478Ssam 2642104478Ssam /* Prepare space for output message (aligned buffer/length). */ 2643104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) { 2644104478Ssam /* Is this likely? */ 2645104478Ssam err = E2BIG; 2646104478Ssam goto errout; 2647104478Ssam } 2648104478Ssam if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) { 2649104478Ssam err = ENOMEM; 2650104478Ssam goto errout; 2651104478Ssam } 2652104478Ssam bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8); 2653104478Ssam 2654104478Ssam mcr->mcr_pkts = htole16(1); 2655104478Ssam mcr->mcr_flags = 0; 2656104478Ssam mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr); 2657104478Ssam mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr); 2658104478Ssam mcr->mcr_ipktbuf.pb_next = 0; 2659104478Ssam mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size); 2660104478Ssam mcr->mcr_reserved = 0; 2661104478Ssam mcr->mcr_pktlen = htole16(msglen); 2662104478Ssam mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr); 2663104478Ssam mcr->mcr_opktbuf.pb_next = 0; 2664104478Ssam mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size); 2665104478Ssam 2666104478Ssam#ifdef DIAGNOSTIC 2667104478Ssam if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) { 2668106579Sjhb panic("%s: rsapriv: invalid msgin %x(0x%jx)", 2669104478Ssam device_get_nameunit(sc->sc_dev), 2670106579Sjhb rp->rpr_msgin.dma_paddr, (uintmax_t)rp->rpr_msgin.dma_size); 2671104478Ssam } 2672104478Ssam if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) { 2673106579Sjhb panic("%s: rsapriv: invalid msgout %x(0x%jx)", 2674104478Ssam device_get_nameunit(sc->sc_dev), 2675106579Sjhb rp->rpr_msgout.dma_paddr, (uintmax_t)rp->rpr_msgout.dma_size); 2676104478Ssam } 2677104478Ssam#endif 2678104478Ssam 2679104478Ssam ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8)); 2680104478Ssam ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV); 2681104478Ssam ctx->rpr_q_len = htole16(padlen); 2682104478Ssam ctx->rpr_p_len = htole16(padlen); 2683104478Ssam 2684104478Ssam /* 2685104478Ssam * ubsec_feed2 will sync mcr and ctx, we just need to sync 2686104478Ssam * everything else. 2687104478Ssam */ 2688108823Ssam ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_PREWRITE); 2689108823Ssam ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_PREREAD); 2690104478Ssam 2691104478Ssam /* Enqueue and we're done... */ 2692115747Ssam mtx_lock(&sc->sc_mcr2lock); 2693104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next); 2694104478Ssam ubsec_feed2(sc); 2695104478Ssam ubsecstats.hst_modexpcrt++; 2696115747Ssam mtx_unlock(&sc->sc_mcr2lock); 2697104478Ssam return (0); 2698104478Ssam 2699104478Ssamerrout: 2700104478Ssam if (rp != NULL) { 2701267448Sjhb if (rp->rpr_q.q_mcr.dma_tag != NULL) 2702104478Ssam ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2703267448Sjhb if (rp->rpr_msgin.dma_tag != NULL) { 2704104478Ssam bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 2705104478Ssam ubsec_dma_free(sc, &rp->rpr_msgin); 2706104478Ssam } 2707267448Sjhb if (rp->rpr_msgout.dma_tag != NULL) { 2708104478Ssam bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 2709104478Ssam ubsec_dma_free(sc, &rp->rpr_msgout); 2710104478Ssam } 2711104478Ssam free(rp, M_DEVBUF); 2712104478Ssam } 2713104478Ssam krp->krp_status = err; 2714104478Ssam crypto_kdone(krp); 2715104478Ssam return (0); 2716104478Ssam} 2717104478Ssam 2718104478Ssam#ifdef UBSEC_DEBUG 2719104478Ssamstatic void 2720104478Ssamubsec_dump_pb(volatile struct ubsec_pktbuf *pb) 2721104478Ssam{ 2722104478Ssam printf("addr 0x%x (0x%x) next 0x%x\n", 2723104478Ssam pb->pb_addr, pb->pb_len, pb->pb_next); 2724104478Ssam} 2725104478Ssam 2726104478Ssamstatic void 2727104478Ssamubsec_dump_ctx2(struct ubsec_ctx_keyop *c) 2728104478Ssam{ 2729104478Ssam printf("CTX (0x%x):\n", c->ctx_len); 2730104478Ssam switch (letoh16(c->ctx_op)) { 2731104478Ssam case UBS_CTXOP_RNGBYPASS: 2732104478Ssam case UBS_CTXOP_RNGSHA1: 2733104478Ssam break; 2734104478Ssam case UBS_CTXOP_MODEXP: 2735104478Ssam { 2736104478Ssam struct ubsec_ctx_modexp *cx = (void *)c; 2737104478Ssam int i, len; 2738104478Ssam 2739104478Ssam printf(" Elen %u, Nlen %u\n", 2740104478Ssam letoh16(cx->me_E_len), letoh16(cx->me_N_len)); 2741104478Ssam len = (cx->me_N_len + 7)/8; 2742104478Ssam for (i = 0; i < len; i++) 2743104478Ssam printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]); 2744104478Ssam printf("\n"); 2745104478Ssam break; 2746104478Ssam } 2747104478Ssam default: 2748104478Ssam printf("unknown context: %x\n", c->ctx_op); 2749104478Ssam } 2750104478Ssam printf("END CTX\n"); 2751104478Ssam} 2752104478Ssam 2753104478Ssamstatic void 2754104478Ssamubsec_dump_mcr(struct ubsec_mcr *mcr) 2755104478Ssam{ 2756104478Ssam volatile struct ubsec_mcr_add *ma; 2757104478Ssam int i; 2758104478Ssam 2759104478Ssam printf("MCR:\n"); 2760104478Ssam printf(" pkts: %u, flags 0x%x\n", 2761104478Ssam letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags)); 2762104478Ssam ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp; 2763104478Ssam for (i = 0; i < letoh16(mcr->mcr_pkts); i++) { 2764104478Ssam printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i, 2765104478Ssam letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen), 2766104478Ssam letoh16(ma->mcr_reserved)); 2767104478Ssam printf(" %d: ipkt ", i); 2768104478Ssam ubsec_dump_pb(&ma->mcr_ipktbuf); 2769104478Ssam printf(" %d: opkt ", i); 2770104478Ssam ubsec_dump_pb(&ma->mcr_opktbuf); 2771104478Ssam ma++; 2772104478Ssam } 2773104478Ssam printf("END MCR\n"); 2774104478Ssam} 2775104478Ssam#endif /* UBSEC_DEBUG */ 2776104478Ssam 2777104478Ssam/* 2778104478Ssam * Return the number of significant bits of a big number. 2779104478Ssam */ 2780104478Ssamstatic int 2781104478Ssamubsec_ksigbits(struct crparam *cr) 2782104478Ssam{ 2783104478Ssam u_int plen = (cr->crp_nbits + 7) / 8; 2784104478Ssam int i, sig = plen * 8; 2785104478Ssam u_int8_t c, *p = cr->crp_p; 2786104478Ssam 2787104478Ssam for (i = plen - 1; i >= 0; i--) { 2788104478Ssam c = p[i]; 2789104478Ssam if (c != 0) { 2790104478Ssam while ((c & 0x80) == 0) { 2791104478Ssam sig--; 2792104478Ssam c <<= 1; 2793104478Ssam } 2794104478Ssam break; 2795104478Ssam } 2796104478Ssam sig -= 8; 2797104478Ssam } 2798104478Ssam return (sig); 2799104478Ssam} 2800104478Ssam 2801104478Ssamstatic void 2802104478Ssamubsec_kshift_r( 2803104478Ssam u_int shiftbits, 2804104478Ssam u_int8_t *src, u_int srcbits, 2805104478Ssam u_int8_t *dst, u_int dstbits) 2806104478Ssam{ 2807104478Ssam u_int slen, dlen; 2808104478Ssam int i, si, di, n; 2809104478Ssam 2810104478Ssam slen = (srcbits + 7) / 8; 2811104478Ssam dlen = (dstbits + 7) / 8; 2812104478Ssam 2813104478Ssam for (i = 0; i < slen; i++) 2814104478Ssam dst[i] = src[i]; 2815104478Ssam for (i = 0; i < dlen - slen; i++) 2816104478Ssam dst[slen + i] = 0; 2817104478Ssam 2818104478Ssam n = shiftbits / 8; 2819104478Ssam if (n != 0) { 2820104478Ssam si = dlen - n - 1; 2821104478Ssam di = dlen - 1; 2822104478Ssam while (si >= 0) 2823104478Ssam dst[di--] = dst[si--]; 2824104478Ssam while (di >= 0) 2825104478Ssam dst[di--] = 0; 2826104478Ssam } 2827104478Ssam 2828104478Ssam n = shiftbits % 8; 2829104478Ssam if (n != 0) { 2830104478Ssam for (i = dlen - 1; i > 0; i--) 2831104478Ssam dst[i] = (dst[i] << n) | 2832104478Ssam (dst[i - 1] >> (8 - n)); 2833104478Ssam dst[0] = dst[0] << n; 2834104478Ssam } 2835104478Ssam} 2836104478Ssam 2837104478Ssamstatic void 2838104478Ssamubsec_kshift_l( 2839104478Ssam u_int shiftbits, 2840104478Ssam u_int8_t *src, u_int srcbits, 2841104478Ssam u_int8_t *dst, u_int dstbits) 2842104478Ssam{ 2843104478Ssam int slen, dlen, i, n; 2844104478Ssam 2845104478Ssam slen = (srcbits + 7) / 8; 2846104478Ssam dlen = (dstbits + 7) / 8; 2847104478Ssam 2848104478Ssam n = shiftbits / 8; 2849104478Ssam for (i = 0; i < slen; i++) 2850104478Ssam dst[i] = src[i + n]; 2851104478Ssam for (i = 0; i < dlen - slen; i++) 2852104478Ssam dst[slen + i] = 0; 2853104478Ssam 2854104478Ssam n = shiftbits % 8; 2855104478Ssam if (n != 0) { 2856104478Ssam for (i = 0; i < (dlen - 1); i++) 2857104478Ssam dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n)); 2858104478Ssam dst[dlen - 1] = dst[dlen - 1] >> n; 2859104478Ssam } 2860104478Ssam} 2861