ubsec.c revision 111416
1104478Ssam/* $FreeBSD: head/sys/dev/ubsec/ubsec.c 111416 2003-02-24 06:03:13Z sam $ */ 2104630Ssam/* $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $ */ 3104478Ssam 4104478Ssam/* 5104478Ssam * Copyright (c) 2000 Jason L. Wright (jason@thought.net) 6104478Ssam * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org) 7104478Ssam * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com) 8104478Ssam * 9104478Ssam * All rights reserved. 10104478Ssam * 11104478Ssam * Redistribution and use in source and binary forms, with or without 12104478Ssam * modification, are permitted provided that the following conditions 13104478Ssam * are met: 14104478Ssam * 1. Redistributions of source code must retain the above copyright 15104478Ssam * notice, this list of conditions and the following disclaimer. 16104478Ssam * 2. Redistributions in binary form must reproduce the above copyright 17104478Ssam * notice, this list of conditions and the following disclaimer in the 18104478Ssam * documentation and/or other materials provided with the distribution. 19104478Ssam * 3. All advertising materials mentioning features or use of this software 20104478Ssam * must display the following acknowledgement: 21104478Ssam * This product includes software developed by Jason L. Wright 22104478Ssam * 4. The name of the author may not be used to endorse or promote products 23104478Ssam * derived from this software without specific prior written permission. 24104478Ssam * 25104478Ssam * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 26104478Ssam * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 27104478Ssam * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 28104478Ssam * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 29104478Ssam * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 30104478Ssam * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 31104478Ssam * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32104478Ssam * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 33104478Ssam * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 34104478Ssam * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35104478Ssam * POSSIBILITY OF SUCH DAMAGE. 36104478Ssam * 37104478Ssam * Effort sponsored in part by the Defense Advanced Research Projects 38104478Ssam * Agency (DARPA) and Air Force Research Laboratory, Air Force 39104478Ssam * Materiel Command, USAF, under agreement number F30602-01-2-0537. 40104478Ssam * 41104478Ssam */ 42104478Ssam 43104478Ssam#define UBSEC_DEBUG 44104478Ssam 45104478Ssam/* 46104478Ssam * uBsec 5[56]01, 58xx hardware crypto accelerator 47104478Ssam */ 48104478Ssam 49104478Ssam#include <sys/param.h> 50104478Ssam#include <sys/systm.h> 51104478Ssam#include <sys/proc.h> 52104478Ssam#include <sys/errno.h> 53104478Ssam#include <sys/malloc.h> 54104478Ssam#include <sys/kernel.h> 55104478Ssam#include <sys/mbuf.h> 56104478Ssam#include <sys/lock.h> 57104478Ssam#include <sys/mutex.h> 58106579Sjhb#include <sys/stdint.h> 59104478Ssam#include <sys/sysctl.h> 60104478Ssam#include <sys/endian.h> 61104478Ssam 62104478Ssam#include <vm/vm.h> 63104478Ssam#include <vm/pmap.h> 64104478Ssam 65104478Ssam#include <machine/clock.h> 66104478Ssam#include <machine/bus.h> 67104478Ssam#include <machine/resource.h> 68104478Ssam#include <sys/bus.h> 69104478Ssam#include <sys/rman.h> 70104478Ssam 71104478Ssam#include <crypto/sha1.h> 72104478Ssam#include <opencrypto/cryptodev.h> 73104478Ssam#include <opencrypto/cryptosoft.h> 74104478Ssam#include <sys/md5.h> 75104478Ssam#include <sys/random.h> 76104478Ssam 77104478Ssam#include <pci/pcivar.h> 78104478Ssam#include <pci/pcireg.h> 79104478Ssam 80104478Ssam/* grr, #defines for gratuitous incompatibility in queue.h */ 81104478Ssam#define SIMPLEQ_HEAD STAILQ_HEAD 82104478Ssam#define SIMPLEQ_ENTRY STAILQ_ENTRY 83104478Ssam#define SIMPLEQ_INIT STAILQ_INIT 84104478Ssam#define SIMPLEQ_INSERT_TAIL STAILQ_INSERT_TAIL 85104478Ssam#define SIMPLEQ_EMPTY STAILQ_EMPTY 86104478Ssam#define SIMPLEQ_FIRST STAILQ_FIRST 87104478Ssam#define SIMPLEQ_REMOVE_HEAD STAILQ_REMOVE_HEAD_UNTIL 88111416Ssam#define SIMPLEQ_FOREACH STAILQ_FOREACH 89104478Ssam/* ditto for endian.h */ 90104478Ssam#define letoh16(x) le16toh(x) 91104478Ssam#define letoh32(x) le32toh(x) 92104478Ssam 93104478Ssam#include <dev/ubsec/ubsecreg.h> 94104478Ssam#include <dev/ubsec/ubsecvar.h> 95104478Ssam 96104478Ssam/* 97104478Ssam * Prototypes and count for the pci_device structure 98104478Ssam */ 99104478Ssamstatic int ubsec_probe(device_t); 100104478Ssamstatic int ubsec_attach(device_t); 101104478Ssamstatic int ubsec_detach(device_t); 102104478Ssamstatic int ubsec_suspend(device_t); 103104478Ssamstatic int ubsec_resume(device_t); 104104478Ssamstatic void ubsec_shutdown(device_t); 105104478Ssam 106104478Ssamstatic device_method_t ubsec_methods[] = { 107104478Ssam /* Device interface */ 108104478Ssam DEVMETHOD(device_probe, ubsec_probe), 109104478Ssam DEVMETHOD(device_attach, ubsec_attach), 110104478Ssam DEVMETHOD(device_detach, ubsec_detach), 111104478Ssam DEVMETHOD(device_suspend, ubsec_suspend), 112104478Ssam DEVMETHOD(device_resume, ubsec_resume), 113104478Ssam DEVMETHOD(device_shutdown, ubsec_shutdown), 114104478Ssam 115104478Ssam /* bus interface */ 116104478Ssam DEVMETHOD(bus_print_child, bus_generic_print_child), 117104478Ssam DEVMETHOD(bus_driver_added, bus_generic_driver_added), 118104478Ssam 119104478Ssam { 0, 0 } 120104478Ssam}; 121104478Ssamstatic driver_t ubsec_driver = { 122104478Ssam "ubsec", 123104478Ssam ubsec_methods, 124104478Ssam sizeof (struct ubsec_softc) 125104478Ssam}; 126104478Ssamstatic devclass_t ubsec_devclass; 127104478Ssam 128104478SsamDRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0); 129105251SmarkmMODULE_DEPEND(ubsec, crypto, 1, 1, 1); 130104478Ssam 131104478Ssamstatic void ubsec_intr(void *); 132104478Ssamstatic int ubsec_newsession(void *, u_int32_t *, struct cryptoini *); 133104478Ssamstatic int ubsec_freesession(void *, u_int64_t); 134104478Ssamstatic int ubsec_process(void *, struct cryptop *, int); 135104478Ssamstatic void ubsec_callback(struct ubsec_softc *, struct ubsec_q *); 136111416Ssamstatic void ubsec_feed(struct ubsec_softc *); 137104478Ssamstatic void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int); 138104478Ssamstatic void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *); 139104478Ssamstatic int ubsec_feed2(struct ubsec_softc *); 140104478Ssamstatic void ubsec_rng(void *); 141104478Ssamstatic int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t, 142104478Ssam struct ubsec_dma_alloc *, int); 143108823Ssam#define ubsec_dma_sync(_dma, _flags) \ 144108823Ssam bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags)) 145104478Ssamstatic void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *); 146104478Ssamstatic int ubsec_dmamap_aligned(struct ubsec_operand *op); 147104478Ssam 148104478Ssamstatic void ubsec_reset_board(struct ubsec_softc *sc); 149104478Ssamstatic void ubsec_init_board(struct ubsec_softc *sc); 150104478Ssamstatic void ubsec_init_pciregs(device_t dev); 151104478Ssamstatic void ubsec_totalreset(struct ubsec_softc *sc); 152104478Ssam 153104478Ssamstatic int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q); 154104478Ssam 155104478Ssamstatic int ubsec_kprocess(void*, struct cryptkop *, int); 156104630Ssamstatic int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int); 157104630Ssamstatic int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int); 158104478Ssamstatic int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int); 159104478Ssamstatic void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *); 160104478Ssamstatic int ubsec_ksigbits(struct crparam *); 161104478Ssamstatic void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 162104478Ssamstatic void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 163104478Ssam 164109595SsamSYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0, "Broadcom driver parameters"); 165109595Ssam 166104478Ssam#ifdef UBSEC_DEBUG 167104478Ssamstatic void ubsec_dump_pb(volatile struct ubsec_pktbuf *); 168104478Ssamstatic void ubsec_dump_mcr(struct ubsec_mcr *); 169104478Ssamstatic void ubsec_dump_ctx2(struct ubsec_ctx_keyop *); 170104478Ssam 171104478Ssamstatic int ubsec_debug = 0; 172109595SsamSYSCTL_INT(_hw_ubsec, OID_AUTO, debug, CTLFLAG_RW, &ubsec_debug, 173109595Ssam 0, "control debugging msgs"); 174104478Ssam#endif 175104478Ssam 176104478Ssam#define READ_REG(sc,r) \ 177104478Ssam bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r)) 178104478Ssam 179104478Ssam#define WRITE_REG(sc,reg,val) \ 180104478Ssam bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val) 181104478Ssam 182104478Ssam#define SWAP32(x) (x) = htole32(ntohl((x))) 183104478Ssam#define HTOLE32(x) (x) = htole32(x) 184104478Ssam 185104478Ssamstruct ubsec_stats ubsecstats; 186109595SsamSYSCTL_STRUCT(_hw_ubsec, OID_AUTO, stats, CTLFLAG_RD, &ubsecstats, 187109595Ssam ubsec_stats, "driver statistics"); 188104478Ssam 189105215Sphkstatic int 190104478Ssamubsec_probe(device_t dev) 191104478Ssam{ 192104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 193104478Ssam (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 || 194104478Ssam pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)) 195104478Ssam return (0); 196104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 197104478Ssam (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 || 198104478Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 || 199104478Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 200110522Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 || 201110522Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 202110522Ssam )) 203104478Ssam return (0); 204104478Ssam return (ENXIO); 205104478Ssam} 206104478Ssam 207104478Ssamstatic const char* 208104478Ssamubsec_partname(struct ubsec_softc *sc) 209104478Ssam{ 210104478Ssam /* XXX sprintf numbers when not decoded */ 211104478Ssam switch (pci_get_vendor(sc->sc_dev)) { 212104478Ssam case PCI_VENDOR_BROADCOM: 213104478Ssam switch (pci_get_device(sc->sc_dev)) { 214104478Ssam case PCI_PRODUCT_BROADCOM_5805: return "Broadcom 5805"; 215104478Ssam case PCI_PRODUCT_BROADCOM_5820: return "Broadcom 5820"; 216104478Ssam case PCI_PRODUCT_BROADCOM_5821: return "Broadcom 5821"; 217104478Ssam case PCI_PRODUCT_BROADCOM_5822: return "Broadcom 5822"; 218110522Ssam case PCI_PRODUCT_BROADCOM_5823: return "Broadcom 5823"; 219104478Ssam } 220104478Ssam return "Broadcom unknown-part"; 221104478Ssam case PCI_VENDOR_BLUESTEEL: 222104478Ssam switch (pci_get_device(sc->sc_dev)) { 223104478Ssam case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601"; 224104478Ssam } 225104478Ssam return "Bluesteel unknown-part"; 226104478Ssam } 227104478Ssam return "Unknown-vendor unknown-part"; 228104478Ssam} 229104478Ssam 230104478Ssamstatic int 231104478Ssamubsec_attach(device_t dev) 232104478Ssam{ 233104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 234104478Ssam struct ubsec_dma *dmap; 235104478Ssam u_int32_t cmd, i; 236104478Ssam int rid; 237104478Ssam 238104478Ssam KASSERT(sc != NULL, ("ubsec_attach: null software carrier!")); 239104478Ssam bzero(sc, sizeof (*sc)); 240104478Ssam sc->sc_dev = dev; 241104478Ssam 242104478Ssam mtx_init(&sc->sc_mtx, device_get_nameunit(dev), "crypto driver", MTX_DEF); 243104478Ssam 244104478Ssam SIMPLEQ_INIT(&sc->sc_queue); 245104478Ssam SIMPLEQ_INIT(&sc->sc_qchip); 246104478Ssam SIMPLEQ_INIT(&sc->sc_queue2); 247104478Ssam SIMPLEQ_INIT(&sc->sc_qchip2); 248104478Ssam SIMPLEQ_INIT(&sc->sc_q2free); 249104478Ssam 250104478Ssam /* XXX handle power management */ 251104478Ssam 252104478Ssam sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR; 253104478Ssam 254104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 255104478Ssam pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601) 256104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 257104478Ssam 258104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 259104478Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805) 260104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 261104478Ssam 262104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 263104478Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820) 264104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 265104478Ssam UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 266104478Ssam 267104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 268104478Ssam (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 269110522Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 || 270110522Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 )) { 271104478Ssam /* NB: the 5821/5822 defines some additional status bits */ 272104478Ssam sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY | 273104478Ssam BS_STAT_MCR2_ALLEMPTY; 274104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 275104478Ssam UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 276104478Ssam } 277104478Ssam 278104478Ssam cmd = pci_read_config(dev, PCIR_COMMAND, 4); 279104478Ssam cmd |= PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN; 280104478Ssam pci_write_config(dev, PCIR_COMMAND, cmd, 4); 281104478Ssam cmd = pci_read_config(dev, PCIR_COMMAND, 4); 282104478Ssam 283104478Ssam if (!(cmd & PCIM_CMD_MEMEN)) { 284104478Ssam device_printf(dev, "failed to enable memory mapping\n"); 285104478Ssam goto bad; 286104478Ssam } 287104478Ssam 288104478Ssam if (!(cmd & PCIM_CMD_BUSMASTEREN)) { 289104478Ssam device_printf(dev, "failed to enable bus mastering\n"); 290104478Ssam goto bad; 291104478Ssam } 292104478Ssam 293104478Ssam /* 294104478Ssam * Setup memory-mapping of PCI registers. 295104478Ssam */ 296104478Ssam rid = BS_BAR; 297104478Ssam sc->sc_sr = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid, 298104478Ssam 0, ~0, 1, RF_ACTIVE); 299104478Ssam if (sc->sc_sr == NULL) { 300104478Ssam device_printf(dev, "cannot map register space\n"); 301104478Ssam goto bad; 302104478Ssam } 303104478Ssam sc->sc_st = rman_get_bustag(sc->sc_sr); 304104478Ssam sc->sc_sh = rman_get_bushandle(sc->sc_sr); 305104478Ssam 306104478Ssam /* 307104478Ssam * Arrange interrupt line. 308104478Ssam */ 309104478Ssam rid = 0; 310104478Ssam sc->sc_irq = bus_alloc_resource(dev, SYS_RES_IRQ, &rid, 311104478Ssam 0, ~0, 1, RF_SHAREABLE|RF_ACTIVE); 312104478Ssam if (sc->sc_irq == NULL) { 313104478Ssam device_printf(dev, "could not map interrupt\n"); 314108823Ssam goto bad1; 315104478Ssam } 316104478Ssam /* 317104478Ssam * NB: Network code assumes we are blocked with splimp() 318104478Ssam * so make sure the IRQ is mapped appropriately. 319104478Ssam */ 320104478Ssam if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET, 321104478Ssam ubsec_intr, sc, &sc->sc_ih)) { 322104478Ssam device_printf(dev, "could not establish interrupt\n"); 323108823Ssam goto bad2; 324104478Ssam } 325104478Ssam 326104478Ssam sc->sc_cid = crypto_get_driverid(0); 327104478Ssam if (sc->sc_cid < 0) { 328104478Ssam device_printf(dev, "could not get crypto driver id\n"); 329108823Ssam goto bad3; 330104478Ssam } 331104478Ssam 332104478Ssam /* 333104478Ssam * Setup DMA descriptor area. 334104478Ssam */ 335104478Ssam if (bus_dma_tag_create(NULL, /* parent */ 336104478Ssam 1, 0, /* alignment, bounds */ 337104478Ssam BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 338104478Ssam BUS_SPACE_MAXADDR, /* highaddr */ 339104478Ssam NULL, NULL, /* filter, filterarg */ 340108823Ssam 0x3ffff, /* maxsize */ 341104478Ssam UBS_MAX_SCATTER, /* nsegments */ 342108823Ssam 0xffff, /* maxsegsize */ 343104478Ssam BUS_DMA_ALLOCNOW, /* flags */ 344104478Ssam &sc->sc_dmat)) { 345104478Ssam device_printf(dev, "cannot allocate DMA tag\n"); 346108823Ssam goto bad4; 347104478Ssam } 348104478Ssam SIMPLEQ_INIT(&sc->sc_freequeue); 349104478Ssam dmap = sc->sc_dmaa; 350104478Ssam for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) { 351104478Ssam struct ubsec_q *q; 352104478Ssam 353104478Ssam q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q), 354104478Ssam M_DEVBUF, M_NOWAIT); 355104478Ssam if (q == NULL) { 356104478Ssam device_printf(dev, "cannot allocate queue buffers\n"); 357104478Ssam break; 358104478Ssam } 359104478Ssam 360104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk), 361104478Ssam &dmap->d_alloc, 0)) { 362104478Ssam device_printf(dev, "cannot allocate dma buffers\n"); 363104478Ssam free(q, M_DEVBUF); 364104478Ssam break; 365104478Ssam } 366104478Ssam dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr; 367104478Ssam 368104478Ssam q->q_dma = dmap; 369104478Ssam sc->sc_queuea[i] = q; 370104478Ssam 371104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 372104478Ssam } 373104478Ssam 374104478Ssam device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc)); 375104478Ssam 376104478Ssam crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0, 377104478Ssam ubsec_newsession, ubsec_freesession, ubsec_process, sc); 378104478Ssam crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0, 379104478Ssam ubsec_newsession, ubsec_freesession, ubsec_process, sc); 380104478Ssam crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0, 381104478Ssam ubsec_newsession, ubsec_freesession, ubsec_process, sc); 382104478Ssam crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0, 383104478Ssam ubsec_newsession, ubsec_freesession, ubsec_process, sc); 384104478Ssam 385104478Ssam /* 386104478Ssam * Reset Broadcom chip 387104478Ssam */ 388104478Ssam ubsec_reset_board(sc); 389104478Ssam 390104478Ssam /* 391104478Ssam * Init Broadcom specific PCI settings 392104478Ssam */ 393104478Ssam ubsec_init_pciregs(dev); 394104478Ssam 395104478Ssam /* 396104478Ssam * Init Broadcom chip 397104478Ssam */ 398104478Ssam ubsec_init_board(sc); 399104478Ssam 400104478Ssam#ifndef UBSEC_NO_RNG 401104478Ssam if (sc->sc_flags & UBS_FLAGS_RNG) { 402104478Ssam sc->sc_statmask |= BS_STAT_MCR2_DONE; 403104478Ssam 404104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 405104478Ssam &sc->sc_rng.rng_q.q_mcr, 0)) 406104478Ssam goto skip_rng; 407104478Ssam 408104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass), 409104478Ssam &sc->sc_rng.rng_q.q_ctx, 0)) { 410104478Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 411104478Ssam goto skip_rng; 412104478Ssam } 413104478Ssam 414104478Ssam if (ubsec_dma_malloc(sc, sizeof(u_int32_t) * 415104478Ssam UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) { 416104478Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 417104478Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 418104478Ssam goto skip_rng; 419104478Ssam } 420104478Ssam 421104478Ssam if (hz >= 100) 422104478Ssam sc->sc_rnghz = hz / 100; 423104478Ssam else 424104478Ssam sc->sc_rnghz = 1; 425104918Ssam /* NB: 1 means the callout runs w/o Giant locked */ 426104918Ssam callout_init(&sc->sc_rngto, 1); 427104478Ssam callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 428104478Ssamskip_rng: 429104478Ssam ; 430104478Ssam } 431104478Ssam#endif /* UBSEC_NO_RNG */ 432104478Ssam 433104478Ssam if (sc->sc_flags & UBS_FLAGS_KEY) { 434104478Ssam sc->sc_statmask |= BS_STAT_MCR2_DONE; 435104478Ssam 436104478Ssam crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0, 437104478Ssam ubsec_kprocess, sc); 438104630Ssam#if 0 439104478Ssam crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0, 440104478Ssam ubsec_kprocess, sc); 441104630Ssam#endif 442104478Ssam } 443104478Ssam return (0); 444108823Ssambad4: 445108823Ssam crypto_unregister_all(sc->sc_cid); 446108823Ssambad3: 447108823Ssam bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 448108823Ssambad2: 449108823Ssam bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 450108823Ssambad1: 451108823Ssam bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 452104478Ssambad: 453104478Ssam mtx_destroy(&sc->sc_mtx); 454104478Ssam return (ENXIO); 455104478Ssam} 456104478Ssam 457104478Ssam/* 458104478Ssam * Detach a device that successfully probed. 459104478Ssam */ 460104478Ssamstatic int 461104478Ssamubsec_detach(device_t dev) 462104478Ssam{ 463104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 464104478Ssam 465104478Ssam KASSERT(sc != NULL, ("ubsec_detach: null software carrier")); 466104478Ssam 467108823Ssam /* XXX wait/abort active ops */ 468108823Ssam 469104478Ssam UBSEC_LOCK(sc); 470104478Ssam 471104478Ssam callout_stop(&sc->sc_rngto); 472104478Ssam 473104478Ssam crypto_unregister_all(sc->sc_cid); 474104478Ssam 475108823Ssam while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 476108823Ssam struct ubsec_q *q; 477108823Ssam 478108823Ssam q = SIMPLEQ_FIRST(&sc->sc_freequeue); 479108823Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q, q_next); 480108823Ssam ubsec_dma_free(sc, &q->q_dma->d_alloc); 481108823Ssam free(q, M_DEVBUF); 482108823Ssam } 483108823Ssam#ifndef UBSEC_NO_RNG 484108823Ssam if (sc->sc_flags & UBS_FLAGS_RNG) { 485108823Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 486108823Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 487108823Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_buf); 488108823Ssam } 489108823Ssam#endif /* UBSEC_NO_RNG */ 490108823Ssam 491104478Ssam bus_generic_detach(dev); 492104478Ssam bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 493104478Ssam bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 494104478Ssam 495104478Ssam bus_dma_tag_destroy(sc->sc_dmat); 496104478Ssam bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 497104478Ssam 498104478Ssam UBSEC_UNLOCK(sc); 499104478Ssam 500104478Ssam mtx_destroy(&sc->sc_mtx); 501104478Ssam 502104478Ssam return (0); 503104478Ssam} 504104478Ssam 505104478Ssam/* 506104478Ssam * Stop all chip i/o so that the kernel's probe routines don't 507104478Ssam * get confused by errant DMAs when rebooting. 508104478Ssam */ 509104478Ssamstatic void 510104478Ssamubsec_shutdown(device_t dev) 511104478Ssam{ 512104478Ssam#ifdef notyet 513104478Ssam ubsec_stop(device_get_softc(dev)); 514104478Ssam#endif 515104478Ssam} 516104478Ssam 517104478Ssam/* 518104478Ssam * Device suspend routine. 519104478Ssam */ 520104478Ssamstatic int 521104478Ssamubsec_suspend(device_t dev) 522104478Ssam{ 523104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 524104478Ssam 525104478Ssam KASSERT(sc != NULL, ("ubsec_suspend: null software carrier")); 526104478Ssam#ifdef notyet 527104478Ssam /* XXX stop the device and save PCI settings */ 528104478Ssam#endif 529104478Ssam sc->sc_suspended = 1; 530104478Ssam 531104478Ssam return (0); 532104478Ssam} 533104478Ssam 534104478Ssamstatic int 535104478Ssamubsec_resume(device_t dev) 536104478Ssam{ 537104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 538104478Ssam 539104478Ssam KASSERT(sc != NULL, ("ubsec_resume: null software carrier")); 540104478Ssam#ifdef notyet 541104478Ssam /* XXX retore PCI settings and start the device */ 542104478Ssam#endif 543104478Ssam sc->sc_suspended = 0; 544104478Ssam return (0); 545104478Ssam} 546104478Ssam 547104478Ssam/* 548104478Ssam * UBSEC Interrupt routine 549104478Ssam */ 550104478Ssamstatic void 551104478Ssamubsec_intr(void *arg) 552104478Ssam{ 553104478Ssam struct ubsec_softc *sc = arg; 554104478Ssam volatile u_int32_t stat; 555104478Ssam struct ubsec_q *q; 556104478Ssam struct ubsec_dma *dmap; 557104478Ssam int npkts = 0, i; 558104478Ssam 559104478Ssam UBSEC_LOCK(sc); 560104478Ssam 561104478Ssam stat = READ_REG(sc, BS_STAT); 562104478Ssam stat &= sc->sc_statmask; 563104478Ssam if (stat == 0) { 564104478Ssam UBSEC_UNLOCK(sc); 565104478Ssam return; 566104478Ssam } 567104478Ssam 568104478Ssam WRITE_REG(sc, BS_STAT, stat); /* IACK */ 569104478Ssam 570104478Ssam /* 571104478Ssam * Check to see if we have any packets waiting for us 572104478Ssam */ 573104478Ssam if ((stat & BS_STAT_MCR1_DONE)) { 574104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 575104478Ssam q = SIMPLEQ_FIRST(&sc->sc_qchip); 576104478Ssam dmap = q->q_dma; 577104478Ssam 578104478Ssam if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0) 579104478Ssam break; 580104478Ssam 581104478Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next); 582104478Ssam 583104478Ssam npkts = q->q_nstacked_mcrs; 584108471Ssam sc->sc_nqchip -= 1+npkts; 585104478Ssam /* 586104478Ssam * search for further sc_qchip ubsec_q's that share 587104478Ssam * the same MCR, and complete them too, they must be 588104478Ssam * at the top. 589104478Ssam */ 590104478Ssam for (i = 0; i < npkts; i++) { 591104478Ssam if(q->q_stacked_mcr[i]) { 592104478Ssam ubsec_callback(sc, q->q_stacked_mcr[i]); 593104478Ssam ubsecstats.hst_opackets++; 594104478Ssam } else { 595104478Ssam break; 596104478Ssam } 597104478Ssam } 598104478Ssam ubsec_callback(sc, q); 599104478Ssam ubsecstats.hst_opackets++; 600104478Ssam } 601104478Ssam 602104478Ssam /* 603104478Ssam * Don't send any more packet to chip if there has been 604104478Ssam * a DMAERR. 605104478Ssam */ 606104478Ssam if (!(stat & BS_STAT_DMAERR)) 607104478Ssam ubsec_feed(sc); 608104478Ssam } 609104478Ssam 610104478Ssam /* 611104478Ssam * Check to see if we have any key setups/rng's waiting for us 612104478Ssam */ 613104478Ssam if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) && 614104478Ssam (stat & BS_STAT_MCR2_DONE)) { 615104478Ssam struct ubsec_q2 *q2; 616104478Ssam struct ubsec_mcr *mcr; 617104478Ssam 618104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) { 619104478Ssam q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 620104478Ssam 621108823Ssam ubsec_dma_sync(&q2->q_mcr, 622104478Ssam BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 623104478Ssam 624104478Ssam mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 625104478Ssam if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) { 626108823Ssam ubsec_dma_sync(&q2->q_mcr, 627104478Ssam BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 628104478Ssam break; 629104478Ssam } 630104478Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q2, q_next); 631104478Ssam ubsec_callback2(sc, q2); 632104478Ssam /* 633104478Ssam * Don't send any more packet to chip if there has been 634104478Ssam * a DMAERR. 635104478Ssam */ 636104478Ssam if (!(stat & BS_STAT_DMAERR)) 637104478Ssam ubsec_feed2(sc); 638104478Ssam } 639104478Ssam } 640104478Ssam 641104478Ssam /* 642104478Ssam * Check to see if we got any DMA Error 643104478Ssam */ 644104478Ssam if (stat & BS_STAT_DMAERR) { 645104478Ssam#ifdef UBSEC_DEBUG 646104478Ssam if (ubsec_debug) { 647104478Ssam volatile u_int32_t a = READ_REG(sc, BS_ERR); 648104478Ssam 649104478Ssam printf("dmaerr %s@%08x\n", 650104478Ssam (a & BS_ERR_READ) ? "read" : "write", 651104478Ssam a & BS_ERR_ADDR); 652104478Ssam } 653104478Ssam#endif /* UBSEC_DEBUG */ 654104478Ssam ubsecstats.hst_dmaerr++; 655104478Ssam ubsec_totalreset(sc); 656104478Ssam ubsec_feed(sc); 657104478Ssam } 658104478Ssam 659104478Ssam if (sc->sc_needwakeup) { /* XXX check high watermark */ 660104478Ssam int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ); 661104478Ssam#ifdef UBSEC_DEBUG 662104478Ssam if (ubsec_debug) 663104478Ssam device_printf(sc->sc_dev, "wakeup crypto (%x)\n", 664104478Ssam sc->sc_needwakeup); 665104478Ssam#endif /* UBSEC_DEBUG */ 666104478Ssam sc->sc_needwakeup &= ~wakeup; 667104478Ssam crypto_unblock(sc->sc_cid, wakeup); 668104478Ssam } 669104478Ssam 670104478Ssam UBSEC_UNLOCK(sc); 671104478Ssam} 672104478Ssam 673104478Ssam/* 674104478Ssam * ubsec_feed() - aggregate and post requests to chip 675104478Ssam */ 676111416Ssamstatic void 677104478Ssamubsec_feed(struct ubsec_softc *sc) 678104478Ssam{ 679104478Ssam struct ubsec_q *q, *q2; 680104478Ssam int npkts, i; 681104478Ssam void *v; 682104478Ssam u_int32_t stat; 683104478Ssam 684108471Ssam /* 685108471Ssam * Decide how many ops to combine in a single MCR. We cannot 686108471Ssam * aggregate more than UBS_MAX_AGGR because this is the number 687111416Ssam * of slots defined in the data structure. Note that 688111416Ssam * aggregation only happens if ops are marked batch'able. 689108471Ssam * Aggregating ops reduces the number of interrupts to the host 690108471Ssam * but also (potentially) increases the latency for processing 691108471Ssam * completed ops as we only get an interrupt when all aggregated 692108471Ssam * ops have completed. 693108471Ssam */ 694111416Ssam if (sc->sc_nqueue == 0) 695111416Ssam return; 696111416Ssam if (sc->sc_nqueue > 1) { 697111416Ssam npkts = 0; 698111416Ssam SIMPLEQ_FOREACH(q, &sc->sc_queue, q_next) { 699111416Ssam npkts++; 700111416Ssam if ((q->q_crp->crp_flags & CRYPTO_F_BATCH) == 0) 701111416Ssam break; 702111416Ssam } 703111416Ssam } else 704111416Ssam npkts = 1; 705111416Ssam /* 706111416Ssam * Check device status before going any further. 707111416Ssam */ 708104478Ssam if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 709108471Ssam if (stat & BS_STAT_DMAERR) { 710104478Ssam ubsec_totalreset(sc); 711104478Ssam ubsecstats.hst_dmaerr++; 712108471Ssam } else 713108471Ssam ubsecstats.hst_mcr1full++; 714111416Ssam return; 715104478Ssam } 716111416Ssam if (sc->sc_nqueue > ubsecstats.hst_maxqueue) 717111416Ssam ubsecstats.hst_maxqueue = sc->sc_nqueue; 718111416Ssam if (npkts > UBS_MAX_AGGR) 719111416Ssam npkts = UBS_MAX_AGGR; 720111416Ssam if (npkts < 2) /* special case 1 op */ 721111416Ssam goto feed1; 722104478Ssam 723111416Ssam ubsecstats.hst_totbatch += npkts-1; 724104478Ssam#ifdef UBSEC_DEBUG 725104478Ssam if (ubsec_debug) 726104478Ssam printf("merging %d records\n", npkts); 727104478Ssam#endif /* UBSEC_DEBUG */ 728104478Ssam 729104478Ssam q = SIMPLEQ_FIRST(&sc->sc_queue); 730104478Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next); 731104478Ssam --sc->sc_nqueue; 732104478Ssam 733104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE); 734104478Ssam if (q->q_dst_map != NULL) 735104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD); 736104478Ssam 737104478Ssam q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */ 738104478Ssam 739104478Ssam for (i = 0; i < q->q_nstacked_mcrs; i++) { 740104478Ssam q2 = SIMPLEQ_FIRST(&sc->sc_queue); 741104478Ssam bus_dmamap_sync(sc->sc_dmat, q2->q_src_map, 742104478Ssam BUS_DMASYNC_PREWRITE); 743104478Ssam if (q2->q_dst_map != NULL) 744104478Ssam bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map, 745104478Ssam BUS_DMASYNC_PREREAD); 746104478Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q2, q_next); 747104478Ssam --sc->sc_nqueue; 748104478Ssam 749104478Ssam v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) - 750104478Ssam sizeof(struct ubsec_mcr_add)); 751104478Ssam bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add)); 752104478Ssam q->q_stacked_mcr[i] = q2; 753104478Ssam } 754104478Ssam q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts); 755104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 756108471Ssam sc->sc_nqchip += npkts; 757108471Ssam if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 758108471Ssam ubsecstats.hst_maxqchip = sc->sc_nqchip; 759108823Ssam ubsec_dma_sync(&q->q_dma->d_alloc, 760104478Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 761104478Ssam WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 762104478Ssam offsetof(struct ubsec_dmachunk, d_mcr)); 763111416Ssam return; 764104478Ssamfeed1: 765111416Ssam q = SIMPLEQ_FIRST(&sc->sc_queue); 766104478Ssam 767111416Ssam bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE); 768111416Ssam if (q->q_dst_map != NULL) 769111416Ssam bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD); 770111416Ssam ubsec_dma_sync(&q->q_dma->d_alloc, 771111416Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 772104478Ssam 773111416Ssam WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 774111416Ssam offsetof(struct ubsec_dmachunk, d_mcr)); 775104478Ssam#ifdef UBSEC_DEBUG 776111416Ssam if (ubsec_debug) 777111416Ssam printf("feed1: q->chip %p %08x stat %08x\n", 778111416Ssam q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr), 779111416Ssam stat); 780104478Ssam#endif /* UBSEC_DEBUG */ 781111416Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next); 782111416Ssam --sc->sc_nqueue; 783111416Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 784111416Ssam sc->sc_nqchip++; 785108471Ssam if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 786108471Ssam ubsecstats.hst_maxqchip = sc->sc_nqchip; 787111416Ssam return; 788104478Ssam} 789104478Ssam 790104478Ssam/* 791104478Ssam * Allocate a new 'session' and return an encoded session id. 'sidp' 792104478Ssam * contains our registration id, and should contain an encoded session 793104478Ssam * id on successful allocation. 794104478Ssam */ 795104478Ssamstatic int 796104478Ssamubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri) 797104478Ssam{ 798104478Ssam struct cryptoini *c, *encini = NULL, *macini = NULL; 799104478Ssam struct ubsec_softc *sc = arg; 800104478Ssam struct ubsec_session *ses = NULL; 801104478Ssam MD5_CTX md5ctx; 802104478Ssam SHA1_CTX sha1ctx; 803104478Ssam int i, sesn; 804104478Ssam 805104478Ssam KASSERT(sc != NULL, ("ubsec_newsession: null softc")); 806104478Ssam if (sidp == NULL || cri == NULL || sc == NULL) 807104478Ssam return (EINVAL); 808104478Ssam 809104478Ssam for (c = cri; c != NULL; c = c->cri_next) { 810104478Ssam if (c->cri_alg == CRYPTO_MD5_HMAC || 811104478Ssam c->cri_alg == CRYPTO_SHA1_HMAC) { 812104478Ssam if (macini) 813104478Ssam return (EINVAL); 814104478Ssam macini = c; 815104478Ssam } else if (c->cri_alg == CRYPTO_DES_CBC || 816104478Ssam c->cri_alg == CRYPTO_3DES_CBC) { 817104478Ssam if (encini) 818104478Ssam return (EINVAL); 819104478Ssam encini = c; 820104478Ssam } else 821104478Ssam return (EINVAL); 822104478Ssam } 823104478Ssam if (encini == NULL && macini == NULL) 824104478Ssam return (EINVAL); 825104478Ssam 826104478Ssam if (sc->sc_sessions == NULL) { 827104478Ssam ses = sc->sc_sessions = (struct ubsec_session *)malloc( 828104478Ssam sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 829104478Ssam if (ses == NULL) 830104478Ssam return (ENOMEM); 831104478Ssam sesn = 0; 832104478Ssam sc->sc_nsessions = 1; 833104478Ssam } else { 834104478Ssam for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 835104478Ssam if (sc->sc_sessions[sesn].ses_used == 0) { 836104478Ssam ses = &sc->sc_sessions[sesn]; 837104478Ssam break; 838104478Ssam } 839104478Ssam } 840104478Ssam 841104478Ssam if (ses == NULL) { 842104478Ssam sesn = sc->sc_nsessions; 843104478Ssam ses = (struct ubsec_session *)malloc((sesn + 1) * 844104478Ssam sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 845104478Ssam if (ses == NULL) 846104478Ssam return (ENOMEM); 847104478Ssam bcopy(sc->sc_sessions, ses, sesn * 848104478Ssam sizeof(struct ubsec_session)); 849104478Ssam bzero(sc->sc_sessions, sesn * 850104478Ssam sizeof(struct ubsec_session)); 851104478Ssam free(sc->sc_sessions, M_DEVBUF); 852104478Ssam sc->sc_sessions = ses; 853104478Ssam ses = &sc->sc_sessions[sesn]; 854104478Ssam sc->sc_nsessions++; 855104478Ssam } 856104478Ssam } 857104478Ssam 858104478Ssam bzero(ses, sizeof(struct ubsec_session)); 859104478Ssam ses->ses_used = 1; 860104478Ssam if (encini) { 861104478Ssam /* get an IV, network byte order */ 862104478Ssam /* XXX may read fewer than requested */ 863104478Ssam read_random(ses->ses_iv, sizeof(ses->ses_iv)); 864104478Ssam 865104478Ssam /* Go ahead and compute key in ubsec's byte order */ 866104478Ssam if (encini->cri_alg == CRYPTO_DES_CBC) { 867104478Ssam bcopy(encini->cri_key, &ses->ses_deskey[0], 8); 868104478Ssam bcopy(encini->cri_key, &ses->ses_deskey[2], 8); 869104478Ssam bcopy(encini->cri_key, &ses->ses_deskey[4], 8); 870104478Ssam } else 871104478Ssam bcopy(encini->cri_key, ses->ses_deskey, 24); 872104478Ssam 873104478Ssam SWAP32(ses->ses_deskey[0]); 874104478Ssam SWAP32(ses->ses_deskey[1]); 875104478Ssam SWAP32(ses->ses_deskey[2]); 876104478Ssam SWAP32(ses->ses_deskey[3]); 877104478Ssam SWAP32(ses->ses_deskey[4]); 878104478Ssam SWAP32(ses->ses_deskey[5]); 879104478Ssam } 880104478Ssam 881104478Ssam if (macini) { 882104478Ssam for (i = 0; i < macini->cri_klen / 8; i++) 883104478Ssam macini->cri_key[i] ^= HMAC_IPAD_VAL; 884104478Ssam 885104478Ssam if (macini->cri_alg == CRYPTO_MD5_HMAC) { 886104478Ssam MD5Init(&md5ctx); 887104478Ssam MD5Update(&md5ctx, macini->cri_key, 888104478Ssam macini->cri_klen / 8); 889104478Ssam MD5Update(&md5ctx, hmac_ipad_buffer, 890104478Ssam HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 891104478Ssam bcopy(md5ctx.state, ses->ses_hminner, 892104478Ssam sizeof(md5ctx.state)); 893104478Ssam } else { 894104478Ssam SHA1Init(&sha1ctx); 895104478Ssam SHA1Update(&sha1ctx, macini->cri_key, 896104478Ssam macini->cri_klen / 8); 897104478Ssam SHA1Update(&sha1ctx, hmac_ipad_buffer, 898104478Ssam HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 899104478Ssam bcopy(sha1ctx.h.b32, ses->ses_hminner, 900104478Ssam sizeof(sha1ctx.h.b32)); 901104478Ssam } 902104478Ssam 903104478Ssam for (i = 0; i < macini->cri_klen / 8; i++) 904104478Ssam macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 905104478Ssam 906104478Ssam if (macini->cri_alg == CRYPTO_MD5_HMAC) { 907104478Ssam MD5Init(&md5ctx); 908104478Ssam MD5Update(&md5ctx, macini->cri_key, 909104478Ssam macini->cri_klen / 8); 910104478Ssam MD5Update(&md5ctx, hmac_opad_buffer, 911104478Ssam HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 912104478Ssam bcopy(md5ctx.state, ses->ses_hmouter, 913104478Ssam sizeof(md5ctx.state)); 914104478Ssam } else { 915104478Ssam SHA1Init(&sha1ctx); 916104478Ssam SHA1Update(&sha1ctx, macini->cri_key, 917104478Ssam macini->cri_klen / 8); 918104478Ssam SHA1Update(&sha1ctx, hmac_opad_buffer, 919104478Ssam HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 920104478Ssam bcopy(sha1ctx.h.b32, ses->ses_hmouter, 921104478Ssam sizeof(sha1ctx.h.b32)); 922104478Ssam } 923104478Ssam 924104478Ssam for (i = 0; i < macini->cri_klen / 8; i++) 925104478Ssam macini->cri_key[i] ^= HMAC_OPAD_VAL; 926104478Ssam } 927104478Ssam 928104478Ssam *sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn); 929104478Ssam return (0); 930104478Ssam} 931104478Ssam 932104478Ssam/* 933104478Ssam * Deallocate a session. 934104478Ssam */ 935104478Ssamstatic int 936104478Ssamubsec_freesession(void *arg, u_int64_t tid) 937104478Ssam{ 938104478Ssam struct ubsec_softc *sc = arg; 939104478Ssam int session; 940104478Ssam u_int32_t sid = ((u_int32_t) tid) & 0xffffffff; 941104478Ssam 942104478Ssam KASSERT(sc != NULL, ("ubsec_freesession: null softc")); 943104478Ssam if (sc == NULL) 944104478Ssam return (EINVAL); 945104478Ssam 946104478Ssam session = UBSEC_SESSION(sid); 947104478Ssam if (session >= sc->sc_nsessions) 948104478Ssam return (EINVAL); 949104478Ssam 950104478Ssam bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session])); 951104478Ssam return (0); 952104478Ssam} 953104478Ssam 954104478Ssamstatic void 955104478Ssamubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error) 956104478Ssam{ 957104478Ssam struct ubsec_operand *op = arg; 958104478Ssam 959104478Ssam KASSERT(nsegs <= UBS_MAX_SCATTER, 960104478Ssam ("Too many DMA segments returned when mapping operand")); 961104478Ssam#ifdef UBSEC_DEBUG 962104478Ssam if (ubsec_debug) 963104478Ssam printf("ubsec_op_cb: mapsize %u nsegs %d\n", 964104478Ssam (u_int) mapsize, nsegs); 965104478Ssam#endif 966104478Ssam op->mapsize = mapsize; 967104478Ssam op->nsegs = nsegs; 968104478Ssam bcopy(seg, op->segs, nsegs * sizeof (seg[0])); 969104478Ssam} 970104478Ssam 971104478Ssamstatic int 972104478Ssamubsec_process(void *arg, struct cryptop *crp, int hint) 973104478Ssam{ 974104478Ssam struct ubsec_q *q = NULL; 975104478Ssam int err = 0, i, j, nicealign; 976104478Ssam struct ubsec_softc *sc = arg; 977104478Ssam struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 978104478Ssam int encoffset = 0, macoffset = 0, cpskip, cpoffset; 979104478Ssam int sskip, dskip, stheend, dtheend; 980104478Ssam int16_t coffset; 981104478Ssam struct ubsec_session *ses; 982104478Ssam struct ubsec_pktctx ctx; 983104478Ssam struct ubsec_dma *dmap = NULL; 984104478Ssam 985104478Ssam if (crp == NULL || crp->crp_callback == NULL || sc == NULL) { 986104478Ssam ubsecstats.hst_invalid++; 987104478Ssam return (EINVAL); 988104478Ssam } 989104478Ssam if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) { 990108471Ssam ubsecstats.hst_badsession++; 991104478Ssam return (EINVAL); 992104478Ssam } 993104478Ssam 994104478Ssam UBSEC_LOCK(sc); 995104478Ssam 996104478Ssam if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 997104478Ssam ubsecstats.hst_queuefull++; 998104478Ssam sc->sc_needwakeup |= CRYPTO_SYMQ; 999104478Ssam UBSEC_UNLOCK(sc); 1000104478Ssam return (ERESTART); 1001104478Ssam } 1002104478Ssam q = SIMPLEQ_FIRST(&sc->sc_freequeue); 1003104478Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q, q_next); 1004104478Ssam UBSEC_UNLOCK(sc); 1005104478Ssam 1006104478Ssam dmap = q->q_dma; /* Save dma pointer */ 1007104478Ssam bzero(q, sizeof(struct ubsec_q)); 1008104478Ssam bzero(&ctx, sizeof(ctx)); 1009104478Ssam 1010104478Ssam q->q_sesn = UBSEC_SESSION(crp->crp_sid); 1011104478Ssam q->q_dma = dmap; 1012104478Ssam ses = &sc->sc_sessions[q->q_sesn]; 1013104478Ssam 1014104478Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) { 1015104478Ssam q->q_src_m = (struct mbuf *)crp->crp_buf; 1016104478Ssam q->q_dst_m = (struct mbuf *)crp->crp_buf; 1017104478Ssam } else if (crp->crp_flags & CRYPTO_F_IOV) { 1018104478Ssam q->q_src_io = (struct uio *)crp->crp_buf; 1019104478Ssam q->q_dst_io = (struct uio *)crp->crp_buf; 1020104478Ssam } else { 1021108471Ssam ubsecstats.hst_badflags++; 1022104478Ssam err = EINVAL; 1023104478Ssam goto errout; /* XXX we don't handle contiguous blocks! */ 1024104478Ssam } 1025104478Ssam 1026104478Ssam bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr)); 1027104478Ssam 1028104478Ssam dmap->d_dma->d_mcr.mcr_pkts = htole16(1); 1029104478Ssam dmap->d_dma->d_mcr.mcr_flags = 0; 1030104478Ssam q->q_crp = crp; 1031104478Ssam 1032104478Ssam crd1 = crp->crp_desc; 1033104478Ssam if (crd1 == NULL) { 1034108471Ssam ubsecstats.hst_nodesc++; 1035104478Ssam err = EINVAL; 1036104478Ssam goto errout; 1037104478Ssam } 1038104478Ssam crd2 = crd1->crd_next; 1039104478Ssam 1040104478Ssam if (crd2 == NULL) { 1041104478Ssam if (crd1->crd_alg == CRYPTO_MD5_HMAC || 1042104478Ssam crd1->crd_alg == CRYPTO_SHA1_HMAC) { 1043104478Ssam maccrd = crd1; 1044104478Ssam enccrd = NULL; 1045104478Ssam } else if (crd1->crd_alg == CRYPTO_DES_CBC || 1046104478Ssam crd1->crd_alg == CRYPTO_3DES_CBC) { 1047104478Ssam maccrd = NULL; 1048104478Ssam enccrd = crd1; 1049104478Ssam } else { 1050108471Ssam ubsecstats.hst_badalg++; 1051104478Ssam err = EINVAL; 1052104478Ssam goto errout; 1053104478Ssam } 1054104478Ssam } else { 1055104478Ssam if ((crd1->crd_alg == CRYPTO_MD5_HMAC || 1056104478Ssam crd1->crd_alg == CRYPTO_SHA1_HMAC) && 1057104478Ssam (crd2->crd_alg == CRYPTO_DES_CBC || 1058104478Ssam crd2->crd_alg == CRYPTO_3DES_CBC) && 1059104478Ssam ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 1060104478Ssam maccrd = crd1; 1061104478Ssam enccrd = crd2; 1062104478Ssam } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 1063104478Ssam crd1->crd_alg == CRYPTO_3DES_CBC) && 1064104478Ssam (crd2->crd_alg == CRYPTO_MD5_HMAC || 1065104478Ssam crd2->crd_alg == CRYPTO_SHA1_HMAC) && 1066104478Ssam (crd1->crd_flags & CRD_F_ENCRYPT)) { 1067104478Ssam enccrd = crd1; 1068104478Ssam maccrd = crd2; 1069104478Ssam } else { 1070104478Ssam /* 1071104478Ssam * We cannot order the ubsec as requested 1072104478Ssam */ 1073108471Ssam ubsecstats.hst_badalg++; 1074104478Ssam err = EINVAL; 1075104478Ssam goto errout; 1076104478Ssam } 1077104478Ssam } 1078104478Ssam 1079104478Ssam if (enccrd) { 1080104478Ssam encoffset = enccrd->crd_skip; 1081104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES); 1082104478Ssam 1083104478Ssam if (enccrd->crd_flags & CRD_F_ENCRYPT) { 1084104478Ssam q->q_flags |= UBSEC_QFLAGS_COPYOUTIV; 1085104478Ssam 1086104478Ssam if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1087104478Ssam bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1088104478Ssam else { 1089104478Ssam ctx.pc_iv[0] = ses->ses_iv[0]; 1090104478Ssam ctx.pc_iv[1] = ses->ses_iv[1]; 1091104478Ssam } 1092104478Ssam 1093104478Ssam if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { 1094104478Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) 1095104478Ssam m_copyback(q->q_src_m, 1096104478Ssam enccrd->crd_inject, 1097104478Ssam 8, (caddr_t)ctx.pc_iv); 1098104478Ssam else if (crp->crp_flags & CRYPTO_F_IOV) 1099104478Ssam cuio_copyback(q->q_src_io, 1100104478Ssam enccrd->crd_inject, 1101104478Ssam 8, (caddr_t)ctx.pc_iv); 1102104478Ssam } 1103104478Ssam } else { 1104104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND); 1105104478Ssam 1106104478Ssam if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1107104478Ssam bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1108104478Ssam else if (crp->crp_flags & CRYPTO_F_IMBUF) 1109104478Ssam m_copydata(q->q_src_m, enccrd->crd_inject, 1110104478Ssam 8, (caddr_t)ctx.pc_iv); 1111104478Ssam else if (crp->crp_flags & CRYPTO_F_IOV) 1112104478Ssam cuio_copydata(q->q_src_io, 1113104478Ssam enccrd->crd_inject, 8, 1114104478Ssam (caddr_t)ctx.pc_iv); 1115104478Ssam } 1116104478Ssam 1117104478Ssam ctx.pc_deskey[0] = ses->ses_deskey[0]; 1118104478Ssam ctx.pc_deskey[1] = ses->ses_deskey[1]; 1119104478Ssam ctx.pc_deskey[2] = ses->ses_deskey[2]; 1120104478Ssam ctx.pc_deskey[3] = ses->ses_deskey[3]; 1121104478Ssam ctx.pc_deskey[4] = ses->ses_deskey[4]; 1122104478Ssam ctx.pc_deskey[5] = ses->ses_deskey[5]; 1123104478Ssam SWAP32(ctx.pc_iv[0]); 1124104478Ssam SWAP32(ctx.pc_iv[1]); 1125104478Ssam } 1126104478Ssam 1127104478Ssam if (maccrd) { 1128104478Ssam macoffset = maccrd->crd_skip; 1129104478Ssam 1130104478Ssam if (maccrd->crd_alg == CRYPTO_MD5_HMAC) 1131104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5); 1132104478Ssam else 1133104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1); 1134104478Ssam 1135104478Ssam for (i = 0; i < 5; i++) { 1136104478Ssam ctx.pc_hminner[i] = ses->ses_hminner[i]; 1137104478Ssam ctx.pc_hmouter[i] = ses->ses_hmouter[i]; 1138104478Ssam 1139104478Ssam HTOLE32(ctx.pc_hminner[i]); 1140104478Ssam HTOLE32(ctx.pc_hmouter[i]); 1141104478Ssam } 1142104478Ssam } 1143104478Ssam 1144104478Ssam if (enccrd && maccrd) { 1145104478Ssam /* 1146104478Ssam * ubsec cannot handle packets where the end of encryption 1147104478Ssam * and authentication are not the same, or where the 1148104478Ssam * encrypted part begins before the authenticated part. 1149104478Ssam */ 1150104478Ssam if ((encoffset + enccrd->crd_len) != 1151104478Ssam (macoffset + maccrd->crd_len)) { 1152104478Ssam ubsecstats.hst_lenmismatch++; 1153104478Ssam err = EINVAL; 1154104478Ssam goto errout; 1155104478Ssam } 1156104478Ssam if (enccrd->crd_skip < maccrd->crd_skip) { 1157104478Ssam ubsecstats.hst_skipmismatch++; 1158104478Ssam err = EINVAL; 1159104478Ssam goto errout; 1160104478Ssam } 1161104478Ssam sskip = maccrd->crd_skip; 1162104478Ssam cpskip = dskip = enccrd->crd_skip; 1163104478Ssam stheend = maccrd->crd_len; 1164104478Ssam dtheend = enccrd->crd_len; 1165104478Ssam coffset = enccrd->crd_skip - maccrd->crd_skip; 1166104478Ssam cpoffset = cpskip + dtheend; 1167104478Ssam#ifdef UBSEC_DEBUG 1168104478Ssam if (ubsec_debug) { 1169104478Ssam printf("mac: skip %d, len %d, inject %d\n", 1170104478Ssam maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject); 1171104478Ssam printf("enc: skip %d, len %d, inject %d\n", 1172104478Ssam enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject); 1173104478Ssam printf("src: skip %d, len %d\n", sskip, stheend); 1174104478Ssam printf("dst: skip %d, len %d\n", dskip, dtheend); 1175104478Ssam printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n", 1176104478Ssam coffset, stheend, cpskip, cpoffset); 1177104478Ssam } 1178104478Ssam#endif 1179104478Ssam } else { 1180104478Ssam cpskip = dskip = sskip = macoffset + encoffset; 1181104478Ssam dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len; 1182104478Ssam cpoffset = cpskip + dtheend; 1183104478Ssam coffset = 0; 1184104478Ssam } 1185104478Ssam ctx.pc_offset = htole16(coffset >> 2); 1186104478Ssam 1187104478Ssam if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) { 1188104478Ssam ubsecstats.hst_nomap++; 1189104478Ssam err = ENOMEM; 1190104478Ssam goto errout; 1191104478Ssam } 1192104478Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) { 1193104478Ssam if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map, 1194104478Ssam q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1195104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1196104478Ssam q->q_src_map = NULL; 1197104478Ssam ubsecstats.hst_noload++; 1198104478Ssam err = ENOMEM; 1199104478Ssam goto errout; 1200104478Ssam } 1201104478Ssam } else if (crp->crp_flags & CRYPTO_F_IOV) { 1202104478Ssam if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map, 1203104478Ssam q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1204104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1205104478Ssam q->q_src_map = NULL; 1206104478Ssam ubsecstats.hst_noload++; 1207104478Ssam err = ENOMEM; 1208104478Ssam goto errout; 1209104478Ssam } 1210104478Ssam } 1211104478Ssam nicealign = ubsec_dmamap_aligned(&q->q_src); 1212104478Ssam 1213104478Ssam dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend); 1214104478Ssam 1215104478Ssam#ifdef UBSEC_DEBUG 1216104478Ssam if (ubsec_debug) 1217104478Ssam printf("src skip: %d nicealign: %u\n", sskip, nicealign); 1218104478Ssam#endif 1219104478Ssam for (i = j = 0; i < q->q_src_nsegs; i++) { 1220104478Ssam struct ubsec_pktbuf *pb; 1221104478Ssam bus_size_t packl = q->q_src_segs[i].ds_len; 1222104478Ssam bus_addr_t packp = q->q_src_segs[i].ds_addr; 1223104478Ssam 1224104478Ssam if (sskip >= packl) { 1225104478Ssam sskip -= packl; 1226104478Ssam continue; 1227104478Ssam } 1228104478Ssam 1229104478Ssam packl -= sskip; 1230104478Ssam packp += sskip; 1231104478Ssam sskip = 0; 1232104478Ssam 1233104478Ssam if (packl > 0xfffc) { 1234104478Ssam err = EIO; 1235104478Ssam goto errout; 1236104478Ssam } 1237104478Ssam 1238104478Ssam if (j == 0) 1239104478Ssam pb = &dmap->d_dma->d_mcr.mcr_ipktbuf; 1240104478Ssam else 1241104478Ssam pb = &dmap->d_dma->d_sbuf[j - 1]; 1242104478Ssam 1243104478Ssam pb->pb_addr = htole32(packp); 1244104478Ssam 1245104478Ssam if (stheend) { 1246104478Ssam if (packl > stheend) { 1247104478Ssam pb->pb_len = htole32(stheend); 1248104478Ssam stheend = 0; 1249104478Ssam } else { 1250104478Ssam pb->pb_len = htole32(packl); 1251104478Ssam stheend -= packl; 1252104478Ssam } 1253104478Ssam } else 1254104478Ssam pb->pb_len = htole32(packl); 1255104478Ssam 1256104478Ssam if ((i + 1) == q->q_src_nsegs) 1257104478Ssam pb->pb_next = 0; 1258104478Ssam else 1259104478Ssam pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1260104478Ssam offsetof(struct ubsec_dmachunk, d_sbuf[j])); 1261104478Ssam j++; 1262104478Ssam } 1263104478Ssam 1264104478Ssam if (enccrd == NULL && maccrd != NULL) { 1265104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0; 1266104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0; 1267104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr + 1268104478Ssam offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1269104478Ssam#ifdef UBSEC_DEBUG 1270104478Ssam if (ubsec_debug) 1271104478Ssam printf("opkt: %x %x %x\n", 1272104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr, 1273104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_len, 1274104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_next); 1275104478Ssam#endif 1276104478Ssam } else { 1277104478Ssam if (crp->crp_flags & CRYPTO_F_IOV) { 1278104478Ssam if (!nicealign) { 1279104478Ssam ubsecstats.hst_iovmisaligned++; 1280104478Ssam err = EINVAL; 1281104478Ssam goto errout; 1282104478Ssam } 1283104478Ssam if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, 1284104478Ssam &q->q_dst_map)) { 1285104478Ssam ubsecstats.hst_nomap++; 1286104478Ssam err = ENOMEM; 1287104478Ssam goto errout; 1288104478Ssam } 1289104478Ssam if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map, 1290104478Ssam q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) { 1291104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1292104478Ssam q->q_dst_map = NULL; 1293104478Ssam ubsecstats.hst_noload++; 1294104478Ssam err = ENOMEM; 1295104478Ssam goto errout; 1296104478Ssam } 1297104478Ssam } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1298104478Ssam if (nicealign) { 1299104478Ssam q->q_dst = q->q_src; 1300104478Ssam } else { 1301104478Ssam int totlen, len; 1302104478Ssam struct mbuf *m, *top, **mp; 1303104478Ssam 1304104478Ssam ubsecstats.hst_unaligned++; 1305104478Ssam totlen = q->q_src_mapsize; 1306104478Ssam if (q->q_src_m->m_flags & M_PKTHDR) { 1307104478Ssam len = MHLEN; 1308111119Simp MGETHDR(m, M_DONTWAIT, MT_DATA); 1309111119Simp if (m && !m_dup_pkthdr(m, q->q_src_m, M_DONTWAIT)) { 1310108466Ssam m_free(m); 1311108466Ssam m = NULL; 1312108466Ssam } 1313104478Ssam } else { 1314104478Ssam len = MLEN; 1315111119Simp MGET(m, M_DONTWAIT, MT_DATA); 1316104478Ssam } 1317104478Ssam if (m == NULL) { 1318104478Ssam ubsecstats.hst_nombuf++; 1319104478Ssam err = sc->sc_nqueue ? ERESTART : ENOMEM; 1320104478Ssam goto errout; 1321104478Ssam } 1322104478Ssam if (totlen >= MINCLSIZE) { 1323111119Simp MCLGET(m, M_DONTWAIT); 1324104478Ssam if ((m->m_flags & M_EXT) == 0) { 1325104478Ssam m_free(m); 1326104478Ssam ubsecstats.hst_nomcl++; 1327104478Ssam err = sc->sc_nqueue ? ERESTART : ENOMEM; 1328104478Ssam goto errout; 1329104478Ssam } 1330104478Ssam len = MCLBYTES; 1331104478Ssam } 1332104478Ssam m->m_len = len; 1333104478Ssam top = NULL; 1334104478Ssam mp = ⊤ 1335104478Ssam 1336104478Ssam while (totlen > 0) { 1337104478Ssam if (top) { 1338111119Simp MGET(m, M_DONTWAIT, MT_DATA); 1339104478Ssam if (m == NULL) { 1340104478Ssam m_freem(top); 1341104478Ssam ubsecstats.hst_nombuf++; 1342104478Ssam err = sc->sc_nqueue ? ERESTART : ENOMEM; 1343104478Ssam goto errout; 1344104478Ssam } 1345104478Ssam len = MLEN; 1346104478Ssam } 1347104478Ssam if (top && totlen >= MINCLSIZE) { 1348111119Simp MCLGET(m, M_DONTWAIT); 1349104478Ssam if ((m->m_flags & M_EXT) == 0) { 1350104478Ssam *mp = m; 1351104478Ssam m_freem(top); 1352104478Ssam ubsecstats.hst_nomcl++; 1353104478Ssam err = sc->sc_nqueue ? ERESTART : ENOMEM; 1354104478Ssam goto errout; 1355104478Ssam } 1356104478Ssam len = MCLBYTES; 1357104478Ssam } 1358104478Ssam m->m_len = len = min(totlen, len); 1359104478Ssam totlen -= len; 1360104478Ssam *mp = m; 1361104478Ssam mp = &m->m_next; 1362104478Ssam } 1363104478Ssam q->q_dst_m = top; 1364104478Ssam ubsec_mcopy(q->q_src_m, q->q_dst_m, 1365104478Ssam cpskip, cpoffset); 1366104478Ssam if (bus_dmamap_create(sc->sc_dmat, 1367104478Ssam BUS_DMA_NOWAIT, &q->q_dst_map) != 0) { 1368104478Ssam ubsecstats.hst_nomap++; 1369104478Ssam err = ENOMEM; 1370104478Ssam goto errout; 1371104478Ssam } 1372104478Ssam if (bus_dmamap_load_mbuf(sc->sc_dmat, 1373104478Ssam q->q_dst_map, q->q_dst_m, 1374104478Ssam ubsec_op_cb, &q->q_dst, 1375104478Ssam BUS_DMA_NOWAIT) != 0) { 1376104478Ssam bus_dmamap_destroy(sc->sc_dmat, 1377104478Ssam q->q_dst_map); 1378104478Ssam q->q_dst_map = NULL; 1379104478Ssam ubsecstats.hst_noload++; 1380104478Ssam err = ENOMEM; 1381104478Ssam goto errout; 1382104478Ssam } 1383104478Ssam } 1384104478Ssam } else { 1385108471Ssam ubsecstats.hst_badflags++; 1386104478Ssam err = EINVAL; 1387104478Ssam goto errout; 1388104478Ssam } 1389104478Ssam 1390104478Ssam#ifdef UBSEC_DEBUG 1391104478Ssam if (ubsec_debug) 1392104478Ssam printf("dst skip: %d\n", dskip); 1393104478Ssam#endif 1394104478Ssam for (i = j = 0; i < q->q_dst_nsegs; i++) { 1395104478Ssam struct ubsec_pktbuf *pb; 1396104478Ssam bus_size_t packl = q->q_dst_segs[i].ds_len; 1397104478Ssam bus_addr_t packp = q->q_dst_segs[i].ds_addr; 1398104478Ssam 1399104478Ssam if (dskip >= packl) { 1400104478Ssam dskip -= packl; 1401104478Ssam continue; 1402104478Ssam } 1403104478Ssam 1404104478Ssam packl -= dskip; 1405104478Ssam packp += dskip; 1406104478Ssam dskip = 0; 1407104478Ssam 1408104478Ssam if (packl > 0xfffc) { 1409104478Ssam err = EIO; 1410104478Ssam goto errout; 1411104478Ssam } 1412104478Ssam 1413104478Ssam if (j == 0) 1414104478Ssam pb = &dmap->d_dma->d_mcr.mcr_opktbuf; 1415104478Ssam else 1416104478Ssam pb = &dmap->d_dma->d_dbuf[j - 1]; 1417104478Ssam 1418104478Ssam pb->pb_addr = htole32(packp); 1419104478Ssam 1420104478Ssam if (dtheend) { 1421104478Ssam if (packl > dtheend) { 1422104478Ssam pb->pb_len = htole32(dtheend); 1423104478Ssam dtheend = 0; 1424104478Ssam } else { 1425104478Ssam pb->pb_len = htole32(packl); 1426104478Ssam dtheend -= packl; 1427104478Ssam } 1428104478Ssam } else 1429104478Ssam pb->pb_len = htole32(packl); 1430104478Ssam 1431104478Ssam if ((i + 1) == q->q_dst_nsegs) { 1432104478Ssam if (maccrd) 1433104478Ssam pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1434104478Ssam offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1435104478Ssam else 1436104478Ssam pb->pb_next = 0; 1437104478Ssam } else 1438104478Ssam pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1439104478Ssam offsetof(struct ubsec_dmachunk, d_dbuf[j])); 1440104478Ssam j++; 1441104478Ssam } 1442104478Ssam } 1443104478Ssam 1444104478Ssam dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr + 1445104478Ssam offsetof(struct ubsec_dmachunk, d_ctx)); 1446104478Ssam 1447104478Ssam if (sc->sc_flags & UBS_FLAGS_LONGCTX) { 1448104478Ssam struct ubsec_pktctx_long *ctxl; 1449104478Ssam 1450104478Ssam ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr + 1451104478Ssam offsetof(struct ubsec_dmachunk, d_ctx)); 1452104478Ssam 1453104478Ssam /* transform small context into long context */ 1454104478Ssam ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long)); 1455104478Ssam ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC); 1456104478Ssam ctxl->pc_flags = ctx.pc_flags; 1457104478Ssam ctxl->pc_offset = ctx.pc_offset; 1458104478Ssam for (i = 0; i < 6; i++) 1459104478Ssam ctxl->pc_deskey[i] = ctx.pc_deskey[i]; 1460104478Ssam for (i = 0; i < 5; i++) 1461104478Ssam ctxl->pc_hminner[i] = ctx.pc_hminner[i]; 1462104478Ssam for (i = 0; i < 5; i++) 1463104478Ssam ctxl->pc_hmouter[i] = ctx.pc_hmouter[i]; 1464104478Ssam ctxl->pc_iv[0] = ctx.pc_iv[0]; 1465104478Ssam ctxl->pc_iv[1] = ctx.pc_iv[1]; 1466104478Ssam } else 1467104478Ssam bcopy(&ctx, dmap->d_alloc.dma_vaddr + 1468104478Ssam offsetof(struct ubsec_dmachunk, d_ctx), 1469104478Ssam sizeof(struct ubsec_pktctx)); 1470104478Ssam 1471104478Ssam UBSEC_LOCK(sc); 1472104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next); 1473104478Ssam sc->sc_nqueue++; 1474104478Ssam ubsecstats.hst_ipackets++; 1475104478Ssam ubsecstats.hst_ibytes += dmap->d_alloc.dma_size; 1476111416Ssam if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= UBS_MAX_AGGR) 1477104478Ssam ubsec_feed(sc); 1478104478Ssam UBSEC_UNLOCK(sc); 1479104478Ssam return (0); 1480104478Ssam 1481104478Ssamerrout: 1482104478Ssam if (q != NULL) { 1483104478Ssam if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 1484104478Ssam m_freem(q->q_dst_m); 1485104478Ssam 1486104478Ssam if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1487104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1488104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1489104478Ssam } 1490104478Ssam if (q->q_src_map != NULL) { 1491104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1492104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1493104478Ssam } 1494104478Ssam 1495104478Ssam UBSEC_LOCK(sc); 1496104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1497104478Ssam UBSEC_UNLOCK(sc); 1498104478Ssam } 1499104478Ssam if (err != ERESTART) { 1500104478Ssam crp->crp_etype = err; 1501104478Ssam crypto_done(crp); 1502104478Ssam } else { 1503104478Ssam sc->sc_needwakeup |= CRYPTO_SYMQ; 1504104478Ssam } 1505104478Ssam return (err); 1506104478Ssam} 1507104478Ssam 1508104478Ssamstatic void 1509104478Ssamubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q) 1510104478Ssam{ 1511104478Ssam struct cryptop *crp = (struct cryptop *)q->q_crp; 1512104478Ssam struct cryptodesc *crd; 1513104478Ssam struct ubsec_dma *dmap = q->q_dma; 1514104478Ssam 1515108823Ssam ubsec_dma_sync(&dmap->d_alloc, 1516104478Ssam BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1517104478Ssam if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1518104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 1519104478Ssam BUS_DMASYNC_POSTREAD); 1520104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1521104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1522104478Ssam } 1523104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE); 1524104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1525104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1526104478Ssam 1527104478Ssam if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) { 1528104478Ssam m_freem(q->q_src_m); 1529104478Ssam crp->crp_buf = (caddr_t)q->q_dst_m; 1530104478Ssam } 1531104478Ssam ubsecstats.hst_obytes += ((struct mbuf *)crp->crp_buf)->m_len; 1532104478Ssam 1533104478Ssam /* copy out IV for future use */ 1534104478Ssam if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) { 1535104478Ssam for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1536104478Ssam if (crd->crd_alg != CRYPTO_DES_CBC && 1537104478Ssam crd->crd_alg != CRYPTO_3DES_CBC) 1538104478Ssam continue; 1539104478Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) 1540104478Ssam m_copydata((struct mbuf *)crp->crp_buf, 1541104478Ssam crd->crd_skip + crd->crd_len - 8, 8, 1542104478Ssam (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv); 1543104478Ssam else if (crp->crp_flags & CRYPTO_F_IOV) { 1544104478Ssam cuio_copydata((struct uio *)crp->crp_buf, 1545104478Ssam crd->crd_skip + crd->crd_len - 8, 8, 1546104478Ssam (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv); 1547104478Ssam } 1548104478Ssam break; 1549104478Ssam } 1550104478Ssam } 1551104478Ssam 1552104478Ssam for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1553104478Ssam if (crd->crd_alg != CRYPTO_MD5_HMAC && 1554104478Ssam crd->crd_alg != CRYPTO_SHA1_HMAC) 1555104478Ssam continue; 1556104478Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) 1557104478Ssam m_copyback((struct mbuf *)crp->crp_buf, 1558104478Ssam crd->crd_inject, 12, 1559104478Ssam (caddr_t)dmap->d_dma->d_macbuf); 1560104478Ssam else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac) 1561104478Ssam bcopy((caddr_t)dmap->d_dma->d_macbuf, 1562104478Ssam crp->crp_mac, 12); 1563104478Ssam break; 1564104478Ssam } 1565104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1566104478Ssam crypto_done(crp); 1567104478Ssam} 1568104478Ssam 1569104478Ssamstatic void 1570104478Ssamubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset) 1571104478Ssam{ 1572104478Ssam int i, j, dlen, slen; 1573104478Ssam caddr_t dptr, sptr; 1574104478Ssam 1575104478Ssam j = 0; 1576104478Ssam sptr = srcm->m_data; 1577104478Ssam slen = srcm->m_len; 1578104478Ssam dptr = dstm->m_data; 1579104478Ssam dlen = dstm->m_len; 1580104478Ssam 1581104478Ssam while (1) { 1582104478Ssam for (i = 0; i < min(slen, dlen); i++) { 1583104478Ssam if (j < hoffset || j >= toffset) 1584104478Ssam *dptr++ = *sptr++; 1585104478Ssam slen--; 1586104478Ssam dlen--; 1587104478Ssam j++; 1588104478Ssam } 1589104478Ssam if (slen == 0) { 1590104478Ssam srcm = srcm->m_next; 1591104478Ssam if (srcm == NULL) 1592104478Ssam return; 1593104478Ssam sptr = srcm->m_data; 1594104478Ssam slen = srcm->m_len; 1595104478Ssam } 1596104478Ssam if (dlen == 0) { 1597104478Ssam dstm = dstm->m_next; 1598104478Ssam if (dstm == NULL) 1599104478Ssam return; 1600104478Ssam dptr = dstm->m_data; 1601104478Ssam dlen = dstm->m_len; 1602104478Ssam } 1603104478Ssam } 1604104478Ssam} 1605104478Ssam 1606104478Ssam/* 1607104478Ssam * feed the key generator, must be called at splimp() or higher. 1608104478Ssam */ 1609104478Ssamstatic int 1610104478Ssamubsec_feed2(struct ubsec_softc *sc) 1611104478Ssam{ 1612104478Ssam struct ubsec_q2 *q; 1613104478Ssam 1614104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) { 1615104478Ssam if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL) 1616104478Ssam break; 1617104478Ssam q = SIMPLEQ_FIRST(&sc->sc_queue2); 1618104478Ssam 1619108823Ssam ubsec_dma_sync(&q->q_mcr, 1620104478Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1621108823Ssam ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_PREWRITE); 1622104478Ssam 1623104478Ssam WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr); 1624104478Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q, q_next); 1625104478Ssam --sc->sc_nqueue2; 1626104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next); 1627104478Ssam } 1628104478Ssam return (0); 1629104478Ssam} 1630104478Ssam 1631104478Ssam/* 1632104478Ssam * Callback for handling random numbers 1633104478Ssam */ 1634104478Ssamstatic void 1635104478Ssamubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q) 1636104478Ssam{ 1637104478Ssam struct cryptkop *krp; 1638104478Ssam struct ubsec_ctx_keyop *ctx; 1639104478Ssam 1640104478Ssam ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr; 1641108823Ssam ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_POSTWRITE); 1642104478Ssam 1643104478Ssam switch (q->q_type) { 1644104478Ssam#ifndef UBSEC_NO_RNG 1645104478Ssam case UBS_CTXOP_RNGBYPASS: { 1646104478Ssam struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q; 1647104478Ssam 1648108823Ssam ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_POSTREAD); 1649104478Ssam random_harvest(rng->rng_buf.dma_vaddr, 1650104478Ssam UBSEC_RNG_BUFSIZ*sizeof (u_int32_t), 1651104478Ssam UBSEC_RNG_BUFSIZ*sizeof (u_int32_t)*NBBY, 0, 1652104478Ssam RANDOM_PURE); 1653104478Ssam rng->rng_used = 0; 1654104478Ssam callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1655104478Ssam break; 1656104478Ssam } 1657104478Ssam#endif 1658104478Ssam case UBS_CTXOP_MODEXP: { 1659104478Ssam struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 1660104478Ssam u_int rlen, clen; 1661104478Ssam 1662104478Ssam krp = me->me_krp; 1663104478Ssam rlen = (me->me_modbits + 7) / 8; 1664104630Ssam clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8; 1665104478Ssam 1666108823Ssam ubsec_dma_sync(&me->me_M, BUS_DMASYNC_POSTWRITE); 1667108823Ssam ubsec_dma_sync(&me->me_E, BUS_DMASYNC_POSTWRITE); 1668108823Ssam ubsec_dma_sync(&me->me_C, BUS_DMASYNC_POSTREAD); 1669108823Ssam ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_POSTWRITE); 1670104478Ssam 1671104478Ssam if (clen < rlen) 1672104478Ssam krp->krp_status = E2BIG; 1673104630Ssam else { 1674104630Ssam if (sc->sc_flags & UBS_FLAGS_HWNORM) { 1675104630Ssam bzero(krp->krp_param[krp->krp_iparams].crp_p, 1676104630Ssam (krp->krp_param[krp->krp_iparams].crp_nbits 1677104630Ssam + 7) / 8); 1678104630Ssam bcopy(me->me_C.dma_vaddr, 1679104630Ssam krp->krp_param[krp->krp_iparams].crp_p, 1680104630Ssam (me->me_modbits + 7) / 8); 1681104630Ssam } else 1682104630Ssam ubsec_kshift_l(me->me_shiftbits, 1683104630Ssam me->me_C.dma_vaddr, me->me_normbits, 1684104630Ssam krp->krp_param[krp->krp_iparams].crp_p, 1685104630Ssam krp->krp_param[krp->krp_iparams].crp_nbits); 1686104630Ssam } 1687104478Ssam 1688104478Ssam crypto_kdone(krp); 1689104478Ssam 1690104478Ssam /* bzero all potentially sensitive data */ 1691104478Ssam bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 1692104478Ssam bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 1693104478Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 1694104478Ssam bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 1695104478Ssam 1696104478Ssam /* Can't free here, so put us on the free list. */ 1697104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next); 1698104478Ssam break; 1699104478Ssam } 1700104478Ssam case UBS_CTXOP_RSAPRIV: { 1701104478Ssam struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 1702104478Ssam u_int len; 1703104478Ssam 1704104478Ssam krp = rp->rpr_krp; 1705108823Ssam ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_POSTWRITE); 1706108823Ssam ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_POSTREAD); 1707104478Ssam 1708104478Ssam len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8; 1709104478Ssam bcopy(rp->rpr_msgout.dma_vaddr, 1710104478Ssam krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len); 1711104478Ssam 1712104478Ssam crypto_kdone(krp); 1713104478Ssam 1714104478Ssam bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 1715104478Ssam bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 1716104478Ssam bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size); 1717104478Ssam 1718104478Ssam /* Can't free here, so put us on the free list. */ 1719104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next); 1720104478Ssam break; 1721104478Ssam } 1722104478Ssam default: 1723104478Ssam device_printf(sc->sc_dev, "unknown ctx op: %x\n", 1724104478Ssam letoh16(ctx->ctx_op)); 1725104478Ssam break; 1726104478Ssam } 1727104478Ssam} 1728104478Ssam 1729104478Ssam#ifndef UBSEC_NO_RNG 1730104478Ssamstatic void 1731104478Ssamubsec_rng(void *vsc) 1732104478Ssam{ 1733104478Ssam struct ubsec_softc *sc = vsc; 1734104478Ssam struct ubsec_q2_rng *rng = &sc->sc_rng; 1735104478Ssam struct ubsec_mcr *mcr; 1736104478Ssam struct ubsec_ctx_rngbypass *ctx; 1737104478Ssam 1738104478Ssam UBSEC_LOCK(sc); 1739104478Ssam if (rng->rng_used) { 1740104478Ssam UBSEC_UNLOCK(sc); 1741104478Ssam return; 1742104478Ssam } 1743104478Ssam sc->sc_nqueue2++; 1744104478Ssam if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE) 1745104478Ssam goto out; 1746104478Ssam 1747104478Ssam mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr; 1748104478Ssam ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr; 1749104478Ssam 1750104478Ssam mcr->mcr_pkts = htole16(1); 1751104478Ssam mcr->mcr_flags = 0; 1752104478Ssam mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr); 1753104478Ssam mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0; 1754104478Ssam mcr->mcr_ipktbuf.pb_len = 0; 1755104478Ssam mcr->mcr_reserved = mcr->mcr_pktlen = 0; 1756104478Ssam mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr); 1757104478Ssam mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) & 1758104478Ssam UBS_PKTBUF_LEN); 1759104478Ssam mcr->mcr_opktbuf.pb_next = 0; 1760104478Ssam 1761104478Ssam ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass)); 1762104478Ssam ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS); 1763104478Ssam rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS; 1764104478Ssam 1765108823Ssam ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_PREREAD); 1766104478Ssam 1767104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next); 1768104478Ssam rng->rng_used = 1; 1769104478Ssam ubsec_feed2(sc); 1770104478Ssam ubsecstats.hst_rng++; 1771104478Ssam UBSEC_UNLOCK(sc); 1772104478Ssam 1773104478Ssam return; 1774104478Ssam 1775104478Ssamout: 1776104478Ssam /* 1777104478Ssam * Something weird happened, generate our own call back. 1778104478Ssam */ 1779104478Ssam sc->sc_nqueue2--; 1780104478Ssam UBSEC_UNLOCK(sc); 1781104478Ssam callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1782104478Ssam} 1783104478Ssam#endif /* UBSEC_NO_RNG */ 1784104478Ssam 1785104478Ssamstatic void 1786104478Ssamubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 1787104478Ssam{ 1788104478Ssam bus_addr_t *paddr = (bus_addr_t*) arg; 1789104478Ssam *paddr = segs->ds_addr; 1790104478Ssam} 1791104478Ssam 1792104478Ssamstatic int 1793104478Ssamubsec_dma_malloc( 1794104478Ssam struct ubsec_softc *sc, 1795104478Ssam bus_size_t size, 1796104478Ssam struct ubsec_dma_alloc *dma, 1797104478Ssam int mapflags 1798104478Ssam) 1799104478Ssam{ 1800104478Ssam int r; 1801104478Ssam 1802108823Ssam /* XXX could specify sc_dmat as parent but that just adds overhead */ 1803108823Ssam r = bus_dma_tag_create(NULL, /* parent */ 1804108823Ssam 1, 0, /* alignment, bounds */ 1805108823Ssam BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 1806108823Ssam BUS_SPACE_MAXADDR, /* highaddr */ 1807108823Ssam NULL, NULL, /* filter, filterarg */ 1808108823Ssam size, /* maxsize */ 1809108823Ssam 1, /* nsegments */ 1810108823Ssam size, /* maxsegsize */ 1811108823Ssam BUS_DMA_ALLOCNOW, /* flags */ 1812108823Ssam &dma->dma_tag); 1813108823Ssam if (r != 0) { 1814108823Ssam device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1815108823Ssam "bus_dma_tag_create failed; error %u\n", r); 1816104478Ssam goto fail_0; 1817108823Ssam } 1818104478Ssam 1819108823Ssam r = bus_dmamap_create(dma->dma_tag, BUS_DMA_NOWAIT, &dma->dma_map); 1820108823Ssam if (r != 0) { 1821108823Ssam device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1822108823Ssam "bus_dmamap_create failed; error %u\n", r); 1823104478Ssam goto fail_1; 1824108823Ssam } 1825104478Ssam 1826108823Ssam r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 1827108823Ssam BUS_DMA_NOWAIT, &dma->dma_map); 1828108823Ssam if (r != 0) { 1829108823Ssam device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1830109094Ssam "bus_dmammem_alloc failed; size %zu, error %u\n", 1831108823Ssam size, r); 1832108823Ssam goto fail_2; 1833108823Ssam } 1834108823Ssam 1835108823Ssam r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 1836104478Ssam size, 1837104478Ssam ubsec_dmamap_cb, 1838104478Ssam &dma->dma_paddr, 1839104478Ssam mapflags | BUS_DMA_NOWAIT); 1840108823Ssam if (r != 0) { 1841108823Ssam device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1842108823Ssam "bus_dmamap_load failed; error %u\n", r); 1843108823Ssam goto fail_3; 1844108823Ssam } 1845104478Ssam 1846104478Ssam dma->dma_size = size; 1847104478Ssam return (0); 1848104478Ssam 1849108823Ssamfail_3: 1850108823Ssam bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1851104478Ssamfail_2: 1852108823Ssam bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1853104478Ssamfail_1: 1854108823Ssam bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1855108823Ssam bus_dma_tag_destroy(dma->dma_tag); 1856104478Ssamfail_0: 1857104478Ssam dma->dma_map = NULL; 1858108823Ssam dma->dma_tag = NULL; 1859104478Ssam return (r); 1860104478Ssam} 1861104478Ssam 1862104478Ssamstatic void 1863104478Ssamubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma) 1864104478Ssam{ 1865108823Ssam bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1866108823Ssam bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1867108823Ssam bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1868108823Ssam bus_dma_tag_destroy(dma->dma_tag); 1869104478Ssam} 1870104478Ssam 1871104478Ssam/* 1872104478Ssam * Resets the board. Values in the regesters are left as is 1873104478Ssam * from the reset (i.e. initial values are assigned elsewhere). 1874104478Ssam */ 1875104478Ssamstatic void 1876104478Ssamubsec_reset_board(struct ubsec_softc *sc) 1877104478Ssam{ 1878104478Ssam volatile u_int32_t ctrl; 1879104478Ssam 1880104478Ssam ctrl = READ_REG(sc, BS_CTRL); 1881104478Ssam ctrl |= BS_CTRL_RESET; 1882104478Ssam WRITE_REG(sc, BS_CTRL, ctrl); 1883104478Ssam 1884104478Ssam /* 1885104478Ssam * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us 1886104478Ssam */ 1887104478Ssam DELAY(10); 1888104478Ssam} 1889104478Ssam 1890104478Ssam/* 1891104478Ssam * Init Broadcom registers 1892104478Ssam */ 1893104478Ssamstatic void 1894104478Ssamubsec_init_board(struct ubsec_softc *sc) 1895104478Ssam{ 1896104630Ssam u_int32_t ctrl; 1897104630Ssam 1898104630Ssam ctrl = READ_REG(sc, BS_CTRL); 1899104630Ssam ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64); 1900104630Ssam ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT; 1901104630Ssam 1902104630Ssam if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) 1903104630Ssam ctrl |= BS_CTRL_MCR2INT; 1904104630Ssam else 1905104630Ssam ctrl &= ~BS_CTRL_MCR2INT; 1906104630Ssam 1907104630Ssam if (sc->sc_flags & UBS_FLAGS_HWNORM) 1908104630Ssam ctrl &= ~BS_CTRL_SWNORM; 1909104630Ssam 1910104630Ssam WRITE_REG(sc, BS_CTRL, ctrl); 1911104478Ssam} 1912104478Ssam 1913104478Ssam/* 1914104478Ssam * Init Broadcom PCI registers 1915104478Ssam */ 1916104478Ssamstatic void 1917104478Ssamubsec_init_pciregs(device_t dev) 1918104478Ssam{ 1919104478Ssam#if 0 1920104478Ssam u_int32_t misc; 1921104478Ssam 1922104478Ssam misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT); 1923104478Ssam misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT)) 1924104478Ssam | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT); 1925104478Ssam misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT)) 1926104478Ssam | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT); 1927104478Ssam pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc); 1928104478Ssam#endif 1929104478Ssam 1930104478Ssam /* 1931104478Ssam * This will set the cache line size to 1, this will 1932104478Ssam * force the BCM58xx chip just to do burst read/writes. 1933104478Ssam * Cache line read/writes are to slow 1934104478Ssam */ 1935104478Ssam pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1); 1936104478Ssam} 1937104478Ssam 1938104478Ssam/* 1939104478Ssam * Clean up after a chip crash. 1940104478Ssam * It is assumed that the caller in splimp() 1941104478Ssam */ 1942104478Ssamstatic void 1943104478Ssamubsec_cleanchip(struct ubsec_softc *sc) 1944104478Ssam{ 1945104478Ssam struct ubsec_q *q; 1946104478Ssam 1947104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 1948104478Ssam q = SIMPLEQ_FIRST(&sc->sc_qchip); 1949104478Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next); 1950104478Ssam ubsec_free_q(sc, q); 1951104478Ssam } 1952108471Ssam sc->sc_nqchip = 0; 1953104478Ssam} 1954104478Ssam 1955104478Ssam/* 1956108823Ssam * free a ubsec_q 1957108823Ssam * It is assumed that the caller is within splimp(). 1958104478Ssam */ 1959104478Ssamstatic int 1960104478Ssamubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q) 1961104478Ssam{ 1962104478Ssam struct ubsec_q *q2; 1963104478Ssam struct cryptop *crp; 1964104478Ssam int npkts; 1965104478Ssam int i; 1966104478Ssam 1967104478Ssam npkts = q->q_nstacked_mcrs; 1968104478Ssam 1969104478Ssam for (i = 0; i < npkts; i++) { 1970104478Ssam if(q->q_stacked_mcr[i]) { 1971104478Ssam q2 = q->q_stacked_mcr[i]; 1972104478Ssam 1973104478Ssam if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m)) 1974104478Ssam m_freem(q2->q_dst_m); 1975104478Ssam 1976104478Ssam crp = (struct cryptop *)q2->q_crp; 1977104478Ssam 1978104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next); 1979104478Ssam 1980104478Ssam crp->crp_etype = EFAULT; 1981104478Ssam crypto_done(crp); 1982104478Ssam } else { 1983104478Ssam break; 1984104478Ssam } 1985104478Ssam } 1986104478Ssam 1987104478Ssam /* 1988104478Ssam * Free header MCR 1989104478Ssam */ 1990104478Ssam if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 1991104478Ssam m_freem(q->q_dst_m); 1992104478Ssam 1993104478Ssam crp = (struct cryptop *)q->q_crp; 1994104478Ssam 1995104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1996104478Ssam 1997104478Ssam crp->crp_etype = EFAULT; 1998104478Ssam crypto_done(crp); 1999104478Ssam return(0); 2000104478Ssam} 2001104478Ssam 2002104478Ssam/* 2003104478Ssam * Routine to reset the chip and clean up. 2004104478Ssam * It is assumed that the caller is in splimp() 2005104478Ssam */ 2006104478Ssamstatic void 2007104478Ssamubsec_totalreset(struct ubsec_softc *sc) 2008104478Ssam{ 2009104478Ssam ubsec_reset_board(sc); 2010104478Ssam ubsec_init_board(sc); 2011104478Ssam ubsec_cleanchip(sc); 2012104478Ssam} 2013104478Ssam 2014104478Ssamstatic int 2015104478Ssamubsec_dmamap_aligned(struct ubsec_operand *op) 2016104478Ssam{ 2017104478Ssam int i; 2018104478Ssam 2019104478Ssam for (i = 0; i < op->nsegs; i++) { 2020104478Ssam if (op->segs[i].ds_addr & 3) 2021104478Ssam return (0); 2022104478Ssam if ((i != (op->nsegs - 1)) && 2023104478Ssam (op->segs[i].ds_len & 3)) 2024104478Ssam return (0); 2025104478Ssam } 2026104478Ssam return (1); 2027104478Ssam} 2028104478Ssam 2029104478Ssamstatic void 2030104478Ssamubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q) 2031104478Ssam{ 2032104478Ssam switch (q->q_type) { 2033104478Ssam case UBS_CTXOP_MODEXP: { 2034104478Ssam struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 2035104478Ssam 2036104478Ssam ubsec_dma_free(sc, &me->me_q.q_mcr); 2037104478Ssam ubsec_dma_free(sc, &me->me_q.q_ctx); 2038104478Ssam ubsec_dma_free(sc, &me->me_M); 2039104478Ssam ubsec_dma_free(sc, &me->me_E); 2040104478Ssam ubsec_dma_free(sc, &me->me_C); 2041104478Ssam ubsec_dma_free(sc, &me->me_epb); 2042104478Ssam free(me, M_DEVBUF); 2043104478Ssam break; 2044104478Ssam } 2045104478Ssam case UBS_CTXOP_RSAPRIV: { 2046104478Ssam struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 2047104478Ssam 2048104478Ssam ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2049104478Ssam ubsec_dma_free(sc, &rp->rpr_q.q_ctx); 2050104478Ssam ubsec_dma_free(sc, &rp->rpr_msgin); 2051104478Ssam ubsec_dma_free(sc, &rp->rpr_msgout); 2052104478Ssam free(rp, M_DEVBUF); 2053104478Ssam break; 2054104478Ssam } 2055104478Ssam default: 2056104478Ssam device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type); 2057104478Ssam break; 2058104478Ssam } 2059104478Ssam} 2060104478Ssam 2061104478Ssamstatic int 2062104478Ssamubsec_kprocess(void *arg, struct cryptkop *krp, int hint) 2063104478Ssam{ 2064104478Ssam struct ubsec_softc *sc = arg; 2065104630Ssam int r; 2066104478Ssam 2067104478Ssam if (krp == NULL || krp->krp_callback == NULL) 2068104478Ssam return (EINVAL); 2069104478Ssam 2070104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) { 2071104478Ssam struct ubsec_q2 *q; 2072104478Ssam 2073104478Ssam q = SIMPLEQ_FIRST(&sc->sc_q2free); 2074104478Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q, q_next); 2075104478Ssam ubsec_kfree(sc, q); 2076104478Ssam } 2077104478Ssam 2078104478Ssam switch (krp->krp_op) { 2079104478Ssam case CRK_MOD_EXP: 2080104630Ssam if (sc->sc_flags & UBS_FLAGS_HWNORM) 2081104630Ssam r = ubsec_kprocess_modexp_hw(sc, krp, hint); 2082104630Ssam else 2083104630Ssam r = ubsec_kprocess_modexp_sw(sc, krp, hint); 2084104630Ssam break; 2085104478Ssam case CRK_MOD_EXP_CRT: 2086104478Ssam return (ubsec_kprocess_rsapriv(sc, krp, hint)); 2087104478Ssam default: 2088104478Ssam device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n", 2089104478Ssam krp->krp_op); 2090104478Ssam krp->krp_status = EOPNOTSUPP; 2091104478Ssam crypto_kdone(krp); 2092104478Ssam return (0); 2093104478Ssam } 2094104630Ssam return (0); /* silence compiler */ 2095104478Ssam} 2096104478Ssam 2097104478Ssam/* 2098104630Ssam * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization) 2099104478Ssam */ 2100104478Ssamstatic int 2101104630Ssamubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2102104478Ssam{ 2103104478Ssam struct ubsec_q2_modexp *me; 2104104478Ssam struct ubsec_mcr *mcr; 2105104478Ssam struct ubsec_ctx_modexp *ctx; 2106104478Ssam struct ubsec_pktbuf *epb; 2107104478Ssam int err = 0; 2108104478Ssam u_int nbits, normbits, mbits, shiftbits, ebits; 2109104478Ssam 2110104478Ssam me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2111104478Ssam if (me == NULL) { 2112104478Ssam err = ENOMEM; 2113104478Ssam goto errout; 2114104478Ssam } 2115104478Ssam bzero(me, sizeof *me); 2116104478Ssam me->me_krp = krp; 2117104478Ssam me->me_q.q_type = UBS_CTXOP_MODEXP; 2118104478Ssam 2119104478Ssam nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2120104478Ssam if (nbits <= 512) 2121104478Ssam normbits = 512; 2122104478Ssam else if (nbits <= 768) 2123104478Ssam normbits = 768; 2124104478Ssam else if (nbits <= 1024) 2125104478Ssam normbits = 1024; 2126104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2127104478Ssam normbits = 1536; 2128104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2129104478Ssam normbits = 2048; 2130104478Ssam else { 2131104478Ssam err = E2BIG; 2132104478Ssam goto errout; 2133104478Ssam } 2134104478Ssam 2135104630Ssam shiftbits = normbits - nbits; 2136104478Ssam 2137104630Ssam me->me_modbits = nbits; 2138104478Ssam me->me_shiftbits = shiftbits; 2139104630Ssam me->me_normbits = normbits; 2140104478Ssam 2141104478Ssam /* Sanity check: result bits must be >= true modulus bits. */ 2142104630Ssam if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2143104478Ssam err = ERANGE; 2144104478Ssam goto errout; 2145104478Ssam } 2146104478Ssam 2147104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2148104478Ssam &me->me_q.q_mcr, 0)) { 2149104478Ssam err = ENOMEM; 2150104478Ssam goto errout; 2151104478Ssam } 2152104478Ssam mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2153104478Ssam 2154104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2155104478Ssam &me->me_q.q_ctx, 0)) { 2156104478Ssam err = ENOMEM; 2157104478Ssam goto errout; 2158104478Ssam } 2159104478Ssam 2160104478Ssam mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2161104478Ssam if (mbits > nbits) { 2162104478Ssam err = E2BIG; 2163104478Ssam goto errout; 2164104478Ssam } 2165104478Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2166104478Ssam err = ENOMEM; 2167104478Ssam goto errout; 2168104478Ssam } 2169104478Ssam ubsec_kshift_r(shiftbits, 2170104478Ssam krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits, 2171104478Ssam me->me_M.dma_vaddr, normbits); 2172104478Ssam 2173104478Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2174104478Ssam err = ENOMEM; 2175104478Ssam goto errout; 2176104478Ssam } 2177104478Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2178104478Ssam 2179104478Ssam ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2180104478Ssam if (ebits > nbits) { 2181104478Ssam err = E2BIG; 2182104478Ssam goto errout; 2183104478Ssam } 2184104478Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2185104478Ssam err = ENOMEM; 2186104478Ssam goto errout; 2187104478Ssam } 2188104478Ssam ubsec_kshift_r(shiftbits, 2189104478Ssam krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits, 2190104478Ssam me->me_E.dma_vaddr, normbits); 2191104478Ssam 2192104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2193104478Ssam &me->me_epb, 0)) { 2194104478Ssam err = ENOMEM; 2195104478Ssam goto errout; 2196104478Ssam } 2197104478Ssam epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2198104478Ssam epb->pb_addr = htole32(me->me_E.dma_paddr); 2199104478Ssam epb->pb_next = 0; 2200104478Ssam epb->pb_len = htole32(normbits / 8); 2201104478Ssam 2202104478Ssam#ifdef UBSEC_DEBUG 2203104478Ssam if (ubsec_debug) { 2204104478Ssam printf("Epb "); 2205104478Ssam ubsec_dump_pb(epb); 2206104478Ssam } 2207104478Ssam#endif 2208104478Ssam 2209104478Ssam mcr->mcr_pkts = htole16(1); 2210104478Ssam mcr->mcr_flags = 0; 2211104478Ssam mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2212104478Ssam mcr->mcr_reserved = 0; 2213104478Ssam mcr->mcr_pktlen = 0; 2214104478Ssam 2215104478Ssam mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2216104478Ssam mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2217104478Ssam mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2218104478Ssam 2219104478Ssam mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2220104478Ssam mcr->mcr_opktbuf.pb_next = 0; 2221104478Ssam mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2222104478Ssam 2223104478Ssam#ifdef DIAGNOSTIC 2224104478Ssam /* Misaligned output buffer will hang the chip. */ 2225104478Ssam if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2226104478Ssam panic("%s: modexp invalid addr 0x%x\n", 2227104478Ssam device_get_nameunit(sc->sc_dev), 2228104478Ssam letoh32(mcr->mcr_opktbuf.pb_addr)); 2229104478Ssam if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2230104478Ssam panic("%s: modexp invalid len 0x%x\n", 2231104478Ssam device_get_nameunit(sc->sc_dev), 2232104478Ssam letoh32(mcr->mcr_opktbuf.pb_len)); 2233104478Ssam#endif 2234104478Ssam 2235104478Ssam ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2236104478Ssam bzero(ctx, sizeof(*ctx)); 2237104478Ssam ubsec_kshift_r(shiftbits, 2238104478Ssam krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits, 2239104478Ssam ctx->me_N, normbits); 2240104478Ssam ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2241104478Ssam ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2242104630Ssam ctx->me_E_len = htole16(nbits); 2243104630Ssam ctx->me_N_len = htole16(nbits); 2244104478Ssam 2245104478Ssam#ifdef UBSEC_DEBUG 2246104478Ssam if (ubsec_debug) { 2247104478Ssam ubsec_dump_mcr(mcr); 2248104478Ssam ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2249104478Ssam } 2250104478Ssam#endif 2251104478Ssam 2252104478Ssam /* 2253104478Ssam * ubsec_feed2 will sync mcr and ctx, we just need to sync 2254104478Ssam * everything else. 2255104478Ssam */ 2256108823Ssam ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE); 2257108823Ssam ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE); 2258108823Ssam ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD); 2259108823Ssam ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE); 2260104478Ssam 2261104478Ssam /* Enqueue and we're done... */ 2262104478Ssam UBSEC_LOCK(sc); 2263104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2264104478Ssam ubsec_feed2(sc); 2265104478Ssam ubsecstats.hst_modexp++; 2266104478Ssam UBSEC_UNLOCK(sc); 2267104478Ssam 2268104478Ssam return (0); 2269104478Ssam 2270104478Ssamerrout: 2271104478Ssam if (me != NULL) { 2272104478Ssam if (me->me_q.q_mcr.dma_map != NULL) 2273104478Ssam ubsec_dma_free(sc, &me->me_q.q_mcr); 2274104478Ssam if (me->me_q.q_ctx.dma_map != NULL) { 2275104478Ssam bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2276104478Ssam ubsec_dma_free(sc, &me->me_q.q_ctx); 2277104478Ssam } 2278104478Ssam if (me->me_M.dma_map != NULL) { 2279104478Ssam bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2280104478Ssam ubsec_dma_free(sc, &me->me_M); 2281104478Ssam } 2282104478Ssam if (me->me_E.dma_map != NULL) { 2283104478Ssam bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2284104478Ssam ubsec_dma_free(sc, &me->me_E); 2285104478Ssam } 2286104478Ssam if (me->me_C.dma_map != NULL) { 2287104478Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2288104478Ssam ubsec_dma_free(sc, &me->me_C); 2289104478Ssam } 2290104478Ssam if (me->me_epb.dma_map != NULL) 2291104478Ssam ubsec_dma_free(sc, &me->me_epb); 2292104478Ssam free(me, M_DEVBUF); 2293104478Ssam } 2294104478Ssam krp->krp_status = err; 2295104478Ssam crypto_kdone(krp); 2296104478Ssam return (0); 2297104478Ssam} 2298104478Ssam 2299104630Ssam/* 2300104630Ssam * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization) 2301104630Ssam */ 2302105215Sphkstatic int 2303104630Ssamubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2304104630Ssam{ 2305104630Ssam struct ubsec_q2_modexp *me; 2306104630Ssam struct ubsec_mcr *mcr; 2307104630Ssam struct ubsec_ctx_modexp *ctx; 2308104630Ssam struct ubsec_pktbuf *epb; 2309104630Ssam int err = 0; 2310104630Ssam u_int nbits, normbits, mbits, shiftbits, ebits; 2311104630Ssam 2312104630Ssam me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2313104630Ssam if (me == NULL) { 2314104630Ssam err = ENOMEM; 2315104630Ssam goto errout; 2316104630Ssam } 2317104630Ssam bzero(me, sizeof *me); 2318104630Ssam me->me_krp = krp; 2319104630Ssam me->me_q.q_type = UBS_CTXOP_MODEXP; 2320104630Ssam 2321104630Ssam nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2322104630Ssam if (nbits <= 512) 2323104630Ssam normbits = 512; 2324104630Ssam else if (nbits <= 768) 2325104630Ssam normbits = 768; 2326104630Ssam else if (nbits <= 1024) 2327104630Ssam normbits = 1024; 2328104630Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2329104630Ssam normbits = 1536; 2330104630Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2331104630Ssam normbits = 2048; 2332104630Ssam else { 2333104630Ssam err = E2BIG; 2334104630Ssam goto errout; 2335104630Ssam } 2336104630Ssam 2337104630Ssam shiftbits = normbits - nbits; 2338104630Ssam 2339104630Ssam /* XXX ??? */ 2340104630Ssam me->me_modbits = nbits; 2341104630Ssam me->me_shiftbits = shiftbits; 2342104630Ssam me->me_normbits = normbits; 2343104630Ssam 2344104630Ssam /* Sanity check: result bits must be >= true modulus bits. */ 2345104630Ssam if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2346104630Ssam err = ERANGE; 2347104630Ssam goto errout; 2348104630Ssam } 2349104630Ssam 2350104630Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2351104630Ssam &me->me_q.q_mcr, 0)) { 2352104630Ssam err = ENOMEM; 2353104630Ssam goto errout; 2354104630Ssam } 2355104630Ssam mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2356104630Ssam 2357104630Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2358104630Ssam &me->me_q.q_ctx, 0)) { 2359104630Ssam err = ENOMEM; 2360104630Ssam goto errout; 2361104630Ssam } 2362104630Ssam 2363104630Ssam mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2364104630Ssam if (mbits > nbits) { 2365104630Ssam err = E2BIG; 2366104630Ssam goto errout; 2367104630Ssam } 2368104630Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2369104630Ssam err = ENOMEM; 2370104630Ssam goto errout; 2371104630Ssam } 2372104630Ssam bzero(me->me_M.dma_vaddr, normbits / 8); 2373104630Ssam bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p, 2374104630Ssam me->me_M.dma_vaddr, (mbits + 7) / 8); 2375104630Ssam 2376104630Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2377104630Ssam err = ENOMEM; 2378104630Ssam goto errout; 2379104630Ssam } 2380104630Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2381104630Ssam 2382104630Ssam ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2383104630Ssam if (ebits > nbits) { 2384104630Ssam err = E2BIG; 2385104630Ssam goto errout; 2386104630Ssam } 2387104630Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2388104630Ssam err = ENOMEM; 2389104630Ssam goto errout; 2390104630Ssam } 2391104630Ssam bzero(me->me_E.dma_vaddr, normbits / 8); 2392104630Ssam bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p, 2393104630Ssam me->me_E.dma_vaddr, (ebits + 7) / 8); 2394104630Ssam 2395104630Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2396104630Ssam &me->me_epb, 0)) { 2397104630Ssam err = ENOMEM; 2398104630Ssam goto errout; 2399104630Ssam } 2400104630Ssam epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2401104630Ssam epb->pb_addr = htole32(me->me_E.dma_paddr); 2402104630Ssam epb->pb_next = 0; 2403104630Ssam epb->pb_len = htole32((ebits + 7) / 8); 2404104630Ssam 2405104630Ssam#ifdef UBSEC_DEBUG 2406108823Ssam if (ubsec_debug) { 2407108823Ssam printf("Epb "); 2408108823Ssam ubsec_dump_pb(epb); 2409108823Ssam } 2410104630Ssam#endif 2411104630Ssam 2412104630Ssam mcr->mcr_pkts = htole16(1); 2413104630Ssam mcr->mcr_flags = 0; 2414104630Ssam mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2415104630Ssam mcr->mcr_reserved = 0; 2416104630Ssam mcr->mcr_pktlen = 0; 2417104630Ssam 2418104630Ssam mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2419104630Ssam mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2420104630Ssam mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2421104630Ssam 2422104630Ssam mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2423104630Ssam mcr->mcr_opktbuf.pb_next = 0; 2424104630Ssam mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2425104630Ssam 2426104630Ssam#ifdef DIAGNOSTIC 2427104630Ssam /* Misaligned output buffer will hang the chip. */ 2428104630Ssam if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2429104630Ssam panic("%s: modexp invalid addr 0x%x\n", 2430104630Ssam device_get_nameunit(sc->sc_dev), 2431104630Ssam letoh32(mcr->mcr_opktbuf.pb_addr)); 2432104630Ssam if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2433104630Ssam panic("%s: modexp invalid len 0x%x\n", 2434104630Ssam device_get_nameunit(sc->sc_dev), 2435104630Ssam letoh32(mcr->mcr_opktbuf.pb_len)); 2436104630Ssam#endif 2437104630Ssam 2438104630Ssam ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2439104630Ssam bzero(ctx, sizeof(*ctx)); 2440104630Ssam bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N, 2441104630Ssam (nbits + 7) / 8); 2442104630Ssam ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2443104630Ssam ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2444104630Ssam ctx->me_E_len = htole16(ebits); 2445104630Ssam ctx->me_N_len = htole16(nbits); 2446104630Ssam 2447104630Ssam#ifdef UBSEC_DEBUG 2448104630Ssam if (ubsec_debug) { 2449104630Ssam ubsec_dump_mcr(mcr); 2450104630Ssam ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2451104630Ssam } 2452104630Ssam#endif 2453104630Ssam 2454104630Ssam /* 2455104630Ssam * ubsec_feed2 will sync mcr and ctx, we just need to sync 2456104630Ssam * everything else. 2457104630Ssam */ 2458108823Ssam ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE); 2459108823Ssam ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE); 2460108823Ssam ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD); 2461108823Ssam ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE); 2462104630Ssam 2463104630Ssam /* Enqueue and we're done... */ 2464104630Ssam UBSEC_LOCK(sc); 2465104630Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2466104630Ssam ubsec_feed2(sc); 2467104630Ssam UBSEC_UNLOCK(sc); 2468104630Ssam 2469104630Ssam return (0); 2470104630Ssam 2471104630Ssamerrout: 2472104630Ssam if (me != NULL) { 2473104630Ssam if (me->me_q.q_mcr.dma_map != NULL) 2474104630Ssam ubsec_dma_free(sc, &me->me_q.q_mcr); 2475104630Ssam if (me->me_q.q_ctx.dma_map != NULL) { 2476104630Ssam bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2477104630Ssam ubsec_dma_free(sc, &me->me_q.q_ctx); 2478104630Ssam } 2479104630Ssam if (me->me_M.dma_map != NULL) { 2480104630Ssam bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2481104630Ssam ubsec_dma_free(sc, &me->me_M); 2482104630Ssam } 2483104630Ssam if (me->me_E.dma_map != NULL) { 2484104630Ssam bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2485104630Ssam ubsec_dma_free(sc, &me->me_E); 2486104630Ssam } 2487104630Ssam if (me->me_C.dma_map != NULL) { 2488104630Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2489104630Ssam ubsec_dma_free(sc, &me->me_C); 2490104630Ssam } 2491104630Ssam if (me->me_epb.dma_map != NULL) 2492104630Ssam ubsec_dma_free(sc, &me->me_epb); 2493104630Ssam free(me, M_DEVBUF); 2494104630Ssam } 2495104630Ssam krp->krp_status = err; 2496104630Ssam crypto_kdone(krp); 2497104630Ssam return (0); 2498104630Ssam} 2499104630Ssam 2500104478Ssamstatic int 2501104478Ssamubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2502104478Ssam{ 2503104478Ssam struct ubsec_q2_rsapriv *rp = NULL; 2504104478Ssam struct ubsec_mcr *mcr; 2505104478Ssam struct ubsec_ctx_rsapriv *ctx; 2506104478Ssam int err = 0; 2507104478Ssam u_int padlen, msglen; 2508104478Ssam 2509104478Ssam msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]); 2510104478Ssam padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]); 2511104478Ssam if (msglen > padlen) 2512104478Ssam padlen = msglen; 2513104478Ssam 2514104478Ssam if (padlen <= 256) 2515104478Ssam padlen = 256; 2516104478Ssam else if (padlen <= 384) 2517104478Ssam padlen = 384; 2518104478Ssam else if (padlen <= 512) 2519104478Ssam padlen = 512; 2520104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768) 2521104478Ssam padlen = 768; 2522104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024) 2523104478Ssam padlen = 1024; 2524104478Ssam else { 2525104478Ssam err = E2BIG; 2526104478Ssam goto errout; 2527104478Ssam } 2528104478Ssam 2529104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) { 2530104478Ssam err = E2BIG; 2531104478Ssam goto errout; 2532104478Ssam } 2533104478Ssam 2534104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) { 2535104478Ssam err = E2BIG; 2536104478Ssam goto errout; 2537104478Ssam } 2538104478Ssam 2539104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) { 2540104478Ssam err = E2BIG; 2541104478Ssam goto errout; 2542104478Ssam } 2543104478Ssam 2544104478Ssam rp = (struct ubsec_q2_rsapriv *)malloc(sizeof *rp, M_DEVBUF, M_NOWAIT); 2545104478Ssam if (rp == NULL) 2546104478Ssam return (ENOMEM); 2547104478Ssam bzero(rp, sizeof *rp); 2548104478Ssam rp->rpr_krp = krp; 2549104478Ssam rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV; 2550104478Ssam 2551104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2552104478Ssam &rp->rpr_q.q_mcr, 0)) { 2553104478Ssam err = ENOMEM; 2554104478Ssam goto errout; 2555104478Ssam } 2556104478Ssam mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr; 2557104478Ssam 2558104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv), 2559104478Ssam &rp->rpr_q.q_ctx, 0)) { 2560104478Ssam err = ENOMEM; 2561104478Ssam goto errout; 2562104478Ssam } 2563104478Ssam ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr; 2564104478Ssam bzero(ctx, sizeof *ctx); 2565104478Ssam 2566104478Ssam /* Copy in p */ 2567104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p, 2568104478Ssam &ctx->rpr_buf[0 * (padlen / 8)], 2569104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8); 2570104478Ssam 2571104478Ssam /* Copy in q */ 2572104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p, 2573104478Ssam &ctx->rpr_buf[1 * (padlen / 8)], 2574104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8); 2575104478Ssam 2576104478Ssam /* Copy in dp */ 2577104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p, 2578104478Ssam &ctx->rpr_buf[2 * (padlen / 8)], 2579104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8); 2580104478Ssam 2581104478Ssam /* Copy in dq */ 2582104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p, 2583104478Ssam &ctx->rpr_buf[3 * (padlen / 8)], 2584104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8); 2585104478Ssam 2586104478Ssam /* Copy in pinv */ 2587104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p, 2588104478Ssam &ctx->rpr_buf[4 * (padlen / 8)], 2589104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8); 2590104478Ssam 2591104478Ssam msglen = padlen * 2; 2592104478Ssam 2593104478Ssam /* Copy in input message (aligned buffer/length). */ 2594104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) { 2595104478Ssam /* Is this likely? */ 2596104478Ssam err = E2BIG; 2597104478Ssam goto errout; 2598104478Ssam } 2599104478Ssam if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) { 2600104478Ssam err = ENOMEM; 2601104478Ssam goto errout; 2602104478Ssam } 2603104478Ssam bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8); 2604104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p, 2605104478Ssam rp->rpr_msgin.dma_vaddr, 2606104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8); 2607104478Ssam 2608104478Ssam /* Prepare space for output message (aligned buffer/length). */ 2609104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) { 2610104478Ssam /* Is this likely? */ 2611104478Ssam err = E2BIG; 2612104478Ssam goto errout; 2613104478Ssam } 2614104478Ssam if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) { 2615104478Ssam err = ENOMEM; 2616104478Ssam goto errout; 2617104478Ssam } 2618104478Ssam bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8); 2619104478Ssam 2620104478Ssam mcr->mcr_pkts = htole16(1); 2621104478Ssam mcr->mcr_flags = 0; 2622104478Ssam mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr); 2623104478Ssam mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr); 2624104478Ssam mcr->mcr_ipktbuf.pb_next = 0; 2625104478Ssam mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size); 2626104478Ssam mcr->mcr_reserved = 0; 2627104478Ssam mcr->mcr_pktlen = htole16(msglen); 2628104478Ssam mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr); 2629104478Ssam mcr->mcr_opktbuf.pb_next = 0; 2630104478Ssam mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size); 2631104478Ssam 2632104478Ssam#ifdef DIAGNOSTIC 2633104478Ssam if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) { 2634106579Sjhb panic("%s: rsapriv: invalid msgin %x(0x%jx)", 2635104478Ssam device_get_nameunit(sc->sc_dev), 2636106579Sjhb rp->rpr_msgin.dma_paddr, (uintmax_t)rp->rpr_msgin.dma_size); 2637104478Ssam } 2638104478Ssam if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) { 2639106579Sjhb panic("%s: rsapriv: invalid msgout %x(0x%jx)", 2640104478Ssam device_get_nameunit(sc->sc_dev), 2641106579Sjhb rp->rpr_msgout.dma_paddr, (uintmax_t)rp->rpr_msgout.dma_size); 2642104478Ssam } 2643104478Ssam#endif 2644104478Ssam 2645104478Ssam ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8)); 2646104478Ssam ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV); 2647104478Ssam ctx->rpr_q_len = htole16(padlen); 2648104478Ssam ctx->rpr_p_len = htole16(padlen); 2649104478Ssam 2650104478Ssam /* 2651104478Ssam * ubsec_feed2 will sync mcr and ctx, we just need to sync 2652104478Ssam * everything else. 2653104478Ssam */ 2654108823Ssam ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_PREWRITE); 2655108823Ssam ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_PREREAD); 2656104478Ssam 2657104478Ssam /* Enqueue and we're done... */ 2658104478Ssam UBSEC_LOCK(sc); 2659104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next); 2660104478Ssam ubsec_feed2(sc); 2661104478Ssam ubsecstats.hst_modexpcrt++; 2662104478Ssam UBSEC_UNLOCK(sc); 2663104478Ssam return (0); 2664104478Ssam 2665104478Ssamerrout: 2666104478Ssam if (rp != NULL) { 2667104478Ssam if (rp->rpr_q.q_mcr.dma_map != NULL) 2668104478Ssam ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2669104478Ssam if (rp->rpr_msgin.dma_map != NULL) { 2670104478Ssam bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 2671104478Ssam ubsec_dma_free(sc, &rp->rpr_msgin); 2672104478Ssam } 2673104478Ssam if (rp->rpr_msgout.dma_map != NULL) { 2674104478Ssam bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 2675104478Ssam ubsec_dma_free(sc, &rp->rpr_msgout); 2676104478Ssam } 2677104478Ssam free(rp, M_DEVBUF); 2678104478Ssam } 2679104478Ssam krp->krp_status = err; 2680104478Ssam crypto_kdone(krp); 2681104478Ssam return (0); 2682104478Ssam} 2683104478Ssam 2684104478Ssam#ifdef UBSEC_DEBUG 2685104478Ssamstatic void 2686104478Ssamubsec_dump_pb(volatile struct ubsec_pktbuf *pb) 2687104478Ssam{ 2688104478Ssam printf("addr 0x%x (0x%x) next 0x%x\n", 2689104478Ssam pb->pb_addr, pb->pb_len, pb->pb_next); 2690104478Ssam} 2691104478Ssam 2692104478Ssamstatic void 2693104478Ssamubsec_dump_ctx2(struct ubsec_ctx_keyop *c) 2694104478Ssam{ 2695104478Ssam printf("CTX (0x%x):\n", c->ctx_len); 2696104478Ssam switch (letoh16(c->ctx_op)) { 2697104478Ssam case UBS_CTXOP_RNGBYPASS: 2698104478Ssam case UBS_CTXOP_RNGSHA1: 2699104478Ssam break; 2700104478Ssam case UBS_CTXOP_MODEXP: 2701104478Ssam { 2702104478Ssam struct ubsec_ctx_modexp *cx = (void *)c; 2703104478Ssam int i, len; 2704104478Ssam 2705104478Ssam printf(" Elen %u, Nlen %u\n", 2706104478Ssam letoh16(cx->me_E_len), letoh16(cx->me_N_len)); 2707104478Ssam len = (cx->me_N_len + 7)/8; 2708104478Ssam for (i = 0; i < len; i++) 2709104478Ssam printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]); 2710104478Ssam printf("\n"); 2711104478Ssam break; 2712104478Ssam } 2713104478Ssam default: 2714104478Ssam printf("unknown context: %x\n", c->ctx_op); 2715104478Ssam } 2716104478Ssam printf("END CTX\n"); 2717104478Ssam} 2718104478Ssam 2719104478Ssamstatic void 2720104478Ssamubsec_dump_mcr(struct ubsec_mcr *mcr) 2721104478Ssam{ 2722104478Ssam volatile struct ubsec_mcr_add *ma; 2723104478Ssam int i; 2724104478Ssam 2725104478Ssam printf("MCR:\n"); 2726104478Ssam printf(" pkts: %u, flags 0x%x\n", 2727104478Ssam letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags)); 2728104478Ssam ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp; 2729104478Ssam for (i = 0; i < letoh16(mcr->mcr_pkts); i++) { 2730104478Ssam printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i, 2731104478Ssam letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen), 2732104478Ssam letoh16(ma->mcr_reserved)); 2733104478Ssam printf(" %d: ipkt ", i); 2734104478Ssam ubsec_dump_pb(&ma->mcr_ipktbuf); 2735104478Ssam printf(" %d: opkt ", i); 2736104478Ssam ubsec_dump_pb(&ma->mcr_opktbuf); 2737104478Ssam ma++; 2738104478Ssam } 2739104478Ssam printf("END MCR\n"); 2740104478Ssam} 2741104478Ssam#endif /* UBSEC_DEBUG */ 2742104478Ssam 2743104478Ssam/* 2744104478Ssam * Return the number of significant bits of a big number. 2745104478Ssam */ 2746104478Ssamstatic int 2747104478Ssamubsec_ksigbits(struct crparam *cr) 2748104478Ssam{ 2749104478Ssam u_int plen = (cr->crp_nbits + 7) / 8; 2750104478Ssam int i, sig = plen * 8; 2751104478Ssam u_int8_t c, *p = cr->crp_p; 2752104478Ssam 2753104478Ssam for (i = plen - 1; i >= 0; i--) { 2754104478Ssam c = p[i]; 2755104478Ssam if (c != 0) { 2756104478Ssam while ((c & 0x80) == 0) { 2757104478Ssam sig--; 2758104478Ssam c <<= 1; 2759104478Ssam } 2760104478Ssam break; 2761104478Ssam } 2762104478Ssam sig -= 8; 2763104478Ssam } 2764104478Ssam return (sig); 2765104478Ssam} 2766104478Ssam 2767104478Ssamstatic void 2768104478Ssamubsec_kshift_r( 2769104478Ssam u_int shiftbits, 2770104478Ssam u_int8_t *src, u_int srcbits, 2771104478Ssam u_int8_t *dst, u_int dstbits) 2772104478Ssam{ 2773104478Ssam u_int slen, dlen; 2774104478Ssam int i, si, di, n; 2775104478Ssam 2776104478Ssam slen = (srcbits + 7) / 8; 2777104478Ssam dlen = (dstbits + 7) / 8; 2778104478Ssam 2779104478Ssam for (i = 0; i < slen; i++) 2780104478Ssam dst[i] = src[i]; 2781104478Ssam for (i = 0; i < dlen - slen; i++) 2782104478Ssam dst[slen + i] = 0; 2783104478Ssam 2784104478Ssam n = shiftbits / 8; 2785104478Ssam if (n != 0) { 2786104478Ssam si = dlen - n - 1; 2787104478Ssam di = dlen - 1; 2788104478Ssam while (si >= 0) 2789104478Ssam dst[di--] = dst[si--]; 2790104478Ssam while (di >= 0) 2791104478Ssam dst[di--] = 0; 2792104478Ssam } 2793104478Ssam 2794104478Ssam n = shiftbits % 8; 2795104478Ssam if (n != 0) { 2796104478Ssam for (i = dlen - 1; i > 0; i--) 2797104478Ssam dst[i] = (dst[i] << n) | 2798104478Ssam (dst[i - 1] >> (8 - n)); 2799104478Ssam dst[0] = dst[0] << n; 2800104478Ssam } 2801104478Ssam} 2802104478Ssam 2803104478Ssamstatic void 2804104478Ssamubsec_kshift_l( 2805104478Ssam u_int shiftbits, 2806104478Ssam u_int8_t *src, u_int srcbits, 2807104478Ssam u_int8_t *dst, u_int dstbits) 2808104478Ssam{ 2809104478Ssam int slen, dlen, i, n; 2810104478Ssam 2811104478Ssam slen = (srcbits + 7) / 8; 2812104478Ssam dlen = (dstbits + 7) / 8; 2813104478Ssam 2814104478Ssam n = shiftbits / 8; 2815104478Ssam for (i = 0; i < slen; i++) 2816104478Ssam dst[i] = src[i + n]; 2817104478Ssam for (i = 0; i < dlen - slen; i++) 2818104478Ssam dst[slen + i] = 0; 2819104478Ssam 2820104478Ssam n = shiftbits % 8; 2821104478Ssam if (n != 0) { 2822104478Ssam for (i = 0; i < (dlen - 1); i++) 2823104478Ssam dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n)); 2824104478Ssam dst[dlen - 1] = dst[dlen - 1] >> n; 2825104478Ssam } 2826104478Ssam} 2827