ubsec.c revision 160931
1104630Ssam/* $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $ */ 2104478Ssam 3139749Simp/*- 4104478Ssam * Copyright (c) 2000 Jason L. Wright (jason@thought.net) 5104478Ssam * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org) 6104478Ssam * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com) 7104478Ssam * 8104478Ssam * All rights reserved. 9104478Ssam * 10104478Ssam * Redistribution and use in source and binary forms, with or without 11104478Ssam * modification, are permitted provided that the following conditions 12104478Ssam * are met: 13104478Ssam * 1. Redistributions of source code must retain the above copyright 14104478Ssam * notice, this list of conditions and the following disclaimer. 15104478Ssam * 2. Redistributions in binary form must reproduce the above copyright 16104478Ssam * notice, this list of conditions and the following disclaimer in the 17104478Ssam * documentation and/or other materials provided with the distribution. 18104478Ssam * 3. All advertising materials mentioning features or use of this software 19104478Ssam * must display the following acknowledgement: 20104478Ssam * This product includes software developed by Jason L. Wright 21104478Ssam * 4. The name of the author may not be used to endorse or promote products 22104478Ssam * derived from this software without specific prior written permission. 23104478Ssam * 24104478Ssam * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 25104478Ssam * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 26104478Ssam * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 27104478Ssam * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 28104478Ssam * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 29104478Ssam * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 30104478Ssam * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31104478Ssam * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 32104478Ssam * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 33104478Ssam * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34104478Ssam * POSSIBILITY OF SUCH DAMAGE. 35104478Ssam * 36104478Ssam * Effort sponsored in part by the Defense Advanced Research Projects 37104478Ssam * Agency (DARPA) and Air Force Research Laboratory, Air Force 38104478Ssam * Materiel Command, USAF, under agreement number F30602-01-2-0537. 39104478Ssam */ 40104478Ssam 41119418Sobrien#include <sys/cdefs.h> 42119418Sobrien__FBSDID("$FreeBSD: head/sys/dev/ubsec/ubsec.c 160931 2006-08-02 17:41:58Z jhb $"); 43119418Sobrien 44104478Ssam/* 45104478Ssam * uBsec 5[56]01, 58xx hardware crypto accelerator 46104478Ssam */ 47104478Ssam 48112124Ssam#include "opt_ubsec.h" 49112124Ssam 50104478Ssam#include <sys/param.h> 51104478Ssam#include <sys/systm.h> 52104478Ssam#include <sys/proc.h> 53104478Ssam#include <sys/errno.h> 54104478Ssam#include <sys/malloc.h> 55104478Ssam#include <sys/kernel.h> 56129879Sphk#include <sys/module.h> 57104478Ssam#include <sys/mbuf.h> 58104478Ssam#include <sys/lock.h> 59104478Ssam#include <sys/mutex.h> 60104478Ssam#include <sys/sysctl.h> 61104478Ssam#include <sys/endian.h> 62104478Ssam 63104478Ssam#include <vm/vm.h> 64104478Ssam#include <vm/pmap.h> 65104478Ssam 66104478Ssam#include <machine/bus.h> 67104478Ssam#include <machine/resource.h> 68104478Ssam#include <sys/bus.h> 69104478Ssam#include <sys/rman.h> 70104478Ssam 71104478Ssam#include <crypto/sha1.h> 72104478Ssam#include <opencrypto/cryptodev.h> 73104478Ssam#include <opencrypto/cryptosoft.h> 74104478Ssam#include <sys/md5.h> 75104478Ssam#include <sys/random.h> 76104478Ssam 77119287Simp#include <dev/pci/pcivar.h> 78119287Simp#include <dev/pci/pcireg.h> 79104478Ssam 80104478Ssam/* grr, #defines for gratuitous incompatibility in queue.h */ 81104478Ssam#define SIMPLEQ_HEAD STAILQ_HEAD 82104478Ssam#define SIMPLEQ_ENTRY STAILQ_ENTRY 83104478Ssam#define SIMPLEQ_INIT STAILQ_INIT 84104478Ssam#define SIMPLEQ_INSERT_TAIL STAILQ_INSERT_TAIL 85104478Ssam#define SIMPLEQ_EMPTY STAILQ_EMPTY 86104478Ssam#define SIMPLEQ_FIRST STAILQ_FIRST 87104478Ssam#define SIMPLEQ_REMOVE_HEAD STAILQ_REMOVE_HEAD_UNTIL 88111416Ssam#define SIMPLEQ_FOREACH STAILQ_FOREACH 89104478Ssam/* ditto for endian.h */ 90104478Ssam#define letoh16(x) le16toh(x) 91104478Ssam#define letoh32(x) le32toh(x) 92104478Ssam 93112124Ssam#ifdef UBSEC_RNDTEST 94112124Ssam#include <dev/rndtest/rndtest.h> 95112124Ssam#endif 96104478Ssam#include <dev/ubsec/ubsecreg.h> 97104478Ssam#include <dev/ubsec/ubsecvar.h> 98104478Ssam 99104478Ssam/* 100104478Ssam * Prototypes and count for the pci_device structure 101104478Ssam */ 102104478Ssamstatic int ubsec_probe(device_t); 103104478Ssamstatic int ubsec_attach(device_t); 104104478Ssamstatic int ubsec_detach(device_t); 105104478Ssamstatic int ubsec_suspend(device_t); 106104478Ssamstatic int ubsec_resume(device_t); 107104478Ssamstatic void ubsec_shutdown(device_t); 108104478Ssam 109104478Ssamstatic device_method_t ubsec_methods[] = { 110104478Ssam /* Device interface */ 111104478Ssam DEVMETHOD(device_probe, ubsec_probe), 112104478Ssam DEVMETHOD(device_attach, ubsec_attach), 113104478Ssam DEVMETHOD(device_detach, ubsec_detach), 114104478Ssam DEVMETHOD(device_suspend, ubsec_suspend), 115104478Ssam DEVMETHOD(device_resume, ubsec_resume), 116104478Ssam DEVMETHOD(device_shutdown, ubsec_shutdown), 117104478Ssam 118104478Ssam /* bus interface */ 119104478Ssam DEVMETHOD(bus_print_child, bus_generic_print_child), 120104478Ssam DEVMETHOD(bus_driver_added, bus_generic_driver_added), 121104478Ssam 122104478Ssam { 0, 0 } 123104478Ssam}; 124104478Ssamstatic driver_t ubsec_driver = { 125104478Ssam "ubsec", 126104478Ssam ubsec_methods, 127104478Ssam sizeof (struct ubsec_softc) 128104478Ssam}; 129104478Ssamstatic devclass_t ubsec_devclass; 130104478Ssam 131104478SsamDRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0); 132105251SmarkmMODULE_DEPEND(ubsec, crypto, 1, 1, 1); 133112124Ssam#ifdef UBSEC_RNDTEST 134112124SsamMODULE_DEPEND(ubsec, rndtest, 1, 1, 1); 135112124Ssam#endif 136104478Ssam 137104478Ssamstatic void ubsec_intr(void *); 138104478Ssamstatic int ubsec_newsession(void *, u_int32_t *, struct cryptoini *); 139104478Ssamstatic int ubsec_freesession(void *, u_int64_t); 140104478Ssamstatic int ubsec_process(void *, struct cryptop *, int); 141104478Ssamstatic void ubsec_callback(struct ubsec_softc *, struct ubsec_q *); 142111416Ssamstatic void ubsec_feed(struct ubsec_softc *); 143104478Ssamstatic void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int); 144104478Ssamstatic void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *); 145104478Ssamstatic int ubsec_feed2(struct ubsec_softc *); 146104478Ssamstatic void ubsec_rng(void *); 147104478Ssamstatic int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t, 148104478Ssam struct ubsec_dma_alloc *, int); 149108823Ssam#define ubsec_dma_sync(_dma, _flags) \ 150108823Ssam bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags)) 151104478Ssamstatic void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *); 152104478Ssamstatic int ubsec_dmamap_aligned(struct ubsec_operand *op); 153104478Ssam 154104478Ssamstatic void ubsec_reset_board(struct ubsec_softc *sc); 155104478Ssamstatic void ubsec_init_board(struct ubsec_softc *sc); 156104478Ssamstatic void ubsec_init_pciregs(device_t dev); 157104478Ssamstatic void ubsec_totalreset(struct ubsec_softc *sc); 158104478Ssam 159104478Ssamstatic int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q); 160104478Ssam 161104478Ssamstatic int ubsec_kprocess(void*, struct cryptkop *, int); 162104630Ssamstatic int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int); 163104630Ssamstatic int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int); 164104478Ssamstatic int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int); 165104478Ssamstatic void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *); 166104478Ssamstatic int ubsec_ksigbits(struct crparam *); 167104478Ssamstatic void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 168104478Ssamstatic void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 169104478Ssam 170109595SsamSYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0, "Broadcom driver parameters"); 171109595Ssam 172104478Ssam#ifdef UBSEC_DEBUG 173104478Ssamstatic void ubsec_dump_pb(volatile struct ubsec_pktbuf *); 174104478Ssamstatic void ubsec_dump_mcr(struct ubsec_mcr *); 175104478Ssamstatic void ubsec_dump_ctx2(struct ubsec_ctx_keyop *); 176104478Ssam 177104478Ssamstatic int ubsec_debug = 0; 178109595SsamSYSCTL_INT(_hw_ubsec, OID_AUTO, debug, CTLFLAG_RW, &ubsec_debug, 179109595Ssam 0, "control debugging msgs"); 180104478Ssam#endif 181104478Ssam 182104478Ssam#define READ_REG(sc,r) \ 183104478Ssam bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r)) 184104478Ssam 185104478Ssam#define WRITE_REG(sc,reg,val) \ 186104478Ssam bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val) 187104478Ssam 188104478Ssam#define SWAP32(x) (x) = htole32(ntohl((x))) 189104478Ssam#define HTOLE32(x) (x) = htole32(x) 190104478Ssam 191104478Ssamstruct ubsec_stats ubsecstats; 192109595SsamSYSCTL_STRUCT(_hw_ubsec, OID_AUTO, stats, CTLFLAG_RD, &ubsecstats, 193109595Ssam ubsec_stats, "driver statistics"); 194104478Ssam 195105215Sphkstatic int 196104478Ssamubsec_probe(device_t dev) 197104478Ssam{ 198114105Ssam if (pci_get_vendor(dev) == PCI_VENDOR_SUN && 199114105Ssam (pci_get_device(dev) == PCI_PRODUCT_SUN_5821 || 200114105Ssam pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K)) 201142880Simp return (BUS_PROBE_DEFAULT); 202104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 203104478Ssam (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 || 204104478Ssam pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)) 205142880Simp return (BUS_PROBE_DEFAULT); 206104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 207111646Ssam (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5801 || 208111646Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 || 209111646Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 || 210104478Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 || 211104478Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 212110522Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 || 213110522Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 214110522Ssam )) 215142880Simp return (BUS_PROBE_DEFAULT); 216104478Ssam return (ENXIO); 217104478Ssam} 218104478Ssam 219104478Ssamstatic const char* 220104478Ssamubsec_partname(struct ubsec_softc *sc) 221104478Ssam{ 222104478Ssam /* XXX sprintf numbers when not decoded */ 223104478Ssam switch (pci_get_vendor(sc->sc_dev)) { 224104478Ssam case PCI_VENDOR_BROADCOM: 225104478Ssam switch (pci_get_device(sc->sc_dev)) { 226111646Ssam case PCI_PRODUCT_BROADCOM_5801: return "Broadcom 5801"; 227111646Ssam case PCI_PRODUCT_BROADCOM_5802: return "Broadcom 5802"; 228104478Ssam case PCI_PRODUCT_BROADCOM_5805: return "Broadcom 5805"; 229104478Ssam case PCI_PRODUCT_BROADCOM_5820: return "Broadcom 5820"; 230104478Ssam case PCI_PRODUCT_BROADCOM_5821: return "Broadcom 5821"; 231104478Ssam case PCI_PRODUCT_BROADCOM_5822: return "Broadcom 5822"; 232110522Ssam case PCI_PRODUCT_BROADCOM_5823: return "Broadcom 5823"; 233104478Ssam } 234104478Ssam return "Broadcom unknown-part"; 235104478Ssam case PCI_VENDOR_BLUESTEEL: 236104478Ssam switch (pci_get_device(sc->sc_dev)) { 237104478Ssam case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601"; 238104478Ssam } 239104478Ssam return "Bluesteel unknown-part"; 240114105Ssam case PCI_VENDOR_SUN: 241114105Ssam switch (pci_get_device(sc->sc_dev)) { 242114105Ssam case PCI_PRODUCT_SUN_5821: return "Sun Crypto 5821"; 243114105Ssam case PCI_PRODUCT_SUN_SCA1K: return "Sun Crypto 1K"; 244114105Ssam } 245114105Ssam return "Sun unknown-part"; 246104478Ssam } 247104478Ssam return "Unknown-vendor unknown-part"; 248104478Ssam} 249104478Ssam 250112124Ssamstatic void 251112124Ssamdefault_harvest(struct rndtest_state *rsp, void *buf, u_int count) 252112124Ssam{ 253112124Ssam random_harvest(buf, count, count*NBBY, 0, RANDOM_PURE); 254112124Ssam} 255112124Ssam 256104478Ssamstatic int 257104478Ssamubsec_attach(device_t dev) 258104478Ssam{ 259104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 260104478Ssam struct ubsec_dma *dmap; 261104478Ssam u_int32_t cmd, i; 262104478Ssam int rid; 263104478Ssam 264104478Ssam bzero(sc, sizeof (*sc)); 265104478Ssam sc->sc_dev = dev; 266104478Ssam 267104478Ssam SIMPLEQ_INIT(&sc->sc_queue); 268104478Ssam SIMPLEQ_INIT(&sc->sc_qchip); 269104478Ssam SIMPLEQ_INIT(&sc->sc_queue2); 270104478Ssam SIMPLEQ_INIT(&sc->sc_qchip2); 271104478Ssam SIMPLEQ_INIT(&sc->sc_q2free); 272104478Ssam 273104478Ssam /* XXX handle power management */ 274104478Ssam 275104478Ssam sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR; 276104478Ssam 277104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 278104478Ssam pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601) 279104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 280104478Ssam 281104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 282111646Ssam (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 || 283111646Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805)) 284104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 285104478Ssam 286104478Ssam if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 287104478Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820) 288104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 289104478Ssam UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 290104478Ssam 291114105Ssam if ((pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 292114105Ssam (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 293114105Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 || 294114105Ssam pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823)) || 295114105Ssam (pci_get_vendor(dev) == PCI_VENDOR_SUN && 296114105Ssam (pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K || 297114105Ssam pci_get_device(dev) == PCI_PRODUCT_SUN_5821))) { 298104478Ssam /* NB: the 5821/5822 defines some additional status bits */ 299104478Ssam sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY | 300104478Ssam BS_STAT_MCR2_ALLEMPTY; 301104478Ssam sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 302104478Ssam UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 303104478Ssam } 304104478Ssam 305104478Ssam cmd = pci_read_config(dev, PCIR_COMMAND, 4); 306104478Ssam cmd |= PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN; 307104478Ssam pci_write_config(dev, PCIR_COMMAND, cmd, 4); 308104478Ssam cmd = pci_read_config(dev, PCIR_COMMAND, 4); 309104478Ssam 310104478Ssam if (!(cmd & PCIM_CMD_MEMEN)) { 311104478Ssam device_printf(dev, "failed to enable memory mapping\n"); 312104478Ssam goto bad; 313104478Ssam } 314104478Ssam 315104478Ssam if (!(cmd & PCIM_CMD_BUSMASTEREN)) { 316104478Ssam device_printf(dev, "failed to enable bus mastering\n"); 317104478Ssam goto bad; 318104478Ssam } 319104478Ssam 320104478Ssam /* 321104478Ssam * Setup memory-mapping of PCI registers. 322104478Ssam */ 323104478Ssam rid = BS_BAR; 324127135Snjl sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 325127135Snjl RF_ACTIVE); 326104478Ssam if (sc->sc_sr == NULL) { 327104478Ssam device_printf(dev, "cannot map register space\n"); 328104478Ssam goto bad; 329104478Ssam } 330104478Ssam sc->sc_st = rman_get_bustag(sc->sc_sr); 331104478Ssam sc->sc_sh = rman_get_bushandle(sc->sc_sr); 332104478Ssam 333104478Ssam /* 334104478Ssam * Arrange interrupt line. 335104478Ssam */ 336104478Ssam rid = 0; 337127135Snjl sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 338127135Snjl RF_SHAREABLE|RF_ACTIVE); 339104478Ssam if (sc->sc_irq == NULL) { 340104478Ssam device_printf(dev, "could not map interrupt\n"); 341108823Ssam goto bad1; 342104478Ssam } 343104478Ssam /* 344104478Ssam * NB: Network code assumes we are blocked with splimp() 345104478Ssam * so make sure the IRQ is mapped appropriately. 346104478Ssam */ 347115747Ssam if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE, 348104478Ssam ubsec_intr, sc, &sc->sc_ih)) { 349104478Ssam device_printf(dev, "could not establish interrupt\n"); 350108823Ssam goto bad2; 351104478Ssam } 352104478Ssam 353104478Ssam sc->sc_cid = crypto_get_driverid(0); 354104478Ssam if (sc->sc_cid < 0) { 355104478Ssam device_printf(dev, "could not get crypto driver id\n"); 356108823Ssam goto bad3; 357104478Ssam } 358104478Ssam 359104478Ssam /* 360104478Ssam * Setup DMA descriptor area. 361104478Ssam */ 362104478Ssam if (bus_dma_tag_create(NULL, /* parent */ 363104478Ssam 1, 0, /* alignment, bounds */ 364104478Ssam BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 365104478Ssam BUS_SPACE_MAXADDR, /* highaddr */ 366104478Ssam NULL, NULL, /* filter, filterarg */ 367108823Ssam 0x3ffff, /* maxsize */ 368104478Ssam UBS_MAX_SCATTER, /* nsegments */ 369108823Ssam 0xffff, /* maxsegsize */ 370104478Ssam BUS_DMA_ALLOCNOW, /* flags */ 371117126Sscottl NULL, NULL, /* lockfunc, lockarg */ 372104478Ssam &sc->sc_dmat)) { 373104478Ssam device_printf(dev, "cannot allocate DMA tag\n"); 374108823Ssam goto bad4; 375104478Ssam } 376104478Ssam SIMPLEQ_INIT(&sc->sc_freequeue); 377104478Ssam dmap = sc->sc_dmaa; 378104478Ssam for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) { 379104478Ssam struct ubsec_q *q; 380104478Ssam 381104478Ssam q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q), 382104478Ssam M_DEVBUF, M_NOWAIT); 383104478Ssam if (q == NULL) { 384104478Ssam device_printf(dev, "cannot allocate queue buffers\n"); 385104478Ssam break; 386104478Ssam } 387104478Ssam 388104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk), 389104478Ssam &dmap->d_alloc, 0)) { 390104478Ssam device_printf(dev, "cannot allocate dma buffers\n"); 391104478Ssam free(q, M_DEVBUF); 392104478Ssam break; 393104478Ssam } 394104478Ssam dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr; 395104478Ssam 396104478Ssam q->q_dma = dmap; 397104478Ssam sc->sc_queuea[i] = q; 398104478Ssam 399104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 400104478Ssam } 401115747Ssam mtx_init(&sc->sc_mcr1lock, device_get_nameunit(dev), 402115747Ssam "mcr1 operations", MTX_DEF); 403115747Ssam mtx_init(&sc->sc_freeqlock, device_get_nameunit(dev), 404115747Ssam "mcr1 free q", MTX_DEF); 405104478Ssam 406104478Ssam device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc)); 407104478Ssam 408104478Ssam crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0, 409104478Ssam ubsec_newsession, ubsec_freesession, ubsec_process, sc); 410104478Ssam crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0, 411104478Ssam ubsec_newsession, ubsec_freesession, ubsec_process, sc); 412104478Ssam crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0, 413104478Ssam ubsec_newsession, ubsec_freesession, ubsec_process, sc); 414104478Ssam crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0, 415104478Ssam ubsec_newsession, ubsec_freesession, ubsec_process, sc); 416104478Ssam 417104478Ssam /* 418104478Ssam * Reset Broadcom chip 419104478Ssam */ 420104478Ssam ubsec_reset_board(sc); 421104478Ssam 422104478Ssam /* 423104478Ssam * Init Broadcom specific PCI settings 424104478Ssam */ 425104478Ssam ubsec_init_pciregs(dev); 426104478Ssam 427104478Ssam /* 428104478Ssam * Init Broadcom chip 429104478Ssam */ 430104478Ssam ubsec_init_board(sc); 431104478Ssam 432104478Ssam#ifndef UBSEC_NO_RNG 433104478Ssam if (sc->sc_flags & UBS_FLAGS_RNG) { 434104478Ssam sc->sc_statmask |= BS_STAT_MCR2_DONE; 435112124Ssam#ifdef UBSEC_RNDTEST 436112124Ssam sc->sc_rndtest = rndtest_attach(dev); 437112124Ssam if (sc->sc_rndtest) 438112124Ssam sc->sc_harvest = rndtest_harvest; 439112124Ssam else 440112124Ssam sc->sc_harvest = default_harvest; 441112124Ssam#else 442112124Ssam sc->sc_harvest = default_harvest; 443112124Ssam#endif 444104478Ssam 445104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 446104478Ssam &sc->sc_rng.rng_q.q_mcr, 0)) 447104478Ssam goto skip_rng; 448104478Ssam 449104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass), 450104478Ssam &sc->sc_rng.rng_q.q_ctx, 0)) { 451104478Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 452104478Ssam goto skip_rng; 453104478Ssam } 454104478Ssam 455104478Ssam if (ubsec_dma_malloc(sc, sizeof(u_int32_t) * 456104478Ssam UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) { 457104478Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 458104478Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 459104478Ssam goto skip_rng; 460104478Ssam } 461104478Ssam 462104478Ssam if (hz >= 100) 463104478Ssam sc->sc_rnghz = hz / 100; 464104478Ssam else 465104478Ssam sc->sc_rnghz = 1; 466119137Ssam callout_init(&sc->sc_rngto, CALLOUT_MPSAFE); 467104478Ssam callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 468104478Ssamskip_rng: 469104478Ssam ; 470104478Ssam } 471104478Ssam#endif /* UBSEC_NO_RNG */ 472115747Ssam mtx_init(&sc->sc_mcr2lock, device_get_nameunit(dev), 473115747Ssam "mcr2 operations", MTX_DEF); 474104478Ssam 475104478Ssam if (sc->sc_flags & UBS_FLAGS_KEY) { 476104478Ssam sc->sc_statmask |= BS_STAT_MCR2_DONE; 477104478Ssam 478104478Ssam crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0, 479104478Ssam ubsec_kprocess, sc); 480104630Ssam#if 0 481104478Ssam crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0, 482104478Ssam ubsec_kprocess, sc); 483104630Ssam#endif 484104478Ssam } 485104478Ssam return (0); 486108823Ssambad4: 487108823Ssam crypto_unregister_all(sc->sc_cid); 488108823Ssambad3: 489108823Ssam bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 490108823Ssambad2: 491108823Ssam bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 492108823Ssambad1: 493108823Ssam bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 494104478Ssambad: 495104478Ssam return (ENXIO); 496104478Ssam} 497104478Ssam 498104478Ssam/* 499104478Ssam * Detach a device that successfully probed. 500104478Ssam */ 501104478Ssamstatic int 502104478Ssamubsec_detach(device_t dev) 503104478Ssam{ 504104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 505104478Ssam 506108823Ssam /* XXX wait/abort active ops */ 507108823Ssam 508115747Ssam /* disable interrupts */ 509115747Ssam WRITE_REG(sc, BS_CTRL, READ_REG(sc, BS_CTRL) &~ 510115747Ssam (BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR)); 511104478Ssam 512104478Ssam callout_stop(&sc->sc_rngto); 513104478Ssam 514104478Ssam crypto_unregister_all(sc->sc_cid); 515104478Ssam 516112124Ssam#ifdef UBSEC_RNDTEST 517112124Ssam if (sc->sc_rndtest) 518112124Ssam rndtest_detach(sc->sc_rndtest); 519112124Ssam#endif 520112124Ssam 521108823Ssam while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 522108823Ssam struct ubsec_q *q; 523108823Ssam 524108823Ssam q = SIMPLEQ_FIRST(&sc->sc_freequeue); 525108823Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q, q_next); 526108823Ssam ubsec_dma_free(sc, &q->q_dma->d_alloc); 527108823Ssam free(q, M_DEVBUF); 528108823Ssam } 529115747Ssam mtx_destroy(&sc->sc_mcr1lock); 530159224Spjd mtx_destroy(&sc->sc_freeqlock); 531108823Ssam#ifndef UBSEC_NO_RNG 532108823Ssam if (sc->sc_flags & UBS_FLAGS_RNG) { 533108823Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 534108823Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 535108823Ssam ubsec_dma_free(sc, &sc->sc_rng.rng_buf); 536108823Ssam } 537108823Ssam#endif /* UBSEC_NO_RNG */ 538115747Ssam mtx_destroy(&sc->sc_mcr2lock); 539108823Ssam 540104478Ssam bus_generic_detach(dev); 541104478Ssam bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 542104478Ssam bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 543104478Ssam 544104478Ssam bus_dma_tag_destroy(sc->sc_dmat); 545104478Ssam bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 546104478Ssam 547104478Ssam return (0); 548104478Ssam} 549104478Ssam 550104478Ssam/* 551104478Ssam * Stop all chip i/o so that the kernel's probe routines don't 552104478Ssam * get confused by errant DMAs when rebooting. 553104478Ssam */ 554104478Ssamstatic void 555104478Ssamubsec_shutdown(device_t dev) 556104478Ssam{ 557104478Ssam#ifdef notyet 558104478Ssam ubsec_stop(device_get_softc(dev)); 559104478Ssam#endif 560104478Ssam} 561104478Ssam 562104478Ssam/* 563104478Ssam * Device suspend routine. 564104478Ssam */ 565104478Ssamstatic int 566104478Ssamubsec_suspend(device_t dev) 567104478Ssam{ 568104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 569104478Ssam 570104478Ssam#ifdef notyet 571104478Ssam /* XXX stop the device and save PCI settings */ 572104478Ssam#endif 573104478Ssam sc->sc_suspended = 1; 574104478Ssam 575104478Ssam return (0); 576104478Ssam} 577104478Ssam 578104478Ssamstatic int 579104478Ssamubsec_resume(device_t dev) 580104478Ssam{ 581104478Ssam struct ubsec_softc *sc = device_get_softc(dev); 582104478Ssam 583104478Ssam#ifdef notyet 584104478Ssam /* XXX retore PCI settings and start the device */ 585104478Ssam#endif 586104478Ssam sc->sc_suspended = 0; 587104478Ssam return (0); 588104478Ssam} 589104478Ssam 590104478Ssam/* 591104478Ssam * UBSEC Interrupt routine 592104478Ssam */ 593104478Ssamstatic void 594104478Ssamubsec_intr(void *arg) 595104478Ssam{ 596104478Ssam struct ubsec_softc *sc = arg; 597104478Ssam volatile u_int32_t stat; 598104478Ssam struct ubsec_q *q; 599104478Ssam struct ubsec_dma *dmap; 600104478Ssam int npkts = 0, i; 601104478Ssam 602104478Ssam stat = READ_REG(sc, BS_STAT); 603104478Ssam stat &= sc->sc_statmask; 604115747Ssam if (stat == 0) 605104478Ssam return; 606104478Ssam 607104478Ssam WRITE_REG(sc, BS_STAT, stat); /* IACK */ 608104478Ssam 609104478Ssam /* 610104478Ssam * Check to see if we have any packets waiting for us 611104478Ssam */ 612104478Ssam if ((stat & BS_STAT_MCR1_DONE)) { 613115747Ssam mtx_lock(&sc->sc_mcr1lock); 614104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 615104478Ssam q = SIMPLEQ_FIRST(&sc->sc_qchip); 616104478Ssam dmap = q->q_dma; 617104478Ssam 618104478Ssam if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0) 619104478Ssam break; 620104478Ssam 621104478Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next); 622104478Ssam 623104478Ssam npkts = q->q_nstacked_mcrs; 624108471Ssam sc->sc_nqchip -= 1+npkts; 625104478Ssam /* 626104478Ssam * search for further sc_qchip ubsec_q's that share 627104478Ssam * the same MCR, and complete them too, they must be 628104478Ssam * at the top. 629104478Ssam */ 630104478Ssam for (i = 0; i < npkts; i++) { 631104478Ssam if(q->q_stacked_mcr[i]) { 632104478Ssam ubsec_callback(sc, q->q_stacked_mcr[i]); 633104478Ssam } else { 634104478Ssam break; 635104478Ssam } 636104478Ssam } 637104478Ssam ubsec_callback(sc, q); 638104478Ssam } 639104478Ssam /* 640104478Ssam * Don't send any more packet to chip if there has been 641104478Ssam * a DMAERR. 642104478Ssam */ 643104478Ssam if (!(stat & BS_STAT_DMAERR)) 644104478Ssam ubsec_feed(sc); 645115747Ssam mtx_unlock(&sc->sc_mcr1lock); 646104478Ssam } 647104478Ssam 648104478Ssam /* 649104478Ssam * Check to see if we have any key setups/rng's waiting for us 650104478Ssam */ 651104478Ssam if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) && 652104478Ssam (stat & BS_STAT_MCR2_DONE)) { 653104478Ssam struct ubsec_q2 *q2; 654104478Ssam struct ubsec_mcr *mcr; 655104478Ssam 656115747Ssam mtx_lock(&sc->sc_mcr2lock); 657104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) { 658104478Ssam q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 659104478Ssam 660108823Ssam ubsec_dma_sync(&q2->q_mcr, 661104478Ssam BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 662104478Ssam 663104478Ssam mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 664104478Ssam if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) { 665108823Ssam ubsec_dma_sync(&q2->q_mcr, 666104478Ssam BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 667104478Ssam break; 668104478Ssam } 669104478Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q2, q_next); 670104478Ssam ubsec_callback2(sc, q2); 671104478Ssam /* 672104478Ssam * Don't send any more packet to chip if there has been 673104478Ssam * a DMAERR. 674104478Ssam */ 675104478Ssam if (!(stat & BS_STAT_DMAERR)) 676104478Ssam ubsec_feed2(sc); 677104478Ssam } 678115747Ssam mtx_unlock(&sc->sc_mcr2lock); 679104478Ssam } 680104478Ssam 681104478Ssam /* 682104478Ssam * Check to see if we got any DMA Error 683104478Ssam */ 684104478Ssam if (stat & BS_STAT_DMAERR) { 685104478Ssam#ifdef UBSEC_DEBUG 686104478Ssam if (ubsec_debug) { 687104478Ssam volatile u_int32_t a = READ_REG(sc, BS_ERR); 688104478Ssam 689104478Ssam printf("dmaerr %s@%08x\n", 690104478Ssam (a & BS_ERR_READ) ? "read" : "write", 691104478Ssam a & BS_ERR_ADDR); 692104478Ssam } 693104478Ssam#endif /* UBSEC_DEBUG */ 694104478Ssam ubsecstats.hst_dmaerr++; 695115747Ssam mtx_lock(&sc->sc_mcr1lock); 696104478Ssam ubsec_totalreset(sc); 697104478Ssam ubsec_feed(sc); 698115747Ssam mtx_unlock(&sc->sc_mcr1lock); 699104478Ssam } 700104478Ssam 701104478Ssam if (sc->sc_needwakeup) { /* XXX check high watermark */ 702158828Spjd int wakeup; 703158828Spjd 704158828Spjd mtx_lock(&sc->sc_freeqlock); 705158828Spjd wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ); 706104478Ssam#ifdef UBSEC_DEBUG 707104478Ssam if (ubsec_debug) 708104478Ssam device_printf(sc->sc_dev, "wakeup crypto (%x)\n", 709104478Ssam sc->sc_needwakeup); 710104478Ssam#endif /* UBSEC_DEBUG */ 711104478Ssam sc->sc_needwakeup &= ~wakeup; 712158828Spjd mtx_unlock(&sc->sc_freeqlock); 713104478Ssam crypto_unblock(sc->sc_cid, wakeup); 714104478Ssam } 715104478Ssam} 716104478Ssam 717104478Ssam/* 718104478Ssam * ubsec_feed() - aggregate and post requests to chip 719104478Ssam */ 720111416Ssamstatic void 721104478Ssamubsec_feed(struct ubsec_softc *sc) 722104478Ssam{ 723104478Ssam struct ubsec_q *q, *q2; 724104478Ssam int npkts, i; 725104478Ssam void *v; 726104478Ssam u_int32_t stat; 727104478Ssam 728108471Ssam /* 729108471Ssam * Decide how many ops to combine in a single MCR. We cannot 730108471Ssam * aggregate more than UBS_MAX_AGGR because this is the number 731111416Ssam * of slots defined in the data structure. Note that 732111416Ssam * aggregation only happens if ops are marked batch'able. 733108471Ssam * Aggregating ops reduces the number of interrupts to the host 734108471Ssam * but also (potentially) increases the latency for processing 735108471Ssam * completed ops as we only get an interrupt when all aggregated 736108471Ssam * ops have completed. 737108471Ssam */ 738111416Ssam if (sc->sc_nqueue == 0) 739111416Ssam return; 740111416Ssam if (sc->sc_nqueue > 1) { 741111416Ssam npkts = 0; 742111416Ssam SIMPLEQ_FOREACH(q, &sc->sc_queue, q_next) { 743111416Ssam npkts++; 744111416Ssam if ((q->q_crp->crp_flags & CRYPTO_F_BATCH) == 0) 745111416Ssam break; 746111416Ssam } 747111416Ssam } else 748111416Ssam npkts = 1; 749111416Ssam /* 750111416Ssam * Check device status before going any further. 751111416Ssam */ 752104478Ssam if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 753108471Ssam if (stat & BS_STAT_DMAERR) { 754104478Ssam ubsec_totalreset(sc); 755104478Ssam ubsecstats.hst_dmaerr++; 756108471Ssam } else 757108471Ssam ubsecstats.hst_mcr1full++; 758111416Ssam return; 759104478Ssam } 760111416Ssam if (sc->sc_nqueue > ubsecstats.hst_maxqueue) 761111416Ssam ubsecstats.hst_maxqueue = sc->sc_nqueue; 762111416Ssam if (npkts > UBS_MAX_AGGR) 763111416Ssam npkts = UBS_MAX_AGGR; 764111416Ssam if (npkts < 2) /* special case 1 op */ 765111416Ssam goto feed1; 766104478Ssam 767111416Ssam ubsecstats.hst_totbatch += npkts-1; 768104478Ssam#ifdef UBSEC_DEBUG 769104478Ssam if (ubsec_debug) 770104478Ssam printf("merging %d records\n", npkts); 771104478Ssam#endif /* UBSEC_DEBUG */ 772104478Ssam 773104478Ssam q = SIMPLEQ_FIRST(&sc->sc_queue); 774104478Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next); 775104478Ssam --sc->sc_nqueue; 776104478Ssam 777104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE); 778104478Ssam if (q->q_dst_map != NULL) 779104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD); 780104478Ssam 781104478Ssam q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */ 782104478Ssam 783104478Ssam for (i = 0; i < q->q_nstacked_mcrs; i++) { 784104478Ssam q2 = SIMPLEQ_FIRST(&sc->sc_queue); 785104478Ssam bus_dmamap_sync(sc->sc_dmat, q2->q_src_map, 786104478Ssam BUS_DMASYNC_PREWRITE); 787104478Ssam if (q2->q_dst_map != NULL) 788104478Ssam bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map, 789104478Ssam BUS_DMASYNC_PREREAD); 790104478Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q2, q_next); 791104478Ssam --sc->sc_nqueue; 792104478Ssam 793104478Ssam v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) - 794104478Ssam sizeof(struct ubsec_mcr_add)); 795104478Ssam bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add)); 796104478Ssam q->q_stacked_mcr[i] = q2; 797104478Ssam } 798104478Ssam q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts); 799104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 800108471Ssam sc->sc_nqchip += npkts; 801108471Ssam if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 802108471Ssam ubsecstats.hst_maxqchip = sc->sc_nqchip; 803108823Ssam ubsec_dma_sync(&q->q_dma->d_alloc, 804104478Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 805104478Ssam WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 806104478Ssam offsetof(struct ubsec_dmachunk, d_mcr)); 807111416Ssam return; 808104478Ssamfeed1: 809111416Ssam q = SIMPLEQ_FIRST(&sc->sc_queue); 810104478Ssam 811111416Ssam bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE); 812111416Ssam if (q->q_dst_map != NULL) 813111416Ssam bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD); 814111416Ssam ubsec_dma_sync(&q->q_dma->d_alloc, 815111416Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 816104478Ssam 817111416Ssam WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 818111416Ssam offsetof(struct ubsec_dmachunk, d_mcr)); 819104478Ssam#ifdef UBSEC_DEBUG 820111416Ssam if (ubsec_debug) 821111416Ssam printf("feed1: q->chip %p %08x stat %08x\n", 822111416Ssam q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr), 823111416Ssam stat); 824104478Ssam#endif /* UBSEC_DEBUG */ 825111416Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next); 826111416Ssam --sc->sc_nqueue; 827111416Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 828111416Ssam sc->sc_nqchip++; 829108471Ssam if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 830108471Ssam ubsecstats.hst_maxqchip = sc->sc_nqchip; 831111416Ssam return; 832104478Ssam} 833104478Ssam 834159225Spjdstatic void 835159225Spjdubsec_setup_enckey(struct ubsec_session *ses, int algo, caddr_t key) 836159225Spjd{ 837159225Spjd 838159225Spjd /* Go ahead and compute key in ubsec's byte order */ 839159225Spjd if (algo == CRYPTO_DES_CBC) { 840159225Spjd bcopy(key, &ses->ses_deskey[0], 8); 841159225Spjd bcopy(key, &ses->ses_deskey[2], 8); 842159225Spjd bcopy(key, &ses->ses_deskey[4], 8); 843159225Spjd } else 844159225Spjd bcopy(key, ses->ses_deskey, 24); 845159225Spjd 846159225Spjd SWAP32(ses->ses_deskey[0]); 847159225Spjd SWAP32(ses->ses_deskey[1]); 848159225Spjd SWAP32(ses->ses_deskey[2]); 849159225Spjd SWAP32(ses->ses_deskey[3]); 850159225Spjd SWAP32(ses->ses_deskey[4]); 851159225Spjd SWAP32(ses->ses_deskey[5]); 852159225Spjd} 853159225Spjd 854159225Spjdstatic void 855159225Spjdubsec_setup_mackey(struct ubsec_session *ses, int algo, caddr_t key, int klen) 856159225Spjd{ 857159225Spjd MD5_CTX md5ctx; 858159225Spjd SHA1_CTX sha1ctx; 859159225Spjd int i; 860159225Spjd 861159225Spjd for (i = 0; i < klen; i++) 862159225Spjd key[i] ^= HMAC_IPAD_VAL; 863159225Spjd 864159225Spjd if (algo == CRYPTO_MD5_HMAC) { 865159225Spjd MD5Init(&md5ctx); 866159225Spjd MD5Update(&md5ctx, key, klen); 867159232Spjd MD5Update(&md5ctx, hmac_ipad_buffer, MD5_HMAC_BLOCK_LEN - klen); 868159225Spjd bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state)); 869159225Spjd } else { 870159225Spjd SHA1Init(&sha1ctx); 871159225Spjd SHA1Update(&sha1ctx, key, klen); 872159232Spjd SHA1Update(&sha1ctx, hmac_ipad_buffer, 873159232Spjd SHA1_HMAC_BLOCK_LEN - klen); 874159225Spjd bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32)); 875159225Spjd } 876159225Spjd 877159225Spjd for (i = 0; i < klen; i++) 878159225Spjd key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 879159225Spjd 880159225Spjd if (algo == CRYPTO_MD5_HMAC) { 881159225Spjd MD5Init(&md5ctx); 882159225Spjd MD5Update(&md5ctx, key, klen); 883159232Spjd MD5Update(&md5ctx, hmac_opad_buffer, MD5_HMAC_BLOCK_LEN - klen); 884159225Spjd bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state)); 885159225Spjd } else { 886159225Spjd SHA1Init(&sha1ctx); 887159225Spjd SHA1Update(&sha1ctx, key, klen); 888159232Spjd SHA1Update(&sha1ctx, hmac_opad_buffer, 889159232Spjd SHA1_HMAC_BLOCK_LEN - klen); 890159225Spjd bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32)); 891159225Spjd } 892159225Spjd 893159225Spjd for (i = 0; i < klen; i++) 894159225Spjd key[i] ^= HMAC_OPAD_VAL; 895159225Spjd} 896159225Spjd 897104478Ssam/* 898104478Ssam * Allocate a new 'session' and return an encoded session id. 'sidp' 899104478Ssam * contains our registration id, and should contain an encoded session 900104478Ssam * id on successful allocation. 901104478Ssam */ 902104478Ssamstatic int 903104478Ssamubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri) 904104478Ssam{ 905104478Ssam struct cryptoini *c, *encini = NULL, *macini = NULL; 906104478Ssam struct ubsec_softc *sc = arg; 907104478Ssam struct ubsec_session *ses = NULL; 908159225Spjd int sesn; 909104478Ssam 910104478Ssam if (sidp == NULL || cri == NULL || sc == NULL) 911104478Ssam return (EINVAL); 912104478Ssam 913104478Ssam for (c = cri; c != NULL; c = c->cri_next) { 914104478Ssam if (c->cri_alg == CRYPTO_MD5_HMAC || 915104478Ssam c->cri_alg == CRYPTO_SHA1_HMAC) { 916104478Ssam if (macini) 917104478Ssam return (EINVAL); 918104478Ssam macini = c; 919104478Ssam } else if (c->cri_alg == CRYPTO_DES_CBC || 920104478Ssam c->cri_alg == CRYPTO_3DES_CBC) { 921104478Ssam if (encini) 922104478Ssam return (EINVAL); 923104478Ssam encini = c; 924104478Ssam } else 925104478Ssam return (EINVAL); 926104478Ssam } 927104478Ssam if (encini == NULL && macini == NULL) 928104478Ssam return (EINVAL); 929104478Ssam 930104478Ssam if (sc->sc_sessions == NULL) { 931104478Ssam ses = sc->sc_sessions = (struct ubsec_session *)malloc( 932104478Ssam sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 933104478Ssam if (ses == NULL) 934104478Ssam return (ENOMEM); 935104478Ssam sesn = 0; 936104478Ssam sc->sc_nsessions = 1; 937104478Ssam } else { 938104478Ssam for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 939104478Ssam if (sc->sc_sessions[sesn].ses_used == 0) { 940104478Ssam ses = &sc->sc_sessions[sesn]; 941104478Ssam break; 942104478Ssam } 943104478Ssam } 944104478Ssam 945104478Ssam if (ses == NULL) { 946104478Ssam sesn = sc->sc_nsessions; 947104478Ssam ses = (struct ubsec_session *)malloc((sesn + 1) * 948104478Ssam sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 949104478Ssam if (ses == NULL) 950104478Ssam return (ENOMEM); 951104478Ssam bcopy(sc->sc_sessions, ses, sesn * 952104478Ssam sizeof(struct ubsec_session)); 953104478Ssam bzero(sc->sc_sessions, sesn * 954104478Ssam sizeof(struct ubsec_session)); 955104478Ssam free(sc->sc_sessions, M_DEVBUF); 956104478Ssam sc->sc_sessions = ses; 957104478Ssam ses = &sc->sc_sessions[sesn]; 958104478Ssam sc->sc_nsessions++; 959104478Ssam } 960104478Ssam } 961104478Ssam bzero(ses, sizeof(struct ubsec_session)); 962104478Ssam ses->ses_used = 1; 963115747Ssam 964104478Ssam if (encini) { 965104478Ssam /* get an IV, network byte order */ 966104478Ssam /* XXX may read fewer than requested */ 967104478Ssam read_random(ses->ses_iv, sizeof(ses->ses_iv)); 968104478Ssam 969159225Spjd if (encini->cri_key != NULL) { 970159225Spjd ubsec_setup_enckey(ses, encini->cri_alg, 971159225Spjd encini->cri_key); 972159225Spjd } 973104478Ssam } 974104478Ssam 975104478Ssam if (macini) { 976158705Spjd ses->ses_mlen = macini->cri_mlen; 977158705Spjd if (ses->ses_mlen == 0) { 978158705Spjd if (macini->cri_alg == CRYPTO_MD5_HMAC) 979159233Spjd ses->ses_mlen = MD5_HASH_LEN; 980158705Spjd else 981159233Spjd ses->ses_mlen = SHA1_HASH_LEN; 982158705Spjd } 983158705Spjd 984159225Spjd if (macini->cri_key != NULL) { 985159225Spjd ubsec_setup_mackey(ses, macini->cri_alg, 986159225Spjd macini->cri_key, macini->cri_klen / 8); 987104478Ssam } 988104478Ssam } 989104478Ssam 990104478Ssam *sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn); 991104478Ssam return (0); 992104478Ssam} 993104478Ssam 994104478Ssam/* 995104478Ssam * Deallocate a session. 996104478Ssam */ 997104478Ssamstatic int 998104478Ssamubsec_freesession(void *arg, u_int64_t tid) 999104478Ssam{ 1000104478Ssam struct ubsec_softc *sc = arg; 1001115747Ssam int session, ret; 1002116924Ssam u_int32_t sid = CRYPTO_SESID2LID(tid); 1003104478Ssam 1004104478Ssam if (sc == NULL) 1005104478Ssam return (EINVAL); 1006104478Ssam 1007104478Ssam session = UBSEC_SESSION(sid); 1008115747Ssam if (session < sc->sc_nsessions) { 1009115747Ssam bzero(&sc->sc_sessions[session], 1010115747Ssam sizeof(sc->sc_sessions[session])); 1011115747Ssam ret = 0; 1012115747Ssam } else 1013115747Ssam ret = EINVAL; 1014104478Ssam 1015115747Ssam return (ret); 1016104478Ssam} 1017104478Ssam 1018104478Ssamstatic void 1019104478Ssamubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error) 1020104478Ssam{ 1021104478Ssam struct ubsec_operand *op = arg; 1022104478Ssam 1023104478Ssam KASSERT(nsegs <= UBS_MAX_SCATTER, 1024104478Ssam ("Too many DMA segments returned when mapping operand")); 1025104478Ssam#ifdef UBSEC_DEBUG 1026104478Ssam if (ubsec_debug) 1027159341Spjd printf("ubsec_op_cb: mapsize %u nsegs %d error %d\n", 1028159341Spjd (u_int) mapsize, nsegs, error); 1029104478Ssam#endif 1030159341Spjd if (error != 0) 1031159341Spjd return; 1032104478Ssam op->mapsize = mapsize; 1033104478Ssam op->nsegs = nsegs; 1034104478Ssam bcopy(seg, op->segs, nsegs * sizeof (seg[0])); 1035104478Ssam} 1036104478Ssam 1037104478Ssamstatic int 1038104478Ssamubsec_process(void *arg, struct cryptop *crp, int hint) 1039104478Ssam{ 1040104478Ssam struct ubsec_q *q = NULL; 1041104478Ssam int err = 0, i, j, nicealign; 1042104478Ssam struct ubsec_softc *sc = arg; 1043104478Ssam struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 1044104478Ssam int encoffset = 0, macoffset = 0, cpskip, cpoffset; 1045104478Ssam int sskip, dskip, stheend, dtheend; 1046104478Ssam int16_t coffset; 1047104478Ssam struct ubsec_session *ses; 1048104478Ssam struct ubsec_pktctx ctx; 1049104478Ssam struct ubsec_dma *dmap = NULL; 1050104478Ssam 1051104478Ssam if (crp == NULL || crp->crp_callback == NULL || sc == NULL) { 1052104478Ssam ubsecstats.hst_invalid++; 1053104478Ssam return (EINVAL); 1054104478Ssam } 1055104478Ssam if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) { 1056108471Ssam ubsecstats.hst_badsession++; 1057104478Ssam return (EINVAL); 1058104478Ssam } 1059104478Ssam 1060115747Ssam mtx_lock(&sc->sc_freeqlock); 1061104478Ssam if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 1062104478Ssam ubsecstats.hst_queuefull++; 1063104478Ssam sc->sc_needwakeup |= CRYPTO_SYMQ; 1064115747Ssam mtx_unlock(&sc->sc_freeqlock); 1065104478Ssam return (ERESTART); 1066104478Ssam } 1067104478Ssam q = SIMPLEQ_FIRST(&sc->sc_freequeue); 1068104478Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q, q_next); 1069115747Ssam mtx_unlock(&sc->sc_freeqlock); 1070104478Ssam 1071104478Ssam dmap = q->q_dma; /* Save dma pointer */ 1072104478Ssam bzero(q, sizeof(struct ubsec_q)); 1073104478Ssam bzero(&ctx, sizeof(ctx)); 1074104478Ssam 1075104478Ssam q->q_sesn = UBSEC_SESSION(crp->crp_sid); 1076104478Ssam q->q_dma = dmap; 1077104478Ssam ses = &sc->sc_sessions[q->q_sesn]; 1078104478Ssam 1079104478Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) { 1080104478Ssam q->q_src_m = (struct mbuf *)crp->crp_buf; 1081104478Ssam q->q_dst_m = (struct mbuf *)crp->crp_buf; 1082104478Ssam } else if (crp->crp_flags & CRYPTO_F_IOV) { 1083104478Ssam q->q_src_io = (struct uio *)crp->crp_buf; 1084104478Ssam q->q_dst_io = (struct uio *)crp->crp_buf; 1085104478Ssam } else { 1086108471Ssam ubsecstats.hst_badflags++; 1087104478Ssam err = EINVAL; 1088104478Ssam goto errout; /* XXX we don't handle contiguous blocks! */ 1089104478Ssam } 1090104478Ssam 1091104478Ssam bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr)); 1092104478Ssam 1093104478Ssam dmap->d_dma->d_mcr.mcr_pkts = htole16(1); 1094104478Ssam dmap->d_dma->d_mcr.mcr_flags = 0; 1095104478Ssam q->q_crp = crp; 1096104478Ssam 1097104478Ssam crd1 = crp->crp_desc; 1098104478Ssam if (crd1 == NULL) { 1099108471Ssam ubsecstats.hst_nodesc++; 1100104478Ssam err = EINVAL; 1101104478Ssam goto errout; 1102104478Ssam } 1103104478Ssam crd2 = crd1->crd_next; 1104104478Ssam 1105104478Ssam if (crd2 == NULL) { 1106104478Ssam if (crd1->crd_alg == CRYPTO_MD5_HMAC || 1107104478Ssam crd1->crd_alg == CRYPTO_SHA1_HMAC) { 1108104478Ssam maccrd = crd1; 1109104478Ssam enccrd = NULL; 1110104478Ssam } else if (crd1->crd_alg == CRYPTO_DES_CBC || 1111104478Ssam crd1->crd_alg == CRYPTO_3DES_CBC) { 1112104478Ssam maccrd = NULL; 1113104478Ssam enccrd = crd1; 1114104478Ssam } else { 1115108471Ssam ubsecstats.hst_badalg++; 1116104478Ssam err = EINVAL; 1117104478Ssam goto errout; 1118104478Ssam } 1119104478Ssam } else { 1120104478Ssam if ((crd1->crd_alg == CRYPTO_MD5_HMAC || 1121104478Ssam crd1->crd_alg == CRYPTO_SHA1_HMAC) && 1122104478Ssam (crd2->crd_alg == CRYPTO_DES_CBC || 1123104478Ssam crd2->crd_alg == CRYPTO_3DES_CBC) && 1124104478Ssam ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 1125104478Ssam maccrd = crd1; 1126104478Ssam enccrd = crd2; 1127104478Ssam } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 1128104478Ssam crd1->crd_alg == CRYPTO_3DES_CBC) && 1129104478Ssam (crd2->crd_alg == CRYPTO_MD5_HMAC || 1130104478Ssam crd2->crd_alg == CRYPTO_SHA1_HMAC) && 1131104478Ssam (crd1->crd_flags & CRD_F_ENCRYPT)) { 1132104478Ssam enccrd = crd1; 1133104478Ssam maccrd = crd2; 1134104478Ssam } else { 1135104478Ssam /* 1136104478Ssam * We cannot order the ubsec as requested 1137104478Ssam */ 1138108471Ssam ubsecstats.hst_badalg++; 1139104478Ssam err = EINVAL; 1140104478Ssam goto errout; 1141104478Ssam } 1142104478Ssam } 1143104478Ssam 1144104478Ssam if (enccrd) { 1145159225Spjd if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) { 1146159225Spjd ubsec_setup_enckey(ses, enccrd->crd_alg, 1147159225Spjd enccrd->crd_key); 1148159225Spjd } 1149159225Spjd 1150104478Ssam encoffset = enccrd->crd_skip; 1151104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES); 1152104478Ssam 1153104478Ssam if (enccrd->crd_flags & CRD_F_ENCRYPT) { 1154104478Ssam q->q_flags |= UBSEC_QFLAGS_COPYOUTIV; 1155104478Ssam 1156104478Ssam if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1157104478Ssam bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1158104478Ssam else { 1159104478Ssam ctx.pc_iv[0] = ses->ses_iv[0]; 1160104478Ssam ctx.pc_iv[1] = ses->ses_iv[1]; 1161104478Ssam } 1162104478Ssam 1163104478Ssam if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { 1164159242Spjd crypto_copyback(crp->crp_flags, crp->crp_buf, 1165159242Spjd enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv); 1166104478Ssam } 1167104478Ssam } else { 1168104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND); 1169104478Ssam 1170104478Ssam if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1171104478Ssam bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1172159242Spjd else { 1173159242Spjd crypto_copydata(crp->crp_flags, crp->crp_buf, 1174159242Spjd enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv); 1175159242Spjd } 1176104478Ssam } 1177104478Ssam 1178104478Ssam ctx.pc_deskey[0] = ses->ses_deskey[0]; 1179104478Ssam ctx.pc_deskey[1] = ses->ses_deskey[1]; 1180104478Ssam ctx.pc_deskey[2] = ses->ses_deskey[2]; 1181104478Ssam ctx.pc_deskey[3] = ses->ses_deskey[3]; 1182104478Ssam ctx.pc_deskey[4] = ses->ses_deskey[4]; 1183104478Ssam ctx.pc_deskey[5] = ses->ses_deskey[5]; 1184104478Ssam SWAP32(ctx.pc_iv[0]); 1185104478Ssam SWAP32(ctx.pc_iv[1]); 1186104478Ssam } 1187104478Ssam 1188104478Ssam if (maccrd) { 1189159225Spjd if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) { 1190159225Spjd ubsec_setup_mackey(ses, maccrd->crd_alg, 1191159225Spjd maccrd->crd_key, maccrd->crd_klen / 8); 1192159225Spjd } 1193159225Spjd 1194104478Ssam macoffset = maccrd->crd_skip; 1195104478Ssam 1196104478Ssam if (maccrd->crd_alg == CRYPTO_MD5_HMAC) 1197104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5); 1198104478Ssam else 1199104478Ssam ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1); 1200104478Ssam 1201104478Ssam for (i = 0; i < 5; i++) { 1202104478Ssam ctx.pc_hminner[i] = ses->ses_hminner[i]; 1203104478Ssam ctx.pc_hmouter[i] = ses->ses_hmouter[i]; 1204104478Ssam 1205104478Ssam HTOLE32(ctx.pc_hminner[i]); 1206104478Ssam HTOLE32(ctx.pc_hmouter[i]); 1207104478Ssam } 1208104478Ssam } 1209104478Ssam 1210104478Ssam if (enccrd && maccrd) { 1211104478Ssam /* 1212104478Ssam * ubsec cannot handle packets where the end of encryption 1213104478Ssam * and authentication are not the same, or where the 1214104478Ssam * encrypted part begins before the authenticated part. 1215104478Ssam */ 1216104478Ssam if ((encoffset + enccrd->crd_len) != 1217104478Ssam (macoffset + maccrd->crd_len)) { 1218104478Ssam ubsecstats.hst_lenmismatch++; 1219104478Ssam err = EINVAL; 1220104478Ssam goto errout; 1221104478Ssam } 1222104478Ssam if (enccrd->crd_skip < maccrd->crd_skip) { 1223104478Ssam ubsecstats.hst_skipmismatch++; 1224104478Ssam err = EINVAL; 1225104478Ssam goto errout; 1226104478Ssam } 1227104478Ssam sskip = maccrd->crd_skip; 1228104478Ssam cpskip = dskip = enccrd->crd_skip; 1229104478Ssam stheend = maccrd->crd_len; 1230104478Ssam dtheend = enccrd->crd_len; 1231104478Ssam coffset = enccrd->crd_skip - maccrd->crd_skip; 1232104478Ssam cpoffset = cpskip + dtheend; 1233104478Ssam#ifdef UBSEC_DEBUG 1234104478Ssam if (ubsec_debug) { 1235104478Ssam printf("mac: skip %d, len %d, inject %d\n", 1236104478Ssam maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject); 1237104478Ssam printf("enc: skip %d, len %d, inject %d\n", 1238104478Ssam enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject); 1239104478Ssam printf("src: skip %d, len %d\n", sskip, stheend); 1240104478Ssam printf("dst: skip %d, len %d\n", dskip, dtheend); 1241104478Ssam printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n", 1242104478Ssam coffset, stheend, cpskip, cpoffset); 1243104478Ssam } 1244104478Ssam#endif 1245104478Ssam } else { 1246104478Ssam cpskip = dskip = sskip = macoffset + encoffset; 1247104478Ssam dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len; 1248104478Ssam cpoffset = cpskip + dtheend; 1249104478Ssam coffset = 0; 1250104478Ssam } 1251104478Ssam ctx.pc_offset = htole16(coffset >> 2); 1252104478Ssam 1253104478Ssam if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) { 1254104478Ssam ubsecstats.hst_nomap++; 1255104478Ssam err = ENOMEM; 1256104478Ssam goto errout; 1257104478Ssam } 1258104478Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) { 1259104478Ssam if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map, 1260104478Ssam q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1261104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1262104478Ssam q->q_src_map = NULL; 1263104478Ssam ubsecstats.hst_noload++; 1264104478Ssam err = ENOMEM; 1265104478Ssam goto errout; 1266104478Ssam } 1267104478Ssam } else if (crp->crp_flags & CRYPTO_F_IOV) { 1268104478Ssam if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map, 1269104478Ssam q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1270104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1271104478Ssam q->q_src_map = NULL; 1272104478Ssam ubsecstats.hst_noload++; 1273104478Ssam err = ENOMEM; 1274104478Ssam goto errout; 1275104478Ssam } 1276104478Ssam } 1277104478Ssam nicealign = ubsec_dmamap_aligned(&q->q_src); 1278104478Ssam 1279104478Ssam dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend); 1280104478Ssam 1281104478Ssam#ifdef UBSEC_DEBUG 1282104478Ssam if (ubsec_debug) 1283104478Ssam printf("src skip: %d nicealign: %u\n", sskip, nicealign); 1284104478Ssam#endif 1285104478Ssam for (i = j = 0; i < q->q_src_nsegs; i++) { 1286104478Ssam struct ubsec_pktbuf *pb; 1287104478Ssam bus_size_t packl = q->q_src_segs[i].ds_len; 1288104478Ssam bus_addr_t packp = q->q_src_segs[i].ds_addr; 1289104478Ssam 1290104478Ssam if (sskip >= packl) { 1291104478Ssam sskip -= packl; 1292104478Ssam continue; 1293104478Ssam } 1294104478Ssam 1295104478Ssam packl -= sskip; 1296104478Ssam packp += sskip; 1297104478Ssam sskip = 0; 1298104478Ssam 1299104478Ssam if (packl > 0xfffc) { 1300104478Ssam err = EIO; 1301104478Ssam goto errout; 1302104478Ssam } 1303104478Ssam 1304104478Ssam if (j == 0) 1305104478Ssam pb = &dmap->d_dma->d_mcr.mcr_ipktbuf; 1306104478Ssam else 1307104478Ssam pb = &dmap->d_dma->d_sbuf[j - 1]; 1308104478Ssam 1309104478Ssam pb->pb_addr = htole32(packp); 1310104478Ssam 1311104478Ssam if (stheend) { 1312104478Ssam if (packl > stheend) { 1313104478Ssam pb->pb_len = htole32(stheend); 1314104478Ssam stheend = 0; 1315104478Ssam } else { 1316104478Ssam pb->pb_len = htole32(packl); 1317104478Ssam stheend -= packl; 1318104478Ssam } 1319104478Ssam } else 1320104478Ssam pb->pb_len = htole32(packl); 1321104478Ssam 1322104478Ssam if ((i + 1) == q->q_src_nsegs) 1323104478Ssam pb->pb_next = 0; 1324104478Ssam else 1325104478Ssam pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1326104478Ssam offsetof(struct ubsec_dmachunk, d_sbuf[j])); 1327104478Ssam j++; 1328104478Ssam } 1329104478Ssam 1330104478Ssam if (enccrd == NULL && maccrd != NULL) { 1331104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0; 1332104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0; 1333104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr + 1334104478Ssam offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1335104478Ssam#ifdef UBSEC_DEBUG 1336104478Ssam if (ubsec_debug) 1337104478Ssam printf("opkt: %x %x %x\n", 1338104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr, 1339104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_len, 1340104478Ssam dmap->d_dma->d_mcr.mcr_opktbuf.pb_next); 1341104478Ssam#endif 1342104478Ssam } else { 1343104478Ssam if (crp->crp_flags & CRYPTO_F_IOV) { 1344104478Ssam if (!nicealign) { 1345104478Ssam ubsecstats.hst_iovmisaligned++; 1346104478Ssam err = EINVAL; 1347104478Ssam goto errout; 1348104478Ssam } 1349104478Ssam if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, 1350104478Ssam &q->q_dst_map)) { 1351104478Ssam ubsecstats.hst_nomap++; 1352104478Ssam err = ENOMEM; 1353104478Ssam goto errout; 1354104478Ssam } 1355104478Ssam if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map, 1356104478Ssam q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) { 1357104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1358104478Ssam q->q_dst_map = NULL; 1359104478Ssam ubsecstats.hst_noload++; 1360104478Ssam err = ENOMEM; 1361104478Ssam goto errout; 1362104478Ssam } 1363104478Ssam } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1364104478Ssam if (nicealign) { 1365104478Ssam q->q_dst = q->q_src; 1366104478Ssam } else { 1367104478Ssam int totlen, len; 1368104478Ssam struct mbuf *m, *top, **mp; 1369104478Ssam 1370104478Ssam ubsecstats.hst_unaligned++; 1371104478Ssam totlen = q->q_src_mapsize; 1372160931Sjhb if (totlen >= MINCLSIZE) { 1373160931Sjhb m = m_getcl(M_DONTWAIT, MT_DATA, 1374160931Sjhb q->q_src_m->m_flags & M_PKTHDR); 1375160931Sjhb len = MCLBYTES; 1376160931Sjhb } else if (q->q_src_m->m_flags & M_PKTHDR) { 1377160931Sjhb m = m_gethdr(M_DONTWAIT, MT_DATA); 1378104478Ssam len = MHLEN; 1379104478Ssam } else { 1380160931Sjhb m = m_get(M_DONTWAIT, MT_DATA); 1381104478Ssam len = MLEN; 1382104478Ssam } 1383160931Sjhb if (m && q->q_src_m->m_flags & M_PKTHDR && 1384160931Sjhb !m_dup_pkthdr(m, q->q_src_m, M_DONTWAIT)) { 1385160931Sjhb m_free(m); 1386160931Sjhb m = NULL; 1387160931Sjhb } 1388104478Ssam if (m == NULL) { 1389104478Ssam ubsecstats.hst_nombuf++; 1390104478Ssam err = sc->sc_nqueue ? ERESTART : ENOMEM; 1391104478Ssam goto errout; 1392104478Ssam } 1393160931Sjhb m->m_len = len = min(totlen, len); 1394160931Sjhb totlen -= len; 1395160931Sjhb top = m; 1396104478Ssam mp = ⊤ 1397104478Ssam 1398104478Ssam while (totlen > 0) { 1399160931Sjhb if (totlen >= MINCLSIZE) { 1400160931Sjhb m = m_getcl(M_DONTWAIT, 1401160931Sjhb MT_DATA, 0); 1402160931Sjhb len = MCLBYTES; 1403160931Sjhb } else { 1404160931Sjhb m = m_get(M_DONTWAIT, MT_DATA); 1405104478Ssam len = MLEN; 1406104478Ssam } 1407160931Sjhb if (m == NULL) { 1408160931Sjhb m_freem(top); 1409160931Sjhb ubsecstats.hst_nombuf++; 1410160931Sjhb err = sc->sc_nqueue ? ERESTART : ENOMEM; 1411160931Sjhb goto errout; 1412104478Ssam } 1413104478Ssam m->m_len = len = min(totlen, len); 1414104478Ssam totlen -= len; 1415104478Ssam *mp = m; 1416104478Ssam mp = &m->m_next; 1417104478Ssam } 1418104478Ssam q->q_dst_m = top; 1419104478Ssam ubsec_mcopy(q->q_src_m, q->q_dst_m, 1420104478Ssam cpskip, cpoffset); 1421104478Ssam if (bus_dmamap_create(sc->sc_dmat, 1422104478Ssam BUS_DMA_NOWAIT, &q->q_dst_map) != 0) { 1423104478Ssam ubsecstats.hst_nomap++; 1424104478Ssam err = ENOMEM; 1425104478Ssam goto errout; 1426104478Ssam } 1427104478Ssam if (bus_dmamap_load_mbuf(sc->sc_dmat, 1428104478Ssam q->q_dst_map, q->q_dst_m, 1429104478Ssam ubsec_op_cb, &q->q_dst, 1430104478Ssam BUS_DMA_NOWAIT) != 0) { 1431104478Ssam bus_dmamap_destroy(sc->sc_dmat, 1432104478Ssam q->q_dst_map); 1433104478Ssam q->q_dst_map = NULL; 1434104478Ssam ubsecstats.hst_noload++; 1435104478Ssam err = ENOMEM; 1436104478Ssam goto errout; 1437104478Ssam } 1438104478Ssam } 1439104478Ssam } else { 1440108471Ssam ubsecstats.hst_badflags++; 1441104478Ssam err = EINVAL; 1442104478Ssam goto errout; 1443104478Ssam } 1444104478Ssam 1445104478Ssam#ifdef UBSEC_DEBUG 1446104478Ssam if (ubsec_debug) 1447104478Ssam printf("dst skip: %d\n", dskip); 1448104478Ssam#endif 1449104478Ssam for (i = j = 0; i < q->q_dst_nsegs; i++) { 1450104478Ssam struct ubsec_pktbuf *pb; 1451104478Ssam bus_size_t packl = q->q_dst_segs[i].ds_len; 1452104478Ssam bus_addr_t packp = q->q_dst_segs[i].ds_addr; 1453104478Ssam 1454104478Ssam if (dskip >= packl) { 1455104478Ssam dskip -= packl; 1456104478Ssam continue; 1457104478Ssam } 1458104478Ssam 1459104478Ssam packl -= dskip; 1460104478Ssam packp += dskip; 1461104478Ssam dskip = 0; 1462104478Ssam 1463104478Ssam if (packl > 0xfffc) { 1464104478Ssam err = EIO; 1465104478Ssam goto errout; 1466104478Ssam } 1467104478Ssam 1468104478Ssam if (j == 0) 1469104478Ssam pb = &dmap->d_dma->d_mcr.mcr_opktbuf; 1470104478Ssam else 1471104478Ssam pb = &dmap->d_dma->d_dbuf[j - 1]; 1472104478Ssam 1473104478Ssam pb->pb_addr = htole32(packp); 1474104478Ssam 1475104478Ssam if (dtheend) { 1476104478Ssam if (packl > dtheend) { 1477104478Ssam pb->pb_len = htole32(dtheend); 1478104478Ssam dtheend = 0; 1479104478Ssam } else { 1480104478Ssam pb->pb_len = htole32(packl); 1481104478Ssam dtheend -= packl; 1482104478Ssam } 1483104478Ssam } else 1484104478Ssam pb->pb_len = htole32(packl); 1485104478Ssam 1486104478Ssam if ((i + 1) == q->q_dst_nsegs) { 1487104478Ssam if (maccrd) 1488104478Ssam pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1489104478Ssam offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1490104478Ssam else 1491104478Ssam pb->pb_next = 0; 1492104478Ssam } else 1493104478Ssam pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1494104478Ssam offsetof(struct ubsec_dmachunk, d_dbuf[j])); 1495104478Ssam j++; 1496104478Ssam } 1497104478Ssam } 1498104478Ssam 1499104478Ssam dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr + 1500104478Ssam offsetof(struct ubsec_dmachunk, d_ctx)); 1501104478Ssam 1502104478Ssam if (sc->sc_flags & UBS_FLAGS_LONGCTX) { 1503104478Ssam struct ubsec_pktctx_long *ctxl; 1504104478Ssam 1505104478Ssam ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr + 1506104478Ssam offsetof(struct ubsec_dmachunk, d_ctx)); 1507104478Ssam 1508104478Ssam /* transform small context into long context */ 1509104478Ssam ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long)); 1510104478Ssam ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC); 1511104478Ssam ctxl->pc_flags = ctx.pc_flags; 1512104478Ssam ctxl->pc_offset = ctx.pc_offset; 1513104478Ssam for (i = 0; i < 6; i++) 1514104478Ssam ctxl->pc_deskey[i] = ctx.pc_deskey[i]; 1515104478Ssam for (i = 0; i < 5; i++) 1516104478Ssam ctxl->pc_hminner[i] = ctx.pc_hminner[i]; 1517104478Ssam for (i = 0; i < 5; i++) 1518104478Ssam ctxl->pc_hmouter[i] = ctx.pc_hmouter[i]; 1519104478Ssam ctxl->pc_iv[0] = ctx.pc_iv[0]; 1520104478Ssam ctxl->pc_iv[1] = ctx.pc_iv[1]; 1521104478Ssam } else 1522104478Ssam bcopy(&ctx, dmap->d_alloc.dma_vaddr + 1523104478Ssam offsetof(struct ubsec_dmachunk, d_ctx), 1524104478Ssam sizeof(struct ubsec_pktctx)); 1525104478Ssam 1526115747Ssam mtx_lock(&sc->sc_mcr1lock); 1527104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next); 1528104478Ssam sc->sc_nqueue++; 1529104478Ssam ubsecstats.hst_ipackets++; 1530104478Ssam ubsecstats.hst_ibytes += dmap->d_alloc.dma_size; 1531111416Ssam if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= UBS_MAX_AGGR) 1532104478Ssam ubsec_feed(sc); 1533115747Ssam mtx_unlock(&sc->sc_mcr1lock); 1534104478Ssam return (0); 1535104478Ssam 1536104478Ssamerrout: 1537104478Ssam if (q != NULL) { 1538104478Ssam if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 1539104478Ssam m_freem(q->q_dst_m); 1540104478Ssam 1541104478Ssam if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1542104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1543104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1544104478Ssam } 1545104478Ssam if (q->q_src_map != NULL) { 1546104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1547104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1548104478Ssam } 1549158828Spjd } 1550158828Spjd if (q != NULL || err == ERESTART) { 1551115747Ssam mtx_lock(&sc->sc_freeqlock); 1552158828Spjd if (q != NULL) 1553158828Spjd SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1554158828Spjd if (err == ERESTART) 1555158828Spjd sc->sc_needwakeup |= CRYPTO_SYMQ; 1556115747Ssam mtx_unlock(&sc->sc_freeqlock); 1557104478Ssam } 1558104478Ssam if (err != ERESTART) { 1559104478Ssam crp->crp_etype = err; 1560104478Ssam crypto_done(crp); 1561104478Ssam } 1562104478Ssam return (err); 1563104478Ssam} 1564104478Ssam 1565104478Ssamstatic void 1566104478Ssamubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q) 1567104478Ssam{ 1568104478Ssam struct cryptop *crp = (struct cryptop *)q->q_crp; 1569104478Ssam struct cryptodesc *crd; 1570104478Ssam struct ubsec_dma *dmap = q->q_dma; 1571104478Ssam 1572112099Ssam ubsecstats.hst_opackets++; 1573112099Ssam ubsecstats.hst_obytes += dmap->d_alloc.dma_size; 1574112099Ssam 1575108823Ssam ubsec_dma_sync(&dmap->d_alloc, 1576104478Ssam BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1577104478Ssam if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1578104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 1579104478Ssam BUS_DMASYNC_POSTREAD); 1580104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1581104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1582104478Ssam } 1583104478Ssam bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE); 1584104478Ssam bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1585104478Ssam bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1586104478Ssam 1587104478Ssam if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) { 1588104478Ssam m_freem(q->q_src_m); 1589104478Ssam crp->crp_buf = (caddr_t)q->q_dst_m; 1590104478Ssam } 1591104478Ssam 1592104478Ssam /* copy out IV for future use */ 1593104478Ssam if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) { 1594104478Ssam for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1595104478Ssam if (crd->crd_alg != CRYPTO_DES_CBC && 1596104478Ssam crd->crd_alg != CRYPTO_3DES_CBC) 1597104478Ssam continue; 1598159242Spjd crypto_copydata(crp->crp_flags, crp->crp_buf, 1599159242Spjd crd->crd_skip + crd->crd_len - 8, 8, 1600159242Spjd (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv); 1601104478Ssam break; 1602104478Ssam } 1603104478Ssam } 1604104478Ssam 1605104478Ssam for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1606104478Ssam if (crd->crd_alg != CRYPTO_MD5_HMAC && 1607104478Ssam crd->crd_alg != CRYPTO_SHA1_HMAC) 1608104478Ssam continue; 1609159242Spjd crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, 1610159242Spjd sc->sc_sessions[q->q_sesn].ses_mlen, 1611159242Spjd (caddr_t)dmap->d_dma->d_macbuf); 1612104478Ssam break; 1613104478Ssam } 1614115747Ssam mtx_lock(&sc->sc_freeqlock); 1615104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1616115747Ssam mtx_unlock(&sc->sc_freeqlock); 1617104478Ssam crypto_done(crp); 1618104478Ssam} 1619104478Ssam 1620104478Ssamstatic void 1621104478Ssamubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset) 1622104478Ssam{ 1623104478Ssam int i, j, dlen, slen; 1624104478Ssam caddr_t dptr, sptr; 1625104478Ssam 1626104478Ssam j = 0; 1627104478Ssam sptr = srcm->m_data; 1628104478Ssam slen = srcm->m_len; 1629104478Ssam dptr = dstm->m_data; 1630104478Ssam dlen = dstm->m_len; 1631104478Ssam 1632104478Ssam while (1) { 1633104478Ssam for (i = 0; i < min(slen, dlen); i++) { 1634104478Ssam if (j < hoffset || j >= toffset) 1635104478Ssam *dptr++ = *sptr++; 1636104478Ssam slen--; 1637104478Ssam dlen--; 1638104478Ssam j++; 1639104478Ssam } 1640104478Ssam if (slen == 0) { 1641104478Ssam srcm = srcm->m_next; 1642104478Ssam if (srcm == NULL) 1643104478Ssam return; 1644104478Ssam sptr = srcm->m_data; 1645104478Ssam slen = srcm->m_len; 1646104478Ssam } 1647104478Ssam if (dlen == 0) { 1648104478Ssam dstm = dstm->m_next; 1649104478Ssam if (dstm == NULL) 1650104478Ssam return; 1651104478Ssam dptr = dstm->m_data; 1652104478Ssam dlen = dstm->m_len; 1653104478Ssam } 1654104478Ssam } 1655104478Ssam} 1656104478Ssam 1657104478Ssam/* 1658104478Ssam * feed the key generator, must be called at splimp() or higher. 1659104478Ssam */ 1660104478Ssamstatic int 1661104478Ssamubsec_feed2(struct ubsec_softc *sc) 1662104478Ssam{ 1663104478Ssam struct ubsec_q2 *q; 1664104478Ssam 1665104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) { 1666104478Ssam if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL) 1667104478Ssam break; 1668104478Ssam q = SIMPLEQ_FIRST(&sc->sc_queue2); 1669104478Ssam 1670108823Ssam ubsec_dma_sync(&q->q_mcr, 1671104478Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1672108823Ssam ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_PREWRITE); 1673104478Ssam 1674104478Ssam WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr); 1675104478Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q, q_next); 1676104478Ssam --sc->sc_nqueue2; 1677104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next); 1678104478Ssam } 1679104478Ssam return (0); 1680104478Ssam} 1681104478Ssam 1682104478Ssam/* 1683104478Ssam * Callback for handling random numbers 1684104478Ssam */ 1685104478Ssamstatic void 1686104478Ssamubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q) 1687104478Ssam{ 1688104478Ssam struct cryptkop *krp; 1689104478Ssam struct ubsec_ctx_keyop *ctx; 1690104478Ssam 1691104478Ssam ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr; 1692108823Ssam ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_POSTWRITE); 1693104478Ssam 1694104478Ssam switch (q->q_type) { 1695104478Ssam#ifndef UBSEC_NO_RNG 1696104478Ssam case UBS_CTXOP_RNGBYPASS: { 1697104478Ssam struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q; 1698104478Ssam 1699108823Ssam ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_POSTREAD); 1700112124Ssam (*sc->sc_harvest)(sc->sc_rndtest, 1701112124Ssam rng->rng_buf.dma_vaddr, 1702112124Ssam UBSEC_RNG_BUFSIZ*sizeof (u_int32_t)); 1703104478Ssam rng->rng_used = 0; 1704104478Ssam callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1705104478Ssam break; 1706104478Ssam } 1707104478Ssam#endif 1708104478Ssam case UBS_CTXOP_MODEXP: { 1709104478Ssam struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 1710104478Ssam u_int rlen, clen; 1711104478Ssam 1712104478Ssam krp = me->me_krp; 1713104478Ssam rlen = (me->me_modbits + 7) / 8; 1714104630Ssam clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8; 1715104478Ssam 1716108823Ssam ubsec_dma_sync(&me->me_M, BUS_DMASYNC_POSTWRITE); 1717108823Ssam ubsec_dma_sync(&me->me_E, BUS_DMASYNC_POSTWRITE); 1718108823Ssam ubsec_dma_sync(&me->me_C, BUS_DMASYNC_POSTREAD); 1719108823Ssam ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_POSTWRITE); 1720104478Ssam 1721104478Ssam if (clen < rlen) 1722104478Ssam krp->krp_status = E2BIG; 1723104630Ssam else { 1724104630Ssam if (sc->sc_flags & UBS_FLAGS_HWNORM) { 1725104630Ssam bzero(krp->krp_param[krp->krp_iparams].crp_p, 1726104630Ssam (krp->krp_param[krp->krp_iparams].crp_nbits 1727104630Ssam + 7) / 8); 1728104630Ssam bcopy(me->me_C.dma_vaddr, 1729104630Ssam krp->krp_param[krp->krp_iparams].crp_p, 1730104630Ssam (me->me_modbits + 7) / 8); 1731104630Ssam } else 1732104630Ssam ubsec_kshift_l(me->me_shiftbits, 1733104630Ssam me->me_C.dma_vaddr, me->me_normbits, 1734104630Ssam krp->krp_param[krp->krp_iparams].crp_p, 1735104630Ssam krp->krp_param[krp->krp_iparams].crp_nbits); 1736104630Ssam } 1737104478Ssam 1738104478Ssam crypto_kdone(krp); 1739104478Ssam 1740104478Ssam /* bzero all potentially sensitive data */ 1741104478Ssam bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 1742104478Ssam bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 1743104478Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 1744104478Ssam bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 1745104478Ssam 1746104478Ssam /* Can't free here, so put us on the free list. */ 1747104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next); 1748104478Ssam break; 1749104478Ssam } 1750104478Ssam case UBS_CTXOP_RSAPRIV: { 1751104478Ssam struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 1752104478Ssam u_int len; 1753104478Ssam 1754104478Ssam krp = rp->rpr_krp; 1755108823Ssam ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_POSTWRITE); 1756108823Ssam ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_POSTREAD); 1757104478Ssam 1758104478Ssam len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8; 1759104478Ssam bcopy(rp->rpr_msgout.dma_vaddr, 1760104478Ssam krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len); 1761104478Ssam 1762104478Ssam crypto_kdone(krp); 1763104478Ssam 1764104478Ssam bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 1765104478Ssam bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 1766104478Ssam bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size); 1767104478Ssam 1768104478Ssam /* Can't free here, so put us on the free list. */ 1769104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next); 1770104478Ssam break; 1771104478Ssam } 1772104478Ssam default: 1773104478Ssam device_printf(sc->sc_dev, "unknown ctx op: %x\n", 1774104478Ssam letoh16(ctx->ctx_op)); 1775104478Ssam break; 1776104478Ssam } 1777104478Ssam} 1778104478Ssam 1779104478Ssam#ifndef UBSEC_NO_RNG 1780104478Ssamstatic void 1781104478Ssamubsec_rng(void *vsc) 1782104478Ssam{ 1783104478Ssam struct ubsec_softc *sc = vsc; 1784104478Ssam struct ubsec_q2_rng *rng = &sc->sc_rng; 1785104478Ssam struct ubsec_mcr *mcr; 1786104478Ssam struct ubsec_ctx_rngbypass *ctx; 1787104478Ssam 1788115747Ssam mtx_lock(&sc->sc_mcr2lock); 1789104478Ssam if (rng->rng_used) { 1790115747Ssam mtx_unlock(&sc->sc_mcr2lock); 1791104478Ssam return; 1792104478Ssam } 1793104478Ssam sc->sc_nqueue2++; 1794104478Ssam if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE) 1795104478Ssam goto out; 1796104478Ssam 1797104478Ssam mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr; 1798104478Ssam ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr; 1799104478Ssam 1800104478Ssam mcr->mcr_pkts = htole16(1); 1801104478Ssam mcr->mcr_flags = 0; 1802104478Ssam mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr); 1803104478Ssam mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0; 1804104478Ssam mcr->mcr_ipktbuf.pb_len = 0; 1805104478Ssam mcr->mcr_reserved = mcr->mcr_pktlen = 0; 1806104478Ssam mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr); 1807104478Ssam mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) & 1808104478Ssam UBS_PKTBUF_LEN); 1809104478Ssam mcr->mcr_opktbuf.pb_next = 0; 1810104478Ssam 1811104478Ssam ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass)); 1812104478Ssam ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS); 1813104478Ssam rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS; 1814104478Ssam 1815108823Ssam ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_PREREAD); 1816104478Ssam 1817104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next); 1818104478Ssam rng->rng_used = 1; 1819104478Ssam ubsec_feed2(sc); 1820104478Ssam ubsecstats.hst_rng++; 1821115747Ssam mtx_unlock(&sc->sc_mcr2lock); 1822104478Ssam 1823104478Ssam return; 1824104478Ssam 1825104478Ssamout: 1826104478Ssam /* 1827104478Ssam * Something weird happened, generate our own call back. 1828104478Ssam */ 1829104478Ssam sc->sc_nqueue2--; 1830115747Ssam mtx_unlock(&sc->sc_mcr2lock); 1831104478Ssam callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1832104478Ssam} 1833104478Ssam#endif /* UBSEC_NO_RNG */ 1834104478Ssam 1835104478Ssamstatic void 1836104478Ssamubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 1837104478Ssam{ 1838104478Ssam bus_addr_t *paddr = (bus_addr_t*) arg; 1839104478Ssam *paddr = segs->ds_addr; 1840104478Ssam} 1841104478Ssam 1842104478Ssamstatic int 1843104478Ssamubsec_dma_malloc( 1844104478Ssam struct ubsec_softc *sc, 1845104478Ssam bus_size_t size, 1846104478Ssam struct ubsec_dma_alloc *dma, 1847104478Ssam int mapflags 1848104478Ssam) 1849104478Ssam{ 1850104478Ssam int r; 1851104478Ssam 1852108823Ssam /* XXX could specify sc_dmat as parent but that just adds overhead */ 1853108823Ssam r = bus_dma_tag_create(NULL, /* parent */ 1854108823Ssam 1, 0, /* alignment, bounds */ 1855108823Ssam BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 1856108823Ssam BUS_SPACE_MAXADDR, /* highaddr */ 1857108823Ssam NULL, NULL, /* filter, filterarg */ 1858108823Ssam size, /* maxsize */ 1859108823Ssam 1, /* nsegments */ 1860108823Ssam size, /* maxsegsize */ 1861108823Ssam BUS_DMA_ALLOCNOW, /* flags */ 1862117126Sscottl NULL, NULL, /* lockfunc, lockarg */ 1863108823Ssam &dma->dma_tag); 1864108823Ssam if (r != 0) { 1865108823Ssam device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1866108823Ssam "bus_dma_tag_create failed; error %u\n", r); 1867104478Ssam goto fail_0; 1868108823Ssam } 1869104478Ssam 1870108823Ssam r = bus_dmamap_create(dma->dma_tag, BUS_DMA_NOWAIT, &dma->dma_map); 1871108823Ssam if (r != 0) { 1872108823Ssam device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1873108823Ssam "bus_dmamap_create failed; error %u\n", r); 1874104478Ssam goto fail_1; 1875108823Ssam } 1876104478Ssam 1877108823Ssam r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 1878108823Ssam BUS_DMA_NOWAIT, &dma->dma_map); 1879108823Ssam if (r != 0) { 1880108823Ssam device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1881109094Ssam "bus_dmammem_alloc failed; size %zu, error %u\n", 1882108823Ssam size, r); 1883108823Ssam goto fail_2; 1884108823Ssam } 1885108823Ssam 1886108823Ssam r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 1887104478Ssam size, 1888104478Ssam ubsec_dmamap_cb, 1889104478Ssam &dma->dma_paddr, 1890104478Ssam mapflags | BUS_DMA_NOWAIT); 1891108823Ssam if (r != 0) { 1892108823Ssam device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1893108823Ssam "bus_dmamap_load failed; error %u\n", r); 1894108823Ssam goto fail_3; 1895108823Ssam } 1896104478Ssam 1897104478Ssam dma->dma_size = size; 1898104478Ssam return (0); 1899104478Ssam 1900108823Ssamfail_3: 1901108823Ssam bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1902104478Ssamfail_2: 1903108823Ssam bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1904104478Ssamfail_1: 1905108823Ssam bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1906108823Ssam bus_dma_tag_destroy(dma->dma_tag); 1907104478Ssamfail_0: 1908104478Ssam dma->dma_map = NULL; 1909108823Ssam dma->dma_tag = NULL; 1910104478Ssam return (r); 1911104478Ssam} 1912104478Ssam 1913104478Ssamstatic void 1914104478Ssamubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma) 1915104478Ssam{ 1916108823Ssam bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1917108823Ssam bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1918108823Ssam bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1919108823Ssam bus_dma_tag_destroy(dma->dma_tag); 1920104478Ssam} 1921104478Ssam 1922104478Ssam/* 1923104478Ssam * Resets the board. Values in the regesters are left as is 1924104478Ssam * from the reset (i.e. initial values are assigned elsewhere). 1925104478Ssam */ 1926104478Ssamstatic void 1927104478Ssamubsec_reset_board(struct ubsec_softc *sc) 1928104478Ssam{ 1929104478Ssam volatile u_int32_t ctrl; 1930104478Ssam 1931104478Ssam ctrl = READ_REG(sc, BS_CTRL); 1932104478Ssam ctrl |= BS_CTRL_RESET; 1933104478Ssam WRITE_REG(sc, BS_CTRL, ctrl); 1934104478Ssam 1935104478Ssam /* 1936104478Ssam * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us 1937104478Ssam */ 1938104478Ssam DELAY(10); 1939104478Ssam} 1940104478Ssam 1941104478Ssam/* 1942104478Ssam * Init Broadcom registers 1943104478Ssam */ 1944104478Ssamstatic void 1945104478Ssamubsec_init_board(struct ubsec_softc *sc) 1946104478Ssam{ 1947104630Ssam u_int32_t ctrl; 1948104630Ssam 1949104630Ssam ctrl = READ_REG(sc, BS_CTRL); 1950104630Ssam ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64); 1951104630Ssam ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT; 1952104630Ssam 1953104630Ssam if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) 1954104630Ssam ctrl |= BS_CTRL_MCR2INT; 1955104630Ssam else 1956104630Ssam ctrl &= ~BS_CTRL_MCR2INT; 1957104630Ssam 1958104630Ssam if (sc->sc_flags & UBS_FLAGS_HWNORM) 1959104630Ssam ctrl &= ~BS_CTRL_SWNORM; 1960104630Ssam 1961104630Ssam WRITE_REG(sc, BS_CTRL, ctrl); 1962104478Ssam} 1963104478Ssam 1964104478Ssam/* 1965104478Ssam * Init Broadcom PCI registers 1966104478Ssam */ 1967104478Ssamstatic void 1968104478Ssamubsec_init_pciregs(device_t dev) 1969104478Ssam{ 1970104478Ssam#if 0 1971104478Ssam u_int32_t misc; 1972104478Ssam 1973104478Ssam misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT); 1974104478Ssam misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT)) 1975104478Ssam | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT); 1976104478Ssam misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT)) 1977104478Ssam | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT); 1978104478Ssam pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc); 1979104478Ssam#endif 1980104478Ssam 1981104478Ssam /* 1982104478Ssam * This will set the cache line size to 1, this will 1983104478Ssam * force the BCM58xx chip just to do burst read/writes. 1984104478Ssam * Cache line read/writes are to slow 1985104478Ssam */ 1986104478Ssam pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1); 1987104478Ssam} 1988104478Ssam 1989104478Ssam/* 1990104478Ssam * Clean up after a chip crash. 1991104478Ssam * It is assumed that the caller in splimp() 1992104478Ssam */ 1993104478Ssamstatic void 1994104478Ssamubsec_cleanchip(struct ubsec_softc *sc) 1995104478Ssam{ 1996104478Ssam struct ubsec_q *q; 1997104478Ssam 1998104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 1999104478Ssam q = SIMPLEQ_FIRST(&sc->sc_qchip); 2000104478Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next); 2001104478Ssam ubsec_free_q(sc, q); 2002104478Ssam } 2003108471Ssam sc->sc_nqchip = 0; 2004104478Ssam} 2005104478Ssam 2006104478Ssam/* 2007108823Ssam * free a ubsec_q 2008108823Ssam * It is assumed that the caller is within splimp(). 2009104478Ssam */ 2010104478Ssamstatic int 2011104478Ssamubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q) 2012104478Ssam{ 2013104478Ssam struct ubsec_q *q2; 2014104478Ssam struct cryptop *crp; 2015104478Ssam int npkts; 2016104478Ssam int i; 2017104478Ssam 2018104478Ssam npkts = q->q_nstacked_mcrs; 2019104478Ssam 2020104478Ssam for (i = 0; i < npkts; i++) { 2021104478Ssam if(q->q_stacked_mcr[i]) { 2022104478Ssam q2 = q->q_stacked_mcr[i]; 2023104478Ssam 2024104478Ssam if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m)) 2025104478Ssam m_freem(q2->q_dst_m); 2026104478Ssam 2027104478Ssam crp = (struct cryptop *)q2->q_crp; 2028104478Ssam 2029104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next); 2030104478Ssam 2031104478Ssam crp->crp_etype = EFAULT; 2032104478Ssam crypto_done(crp); 2033104478Ssam } else { 2034104478Ssam break; 2035104478Ssam } 2036104478Ssam } 2037104478Ssam 2038104478Ssam /* 2039104478Ssam * Free header MCR 2040104478Ssam */ 2041104478Ssam if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 2042104478Ssam m_freem(q->q_dst_m); 2043104478Ssam 2044104478Ssam crp = (struct cryptop *)q->q_crp; 2045104478Ssam 2046104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 2047104478Ssam 2048104478Ssam crp->crp_etype = EFAULT; 2049104478Ssam crypto_done(crp); 2050104478Ssam return(0); 2051104478Ssam} 2052104478Ssam 2053104478Ssam/* 2054104478Ssam * Routine to reset the chip and clean up. 2055104478Ssam * It is assumed that the caller is in splimp() 2056104478Ssam */ 2057104478Ssamstatic void 2058104478Ssamubsec_totalreset(struct ubsec_softc *sc) 2059104478Ssam{ 2060104478Ssam ubsec_reset_board(sc); 2061104478Ssam ubsec_init_board(sc); 2062104478Ssam ubsec_cleanchip(sc); 2063104478Ssam} 2064104478Ssam 2065104478Ssamstatic int 2066104478Ssamubsec_dmamap_aligned(struct ubsec_operand *op) 2067104478Ssam{ 2068104478Ssam int i; 2069104478Ssam 2070104478Ssam for (i = 0; i < op->nsegs; i++) { 2071104478Ssam if (op->segs[i].ds_addr & 3) 2072104478Ssam return (0); 2073104478Ssam if ((i != (op->nsegs - 1)) && 2074104478Ssam (op->segs[i].ds_len & 3)) 2075104478Ssam return (0); 2076104478Ssam } 2077104478Ssam return (1); 2078104478Ssam} 2079104478Ssam 2080104478Ssamstatic void 2081104478Ssamubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q) 2082104478Ssam{ 2083104478Ssam switch (q->q_type) { 2084104478Ssam case UBS_CTXOP_MODEXP: { 2085104478Ssam struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 2086104478Ssam 2087104478Ssam ubsec_dma_free(sc, &me->me_q.q_mcr); 2088104478Ssam ubsec_dma_free(sc, &me->me_q.q_ctx); 2089104478Ssam ubsec_dma_free(sc, &me->me_M); 2090104478Ssam ubsec_dma_free(sc, &me->me_E); 2091104478Ssam ubsec_dma_free(sc, &me->me_C); 2092104478Ssam ubsec_dma_free(sc, &me->me_epb); 2093104478Ssam free(me, M_DEVBUF); 2094104478Ssam break; 2095104478Ssam } 2096104478Ssam case UBS_CTXOP_RSAPRIV: { 2097104478Ssam struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 2098104478Ssam 2099104478Ssam ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2100104478Ssam ubsec_dma_free(sc, &rp->rpr_q.q_ctx); 2101104478Ssam ubsec_dma_free(sc, &rp->rpr_msgin); 2102104478Ssam ubsec_dma_free(sc, &rp->rpr_msgout); 2103104478Ssam free(rp, M_DEVBUF); 2104104478Ssam break; 2105104478Ssam } 2106104478Ssam default: 2107104478Ssam device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type); 2108104478Ssam break; 2109104478Ssam } 2110104478Ssam} 2111104478Ssam 2112104478Ssamstatic int 2113104478Ssamubsec_kprocess(void *arg, struct cryptkop *krp, int hint) 2114104478Ssam{ 2115104478Ssam struct ubsec_softc *sc = arg; 2116104630Ssam int r; 2117104478Ssam 2118104478Ssam if (krp == NULL || krp->krp_callback == NULL) 2119104478Ssam return (EINVAL); 2120104478Ssam 2121104478Ssam while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) { 2122104478Ssam struct ubsec_q2 *q; 2123104478Ssam 2124104478Ssam q = SIMPLEQ_FIRST(&sc->sc_q2free); 2125104478Ssam SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q, q_next); 2126104478Ssam ubsec_kfree(sc, q); 2127104478Ssam } 2128104478Ssam 2129104478Ssam switch (krp->krp_op) { 2130104478Ssam case CRK_MOD_EXP: 2131104630Ssam if (sc->sc_flags & UBS_FLAGS_HWNORM) 2132104630Ssam r = ubsec_kprocess_modexp_hw(sc, krp, hint); 2133104630Ssam else 2134104630Ssam r = ubsec_kprocess_modexp_sw(sc, krp, hint); 2135104630Ssam break; 2136104478Ssam case CRK_MOD_EXP_CRT: 2137104478Ssam return (ubsec_kprocess_rsapriv(sc, krp, hint)); 2138104478Ssam default: 2139104478Ssam device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n", 2140104478Ssam krp->krp_op); 2141104478Ssam krp->krp_status = EOPNOTSUPP; 2142104478Ssam crypto_kdone(krp); 2143104478Ssam return (0); 2144104478Ssam } 2145104630Ssam return (0); /* silence compiler */ 2146104478Ssam} 2147104478Ssam 2148104478Ssam/* 2149104630Ssam * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization) 2150104478Ssam */ 2151104478Ssamstatic int 2152104630Ssamubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2153104478Ssam{ 2154104478Ssam struct ubsec_q2_modexp *me; 2155104478Ssam struct ubsec_mcr *mcr; 2156104478Ssam struct ubsec_ctx_modexp *ctx; 2157104478Ssam struct ubsec_pktbuf *epb; 2158104478Ssam int err = 0; 2159104478Ssam u_int nbits, normbits, mbits, shiftbits, ebits; 2160104478Ssam 2161104478Ssam me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2162104478Ssam if (me == NULL) { 2163104478Ssam err = ENOMEM; 2164104478Ssam goto errout; 2165104478Ssam } 2166104478Ssam bzero(me, sizeof *me); 2167104478Ssam me->me_krp = krp; 2168104478Ssam me->me_q.q_type = UBS_CTXOP_MODEXP; 2169104478Ssam 2170104478Ssam nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2171104478Ssam if (nbits <= 512) 2172104478Ssam normbits = 512; 2173104478Ssam else if (nbits <= 768) 2174104478Ssam normbits = 768; 2175104478Ssam else if (nbits <= 1024) 2176104478Ssam normbits = 1024; 2177104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2178104478Ssam normbits = 1536; 2179104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2180104478Ssam normbits = 2048; 2181104478Ssam else { 2182104478Ssam err = E2BIG; 2183104478Ssam goto errout; 2184104478Ssam } 2185104478Ssam 2186104630Ssam shiftbits = normbits - nbits; 2187104478Ssam 2188104630Ssam me->me_modbits = nbits; 2189104478Ssam me->me_shiftbits = shiftbits; 2190104630Ssam me->me_normbits = normbits; 2191104478Ssam 2192104478Ssam /* Sanity check: result bits must be >= true modulus bits. */ 2193104630Ssam if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2194104478Ssam err = ERANGE; 2195104478Ssam goto errout; 2196104478Ssam } 2197104478Ssam 2198104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2199104478Ssam &me->me_q.q_mcr, 0)) { 2200104478Ssam err = ENOMEM; 2201104478Ssam goto errout; 2202104478Ssam } 2203104478Ssam mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2204104478Ssam 2205104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2206104478Ssam &me->me_q.q_ctx, 0)) { 2207104478Ssam err = ENOMEM; 2208104478Ssam goto errout; 2209104478Ssam } 2210104478Ssam 2211104478Ssam mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2212104478Ssam if (mbits > nbits) { 2213104478Ssam err = E2BIG; 2214104478Ssam goto errout; 2215104478Ssam } 2216104478Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2217104478Ssam err = ENOMEM; 2218104478Ssam goto errout; 2219104478Ssam } 2220104478Ssam ubsec_kshift_r(shiftbits, 2221104478Ssam krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits, 2222104478Ssam me->me_M.dma_vaddr, normbits); 2223104478Ssam 2224104478Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2225104478Ssam err = ENOMEM; 2226104478Ssam goto errout; 2227104478Ssam } 2228104478Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2229104478Ssam 2230104478Ssam ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2231104478Ssam if (ebits > nbits) { 2232104478Ssam err = E2BIG; 2233104478Ssam goto errout; 2234104478Ssam } 2235104478Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2236104478Ssam err = ENOMEM; 2237104478Ssam goto errout; 2238104478Ssam } 2239104478Ssam ubsec_kshift_r(shiftbits, 2240104478Ssam krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits, 2241104478Ssam me->me_E.dma_vaddr, normbits); 2242104478Ssam 2243104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2244104478Ssam &me->me_epb, 0)) { 2245104478Ssam err = ENOMEM; 2246104478Ssam goto errout; 2247104478Ssam } 2248104478Ssam epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2249104478Ssam epb->pb_addr = htole32(me->me_E.dma_paddr); 2250104478Ssam epb->pb_next = 0; 2251104478Ssam epb->pb_len = htole32(normbits / 8); 2252104478Ssam 2253104478Ssam#ifdef UBSEC_DEBUG 2254104478Ssam if (ubsec_debug) { 2255104478Ssam printf("Epb "); 2256104478Ssam ubsec_dump_pb(epb); 2257104478Ssam } 2258104478Ssam#endif 2259104478Ssam 2260104478Ssam mcr->mcr_pkts = htole16(1); 2261104478Ssam mcr->mcr_flags = 0; 2262104478Ssam mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2263104478Ssam mcr->mcr_reserved = 0; 2264104478Ssam mcr->mcr_pktlen = 0; 2265104478Ssam 2266104478Ssam mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2267104478Ssam mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2268104478Ssam mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2269104478Ssam 2270104478Ssam mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2271104478Ssam mcr->mcr_opktbuf.pb_next = 0; 2272104478Ssam mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2273104478Ssam 2274104478Ssam#ifdef DIAGNOSTIC 2275104478Ssam /* Misaligned output buffer will hang the chip. */ 2276104478Ssam if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2277104478Ssam panic("%s: modexp invalid addr 0x%x\n", 2278104478Ssam device_get_nameunit(sc->sc_dev), 2279104478Ssam letoh32(mcr->mcr_opktbuf.pb_addr)); 2280104478Ssam if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2281104478Ssam panic("%s: modexp invalid len 0x%x\n", 2282104478Ssam device_get_nameunit(sc->sc_dev), 2283104478Ssam letoh32(mcr->mcr_opktbuf.pb_len)); 2284104478Ssam#endif 2285104478Ssam 2286104478Ssam ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2287104478Ssam bzero(ctx, sizeof(*ctx)); 2288104478Ssam ubsec_kshift_r(shiftbits, 2289104478Ssam krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits, 2290104478Ssam ctx->me_N, normbits); 2291104478Ssam ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2292104478Ssam ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2293104630Ssam ctx->me_E_len = htole16(nbits); 2294104630Ssam ctx->me_N_len = htole16(nbits); 2295104478Ssam 2296104478Ssam#ifdef UBSEC_DEBUG 2297104478Ssam if (ubsec_debug) { 2298104478Ssam ubsec_dump_mcr(mcr); 2299104478Ssam ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2300104478Ssam } 2301104478Ssam#endif 2302104478Ssam 2303104478Ssam /* 2304104478Ssam * ubsec_feed2 will sync mcr and ctx, we just need to sync 2305104478Ssam * everything else. 2306104478Ssam */ 2307108823Ssam ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE); 2308108823Ssam ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE); 2309108823Ssam ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD); 2310108823Ssam ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE); 2311104478Ssam 2312104478Ssam /* Enqueue and we're done... */ 2313115747Ssam mtx_lock(&sc->sc_mcr2lock); 2314104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2315104478Ssam ubsec_feed2(sc); 2316104478Ssam ubsecstats.hst_modexp++; 2317115747Ssam mtx_unlock(&sc->sc_mcr2lock); 2318104478Ssam 2319104478Ssam return (0); 2320104478Ssam 2321104478Ssamerrout: 2322104478Ssam if (me != NULL) { 2323104478Ssam if (me->me_q.q_mcr.dma_map != NULL) 2324104478Ssam ubsec_dma_free(sc, &me->me_q.q_mcr); 2325104478Ssam if (me->me_q.q_ctx.dma_map != NULL) { 2326104478Ssam bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2327104478Ssam ubsec_dma_free(sc, &me->me_q.q_ctx); 2328104478Ssam } 2329104478Ssam if (me->me_M.dma_map != NULL) { 2330104478Ssam bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2331104478Ssam ubsec_dma_free(sc, &me->me_M); 2332104478Ssam } 2333104478Ssam if (me->me_E.dma_map != NULL) { 2334104478Ssam bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2335104478Ssam ubsec_dma_free(sc, &me->me_E); 2336104478Ssam } 2337104478Ssam if (me->me_C.dma_map != NULL) { 2338104478Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2339104478Ssam ubsec_dma_free(sc, &me->me_C); 2340104478Ssam } 2341104478Ssam if (me->me_epb.dma_map != NULL) 2342104478Ssam ubsec_dma_free(sc, &me->me_epb); 2343104478Ssam free(me, M_DEVBUF); 2344104478Ssam } 2345104478Ssam krp->krp_status = err; 2346104478Ssam crypto_kdone(krp); 2347104478Ssam return (0); 2348104478Ssam} 2349104478Ssam 2350104630Ssam/* 2351104630Ssam * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization) 2352104630Ssam */ 2353105215Sphkstatic int 2354104630Ssamubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2355104630Ssam{ 2356104630Ssam struct ubsec_q2_modexp *me; 2357104630Ssam struct ubsec_mcr *mcr; 2358104630Ssam struct ubsec_ctx_modexp *ctx; 2359104630Ssam struct ubsec_pktbuf *epb; 2360104630Ssam int err = 0; 2361104630Ssam u_int nbits, normbits, mbits, shiftbits, ebits; 2362104630Ssam 2363104630Ssam me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2364104630Ssam if (me == NULL) { 2365104630Ssam err = ENOMEM; 2366104630Ssam goto errout; 2367104630Ssam } 2368104630Ssam bzero(me, sizeof *me); 2369104630Ssam me->me_krp = krp; 2370104630Ssam me->me_q.q_type = UBS_CTXOP_MODEXP; 2371104630Ssam 2372104630Ssam nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2373104630Ssam if (nbits <= 512) 2374104630Ssam normbits = 512; 2375104630Ssam else if (nbits <= 768) 2376104630Ssam normbits = 768; 2377104630Ssam else if (nbits <= 1024) 2378104630Ssam normbits = 1024; 2379104630Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2380104630Ssam normbits = 1536; 2381104630Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2382104630Ssam normbits = 2048; 2383104630Ssam else { 2384104630Ssam err = E2BIG; 2385104630Ssam goto errout; 2386104630Ssam } 2387104630Ssam 2388104630Ssam shiftbits = normbits - nbits; 2389104630Ssam 2390104630Ssam /* XXX ??? */ 2391104630Ssam me->me_modbits = nbits; 2392104630Ssam me->me_shiftbits = shiftbits; 2393104630Ssam me->me_normbits = normbits; 2394104630Ssam 2395104630Ssam /* Sanity check: result bits must be >= true modulus bits. */ 2396104630Ssam if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2397104630Ssam err = ERANGE; 2398104630Ssam goto errout; 2399104630Ssam } 2400104630Ssam 2401104630Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2402104630Ssam &me->me_q.q_mcr, 0)) { 2403104630Ssam err = ENOMEM; 2404104630Ssam goto errout; 2405104630Ssam } 2406104630Ssam mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2407104630Ssam 2408104630Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2409104630Ssam &me->me_q.q_ctx, 0)) { 2410104630Ssam err = ENOMEM; 2411104630Ssam goto errout; 2412104630Ssam } 2413104630Ssam 2414104630Ssam mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2415104630Ssam if (mbits > nbits) { 2416104630Ssam err = E2BIG; 2417104630Ssam goto errout; 2418104630Ssam } 2419104630Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2420104630Ssam err = ENOMEM; 2421104630Ssam goto errout; 2422104630Ssam } 2423104630Ssam bzero(me->me_M.dma_vaddr, normbits / 8); 2424104630Ssam bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p, 2425104630Ssam me->me_M.dma_vaddr, (mbits + 7) / 8); 2426104630Ssam 2427104630Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2428104630Ssam err = ENOMEM; 2429104630Ssam goto errout; 2430104630Ssam } 2431104630Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2432104630Ssam 2433104630Ssam ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2434104630Ssam if (ebits > nbits) { 2435104630Ssam err = E2BIG; 2436104630Ssam goto errout; 2437104630Ssam } 2438104630Ssam if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2439104630Ssam err = ENOMEM; 2440104630Ssam goto errout; 2441104630Ssam } 2442104630Ssam bzero(me->me_E.dma_vaddr, normbits / 8); 2443104630Ssam bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p, 2444104630Ssam me->me_E.dma_vaddr, (ebits + 7) / 8); 2445104630Ssam 2446104630Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2447104630Ssam &me->me_epb, 0)) { 2448104630Ssam err = ENOMEM; 2449104630Ssam goto errout; 2450104630Ssam } 2451104630Ssam epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2452104630Ssam epb->pb_addr = htole32(me->me_E.dma_paddr); 2453104630Ssam epb->pb_next = 0; 2454104630Ssam epb->pb_len = htole32((ebits + 7) / 8); 2455104630Ssam 2456104630Ssam#ifdef UBSEC_DEBUG 2457108823Ssam if (ubsec_debug) { 2458108823Ssam printf("Epb "); 2459108823Ssam ubsec_dump_pb(epb); 2460108823Ssam } 2461104630Ssam#endif 2462104630Ssam 2463104630Ssam mcr->mcr_pkts = htole16(1); 2464104630Ssam mcr->mcr_flags = 0; 2465104630Ssam mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2466104630Ssam mcr->mcr_reserved = 0; 2467104630Ssam mcr->mcr_pktlen = 0; 2468104630Ssam 2469104630Ssam mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2470104630Ssam mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2471104630Ssam mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2472104630Ssam 2473104630Ssam mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2474104630Ssam mcr->mcr_opktbuf.pb_next = 0; 2475104630Ssam mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2476104630Ssam 2477104630Ssam#ifdef DIAGNOSTIC 2478104630Ssam /* Misaligned output buffer will hang the chip. */ 2479104630Ssam if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2480104630Ssam panic("%s: modexp invalid addr 0x%x\n", 2481104630Ssam device_get_nameunit(sc->sc_dev), 2482104630Ssam letoh32(mcr->mcr_opktbuf.pb_addr)); 2483104630Ssam if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2484104630Ssam panic("%s: modexp invalid len 0x%x\n", 2485104630Ssam device_get_nameunit(sc->sc_dev), 2486104630Ssam letoh32(mcr->mcr_opktbuf.pb_len)); 2487104630Ssam#endif 2488104630Ssam 2489104630Ssam ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2490104630Ssam bzero(ctx, sizeof(*ctx)); 2491104630Ssam bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N, 2492104630Ssam (nbits + 7) / 8); 2493104630Ssam ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2494104630Ssam ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2495104630Ssam ctx->me_E_len = htole16(ebits); 2496104630Ssam ctx->me_N_len = htole16(nbits); 2497104630Ssam 2498104630Ssam#ifdef UBSEC_DEBUG 2499104630Ssam if (ubsec_debug) { 2500104630Ssam ubsec_dump_mcr(mcr); 2501104630Ssam ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2502104630Ssam } 2503104630Ssam#endif 2504104630Ssam 2505104630Ssam /* 2506104630Ssam * ubsec_feed2 will sync mcr and ctx, we just need to sync 2507104630Ssam * everything else. 2508104630Ssam */ 2509108823Ssam ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE); 2510108823Ssam ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE); 2511108823Ssam ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD); 2512108823Ssam ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE); 2513104630Ssam 2514104630Ssam /* Enqueue and we're done... */ 2515115747Ssam mtx_lock(&sc->sc_mcr2lock); 2516104630Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2517104630Ssam ubsec_feed2(sc); 2518115747Ssam mtx_unlock(&sc->sc_mcr2lock); 2519104630Ssam 2520104630Ssam return (0); 2521104630Ssam 2522104630Ssamerrout: 2523104630Ssam if (me != NULL) { 2524104630Ssam if (me->me_q.q_mcr.dma_map != NULL) 2525104630Ssam ubsec_dma_free(sc, &me->me_q.q_mcr); 2526104630Ssam if (me->me_q.q_ctx.dma_map != NULL) { 2527104630Ssam bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2528104630Ssam ubsec_dma_free(sc, &me->me_q.q_ctx); 2529104630Ssam } 2530104630Ssam if (me->me_M.dma_map != NULL) { 2531104630Ssam bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2532104630Ssam ubsec_dma_free(sc, &me->me_M); 2533104630Ssam } 2534104630Ssam if (me->me_E.dma_map != NULL) { 2535104630Ssam bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2536104630Ssam ubsec_dma_free(sc, &me->me_E); 2537104630Ssam } 2538104630Ssam if (me->me_C.dma_map != NULL) { 2539104630Ssam bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2540104630Ssam ubsec_dma_free(sc, &me->me_C); 2541104630Ssam } 2542104630Ssam if (me->me_epb.dma_map != NULL) 2543104630Ssam ubsec_dma_free(sc, &me->me_epb); 2544104630Ssam free(me, M_DEVBUF); 2545104630Ssam } 2546104630Ssam krp->krp_status = err; 2547104630Ssam crypto_kdone(krp); 2548104630Ssam return (0); 2549104630Ssam} 2550104630Ssam 2551104478Ssamstatic int 2552104478Ssamubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2553104478Ssam{ 2554104478Ssam struct ubsec_q2_rsapriv *rp = NULL; 2555104478Ssam struct ubsec_mcr *mcr; 2556104478Ssam struct ubsec_ctx_rsapriv *ctx; 2557104478Ssam int err = 0; 2558104478Ssam u_int padlen, msglen; 2559104478Ssam 2560104478Ssam msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]); 2561104478Ssam padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]); 2562104478Ssam if (msglen > padlen) 2563104478Ssam padlen = msglen; 2564104478Ssam 2565104478Ssam if (padlen <= 256) 2566104478Ssam padlen = 256; 2567104478Ssam else if (padlen <= 384) 2568104478Ssam padlen = 384; 2569104478Ssam else if (padlen <= 512) 2570104478Ssam padlen = 512; 2571104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768) 2572104478Ssam padlen = 768; 2573104478Ssam else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024) 2574104478Ssam padlen = 1024; 2575104478Ssam else { 2576104478Ssam err = E2BIG; 2577104478Ssam goto errout; 2578104478Ssam } 2579104478Ssam 2580104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) { 2581104478Ssam err = E2BIG; 2582104478Ssam goto errout; 2583104478Ssam } 2584104478Ssam 2585104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) { 2586104478Ssam err = E2BIG; 2587104478Ssam goto errout; 2588104478Ssam } 2589104478Ssam 2590104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) { 2591104478Ssam err = E2BIG; 2592104478Ssam goto errout; 2593104478Ssam } 2594104478Ssam 2595104478Ssam rp = (struct ubsec_q2_rsapriv *)malloc(sizeof *rp, M_DEVBUF, M_NOWAIT); 2596104478Ssam if (rp == NULL) 2597104478Ssam return (ENOMEM); 2598104478Ssam bzero(rp, sizeof *rp); 2599104478Ssam rp->rpr_krp = krp; 2600104478Ssam rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV; 2601104478Ssam 2602104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2603104478Ssam &rp->rpr_q.q_mcr, 0)) { 2604104478Ssam err = ENOMEM; 2605104478Ssam goto errout; 2606104478Ssam } 2607104478Ssam mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr; 2608104478Ssam 2609104478Ssam if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv), 2610104478Ssam &rp->rpr_q.q_ctx, 0)) { 2611104478Ssam err = ENOMEM; 2612104478Ssam goto errout; 2613104478Ssam } 2614104478Ssam ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr; 2615104478Ssam bzero(ctx, sizeof *ctx); 2616104478Ssam 2617104478Ssam /* Copy in p */ 2618104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p, 2619104478Ssam &ctx->rpr_buf[0 * (padlen / 8)], 2620104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8); 2621104478Ssam 2622104478Ssam /* Copy in q */ 2623104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p, 2624104478Ssam &ctx->rpr_buf[1 * (padlen / 8)], 2625104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8); 2626104478Ssam 2627104478Ssam /* Copy in dp */ 2628104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p, 2629104478Ssam &ctx->rpr_buf[2 * (padlen / 8)], 2630104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8); 2631104478Ssam 2632104478Ssam /* Copy in dq */ 2633104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p, 2634104478Ssam &ctx->rpr_buf[3 * (padlen / 8)], 2635104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8); 2636104478Ssam 2637104478Ssam /* Copy in pinv */ 2638104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p, 2639104478Ssam &ctx->rpr_buf[4 * (padlen / 8)], 2640104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8); 2641104478Ssam 2642104478Ssam msglen = padlen * 2; 2643104478Ssam 2644104478Ssam /* Copy in input message (aligned buffer/length). */ 2645104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) { 2646104478Ssam /* Is this likely? */ 2647104478Ssam err = E2BIG; 2648104478Ssam goto errout; 2649104478Ssam } 2650104478Ssam if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) { 2651104478Ssam err = ENOMEM; 2652104478Ssam goto errout; 2653104478Ssam } 2654104478Ssam bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8); 2655104478Ssam bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p, 2656104478Ssam rp->rpr_msgin.dma_vaddr, 2657104478Ssam (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8); 2658104478Ssam 2659104478Ssam /* Prepare space for output message (aligned buffer/length). */ 2660104478Ssam if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) { 2661104478Ssam /* Is this likely? */ 2662104478Ssam err = E2BIG; 2663104478Ssam goto errout; 2664104478Ssam } 2665104478Ssam if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) { 2666104478Ssam err = ENOMEM; 2667104478Ssam goto errout; 2668104478Ssam } 2669104478Ssam bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8); 2670104478Ssam 2671104478Ssam mcr->mcr_pkts = htole16(1); 2672104478Ssam mcr->mcr_flags = 0; 2673104478Ssam mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr); 2674104478Ssam mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr); 2675104478Ssam mcr->mcr_ipktbuf.pb_next = 0; 2676104478Ssam mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size); 2677104478Ssam mcr->mcr_reserved = 0; 2678104478Ssam mcr->mcr_pktlen = htole16(msglen); 2679104478Ssam mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr); 2680104478Ssam mcr->mcr_opktbuf.pb_next = 0; 2681104478Ssam mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size); 2682104478Ssam 2683104478Ssam#ifdef DIAGNOSTIC 2684104478Ssam if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) { 2685106579Sjhb panic("%s: rsapriv: invalid msgin %x(0x%jx)", 2686104478Ssam device_get_nameunit(sc->sc_dev), 2687106579Sjhb rp->rpr_msgin.dma_paddr, (uintmax_t)rp->rpr_msgin.dma_size); 2688104478Ssam } 2689104478Ssam if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) { 2690106579Sjhb panic("%s: rsapriv: invalid msgout %x(0x%jx)", 2691104478Ssam device_get_nameunit(sc->sc_dev), 2692106579Sjhb rp->rpr_msgout.dma_paddr, (uintmax_t)rp->rpr_msgout.dma_size); 2693104478Ssam } 2694104478Ssam#endif 2695104478Ssam 2696104478Ssam ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8)); 2697104478Ssam ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV); 2698104478Ssam ctx->rpr_q_len = htole16(padlen); 2699104478Ssam ctx->rpr_p_len = htole16(padlen); 2700104478Ssam 2701104478Ssam /* 2702104478Ssam * ubsec_feed2 will sync mcr and ctx, we just need to sync 2703104478Ssam * everything else. 2704104478Ssam */ 2705108823Ssam ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_PREWRITE); 2706108823Ssam ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_PREREAD); 2707104478Ssam 2708104478Ssam /* Enqueue and we're done... */ 2709115747Ssam mtx_lock(&sc->sc_mcr2lock); 2710104478Ssam SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next); 2711104478Ssam ubsec_feed2(sc); 2712104478Ssam ubsecstats.hst_modexpcrt++; 2713115747Ssam mtx_unlock(&sc->sc_mcr2lock); 2714104478Ssam return (0); 2715104478Ssam 2716104478Ssamerrout: 2717104478Ssam if (rp != NULL) { 2718104478Ssam if (rp->rpr_q.q_mcr.dma_map != NULL) 2719104478Ssam ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2720104478Ssam if (rp->rpr_msgin.dma_map != NULL) { 2721104478Ssam bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 2722104478Ssam ubsec_dma_free(sc, &rp->rpr_msgin); 2723104478Ssam } 2724104478Ssam if (rp->rpr_msgout.dma_map != NULL) { 2725104478Ssam bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 2726104478Ssam ubsec_dma_free(sc, &rp->rpr_msgout); 2727104478Ssam } 2728104478Ssam free(rp, M_DEVBUF); 2729104478Ssam } 2730104478Ssam krp->krp_status = err; 2731104478Ssam crypto_kdone(krp); 2732104478Ssam return (0); 2733104478Ssam} 2734104478Ssam 2735104478Ssam#ifdef UBSEC_DEBUG 2736104478Ssamstatic void 2737104478Ssamubsec_dump_pb(volatile struct ubsec_pktbuf *pb) 2738104478Ssam{ 2739104478Ssam printf("addr 0x%x (0x%x) next 0x%x\n", 2740104478Ssam pb->pb_addr, pb->pb_len, pb->pb_next); 2741104478Ssam} 2742104478Ssam 2743104478Ssamstatic void 2744104478Ssamubsec_dump_ctx2(struct ubsec_ctx_keyop *c) 2745104478Ssam{ 2746104478Ssam printf("CTX (0x%x):\n", c->ctx_len); 2747104478Ssam switch (letoh16(c->ctx_op)) { 2748104478Ssam case UBS_CTXOP_RNGBYPASS: 2749104478Ssam case UBS_CTXOP_RNGSHA1: 2750104478Ssam break; 2751104478Ssam case UBS_CTXOP_MODEXP: 2752104478Ssam { 2753104478Ssam struct ubsec_ctx_modexp *cx = (void *)c; 2754104478Ssam int i, len; 2755104478Ssam 2756104478Ssam printf(" Elen %u, Nlen %u\n", 2757104478Ssam letoh16(cx->me_E_len), letoh16(cx->me_N_len)); 2758104478Ssam len = (cx->me_N_len + 7)/8; 2759104478Ssam for (i = 0; i < len; i++) 2760104478Ssam printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]); 2761104478Ssam printf("\n"); 2762104478Ssam break; 2763104478Ssam } 2764104478Ssam default: 2765104478Ssam printf("unknown context: %x\n", c->ctx_op); 2766104478Ssam } 2767104478Ssam printf("END CTX\n"); 2768104478Ssam} 2769104478Ssam 2770104478Ssamstatic void 2771104478Ssamubsec_dump_mcr(struct ubsec_mcr *mcr) 2772104478Ssam{ 2773104478Ssam volatile struct ubsec_mcr_add *ma; 2774104478Ssam int i; 2775104478Ssam 2776104478Ssam printf("MCR:\n"); 2777104478Ssam printf(" pkts: %u, flags 0x%x\n", 2778104478Ssam letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags)); 2779104478Ssam ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp; 2780104478Ssam for (i = 0; i < letoh16(mcr->mcr_pkts); i++) { 2781104478Ssam printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i, 2782104478Ssam letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen), 2783104478Ssam letoh16(ma->mcr_reserved)); 2784104478Ssam printf(" %d: ipkt ", i); 2785104478Ssam ubsec_dump_pb(&ma->mcr_ipktbuf); 2786104478Ssam printf(" %d: opkt ", i); 2787104478Ssam ubsec_dump_pb(&ma->mcr_opktbuf); 2788104478Ssam ma++; 2789104478Ssam } 2790104478Ssam printf("END MCR\n"); 2791104478Ssam} 2792104478Ssam#endif /* UBSEC_DEBUG */ 2793104478Ssam 2794104478Ssam/* 2795104478Ssam * Return the number of significant bits of a big number. 2796104478Ssam */ 2797104478Ssamstatic int 2798104478Ssamubsec_ksigbits(struct crparam *cr) 2799104478Ssam{ 2800104478Ssam u_int plen = (cr->crp_nbits + 7) / 8; 2801104478Ssam int i, sig = plen * 8; 2802104478Ssam u_int8_t c, *p = cr->crp_p; 2803104478Ssam 2804104478Ssam for (i = plen - 1; i >= 0; i--) { 2805104478Ssam c = p[i]; 2806104478Ssam if (c != 0) { 2807104478Ssam while ((c & 0x80) == 0) { 2808104478Ssam sig--; 2809104478Ssam c <<= 1; 2810104478Ssam } 2811104478Ssam break; 2812104478Ssam } 2813104478Ssam sig -= 8; 2814104478Ssam } 2815104478Ssam return (sig); 2816104478Ssam} 2817104478Ssam 2818104478Ssamstatic void 2819104478Ssamubsec_kshift_r( 2820104478Ssam u_int shiftbits, 2821104478Ssam u_int8_t *src, u_int srcbits, 2822104478Ssam u_int8_t *dst, u_int dstbits) 2823104478Ssam{ 2824104478Ssam u_int slen, dlen; 2825104478Ssam int i, si, di, n; 2826104478Ssam 2827104478Ssam slen = (srcbits + 7) / 8; 2828104478Ssam dlen = (dstbits + 7) / 8; 2829104478Ssam 2830104478Ssam for (i = 0; i < slen; i++) 2831104478Ssam dst[i] = src[i]; 2832104478Ssam for (i = 0; i < dlen - slen; i++) 2833104478Ssam dst[slen + i] = 0; 2834104478Ssam 2835104478Ssam n = shiftbits / 8; 2836104478Ssam if (n != 0) { 2837104478Ssam si = dlen - n - 1; 2838104478Ssam di = dlen - 1; 2839104478Ssam while (si >= 0) 2840104478Ssam dst[di--] = dst[si--]; 2841104478Ssam while (di >= 0) 2842104478Ssam dst[di--] = 0; 2843104478Ssam } 2844104478Ssam 2845104478Ssam n = shiftbits % 8; 2846104478Ssam if (n != 0) { 2847104478Ssam for (i = dlen - 1; i > 0; i--) 2848104478Ssam dst[i] = (dst[i] << n) | 2849104478Ssam (dst[i - 1] >> (8 - n)); 2850104478Ssam dst[0] = dst[0] << n; 2851104478Ssam } 2852104478Ssam} 2853104478Ssam 2854104478Ssamstatic void 2855104478Ssamubsec_kshift_l( 2856104478Ssam u_int shiftbits, 2857104478Ssam u_int8_t *src, u_int srcbits, 2858104478Ssam u_int8_t *dst, u_int dstbits) 2859104478Ssam{ 2860104478Ssam int slen, dlen, i, n; 2861104478Ssam 2862104478Ssam slen = (srcbits + 7) / 8; 2863104478Ssam dlen = (dstbits + 7) / 8; 2864104478Ssam 2865104478Ssam n = shiftbits / 8; 2866104478Ssam for (i = 0; i < slen; i++) 2867104478Ssam dst[i] = src[i + n]; 2868104478Ssam for (i = 0; i < dlen - slen; i++) 2869104478Ssam dst[slen + i] = 0; 2870104478Ssam 2871104478Ssam n = shiftbits % 8; 2872104478Ssam if (n != 0) { 2873104478Ssam for (i = 0; i < (dlen - 1); i++) 2874104478Ssam dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n)); 2875104478Ssam dst[dlen - 1] = dst[dlen - 1] >> n; 2876104478Ssam } 2877104478Ssam} 2878