hifn7751.c revision 115748
1104477Ssam/* $FreeBSD: head/sys/dev/hifn/hifn7751.c 115748 2003-06-02 23:34:59Z sam $ */ 2104477Ssam/* $OpenBSD: hifn7751.c,v 1.120 2002/05/17 00:33:34 deraadt Exp $ */ 3104477Ssam 4104477Ssam/* 5104477Ssam * Invertex AEON / Hifn 7751 driver 6104477Ssam * Copyright (c) 1999 Invertex Inc. All rights reserved. 7104477Ssam * Copyright (c) 1999 Theo de Raadt 8104477Ssam * Copyright (c) 2000-2001 Network Security Technologies, Inc. 9104477Ssam * http://www.netsec.net 10104477Ssam * 11104477Ssam * This driver is based on a previous driver by Invertex, for which they 12104477Ssam * requested: Please send any comments, feedback, bug-fixes, or feature 13104477Ssam * requests to software@invertex.com. 14104477Ssam * 15104477Ssam * Redistribution and use in source and binary forms, with or without 16104477Ssam * modification, are permitted provided that the following conditions 17104477Ssam * are met: 18104477Ssam * 19104477Ssam * 1. Redistributions of source code must retain the above copyright 20104477Ssam * notice, this list of conditions and the following disclaimer. 21104477Ssam * 2. Redistributions in binary form must reproduce the above copyright 22104477Ssam * notice, this list of conditions and the following disclaimer in the 23104477Ssam * documentation and/or other materials provided with the distribution. 24104477Ssam * 3. The name of the author may not be used to endorse or promote products 25104477Ssam * derived from this software without specific prior written permission. 26104477Ssam * 27104477Ssam * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 28104477Ssam * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 29104477Ssam * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 30104477Ssam * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 31104477Ssam * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 32104477Ssam * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33104477Ssam * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34104477Ssam * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35104477Ssam * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 36104477Ssam * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37104477Ssam * 38104477Ssam * Effort sponsored in part by the Defense Advanced Research Projects 39104477Ssam * Agency (DARPA) and Air Force Research Laboratory, Air Force 40104477Ssam * Materiel Command, USAF, under agreement number F30602-01-2-0537. 41104477Ssam * 42104477Ssam */ 43104477Ssam 44104477Ssam/* 45104477Ssam * Driver for the Hifn 7751 encryption processor. 46104477Ssam */ 47112124Ssam#include "opt_hifn.h" 48104477Ssam 49104477Ssam#include <sys/param.h> 50104477Ssam#include <sys/systm.h> 51104477Ssam#include <sys/proc.h> 52104477Ssam#include <sys/errno.h> 53104477Ssam#include <sys/malloc.h> 54104477Ssam#include <sys/kernel.h> 55104477Ssam#include <sys/mbuf.h> 56104477Ssam#include <sys/lock.h> 57104477Ssam#include <sys/mutex.h> 58104477Ssam#include <sys/sysctl.h> 59104477Ssam 60104477Ssam#include <vm/vm.h> 61104477Ssam#include <vm/pmap.h> 62104477Ssam 63104477Ssam#include <machine/clock.h> 64104477Ssam#include <machine/bus.h> 65104477Ssam#include <machine/resource.h> 66104477Ssam#include <sys/bus.h> 67104477Ssam#include <sys/rman.h> 68104477Ssam 69104477Ssam#include <opencrypto/cryptodev.h> 70104477Ssam#include <sys/random.h> 71104477Ssam 72104477Ssam#include <pci/pcivar.h> 73104477Ssam#include <pci/pcireg.h> 74112124Ssam 75112124Ssam#ifdef HIFN_RNDTEST 76112124Ssam#include <dev/rndtest/rndtest.h> 77112124Ssam#endif 78104477Ssam#include <dev/hifn/hifn7751reg.h> 79104477Ssam#include <dev/hifn/hifn7751var.h> 80104477Ssam 81104477Ssam/* 82104477Ssam * Prototypes and count for the pci_device structure 83104477Ssam */ 84104477Ssamstatic int hifn_probe(device_t); 85104477Ssamstatic int hifn_attach(device_t); 86104477Ssamstatic int hifn_detach(device_t); 87104477Ssamstatic int hifn_suspend(device_t); 88104477Ssamstatic int hifn_resume(device_t); 89104477Ssamstatic void hifn_shutdown(device_t); 90104477Ssam 91104477Ssamstatic device_method_t hifn_methods[] = { 92104477Ssam /* Device interface */ 93104477Ssam DEVMETHOD(device_probe, hifn_probe), 94104477Ssam DEVMETHOD(device_attach, hifn_attach), 95104477Ssam DEVMETHOD(device_detach, hifn_detach), 96104477Ssam DEVMETHOD(device_suspend, hifn_suspend), 97104477Ssam DEVMETHOD(device_resume, hifn_resume), 98104477Ssam DEVMETHOD(device_shutdown, hifn_shutdown), 99104477Ssam 100104477Ssam /* bus interface */ 101104477Ssam DEVMETHOD(bus_print_child, bus_generic_print_child), 102104477Ssam DEVMETHOD(bus_driver_added, bus_generic_driver_added), 103104477Ssam 104104477Ssam { 0, 0 } 105104477Ssam}; 106104477Ssamstatic driver_t hifn_driver = { 107104477Ssam "hifn", 108104477Ssam hifn_methods, 109104477Ssam sizeof (struct hifn_softc) 110104477Ssam}; 111104477Ssamstatic devclass_t hifn_devclass; 112104477Ssam 113104477SsamDRIVER_MODULE(hifn, pci, hifn_driver, hifn_devclass, 0, 0); 114105251SmarkmMODULE_DEPEND(hifn, crypto, 1, 1, 1); 115112124Ssam#ifdef HIFN_RNDTEST 116112124SsamMODULE_DEPEND(hifn, rndtest, 1, 1, 1); 117112124Ssam#endif 118104477Ssam 119104477Ssamstatic void hifn_reset_board(struct hifn_softc *, int); 120104477Ssamstatic void hifn_reset_puc(struct hifn_softc *); 121104477Ssamstatic void hifn_puc_wait(struct hifn_softc *); 122104477Ssamstatic int hifn_enable_crypto(struct hifn_softc *); 123104477Ssamstatic void hifn_set_retry(struct hifn_softc *sc); 124104477Ssamstatic void hifn_init_dma(struct hifn_softc *); 125104477Ssamstatic void hifn_init_pci_registers(struct hifn_softc *); 126104477Ssamstatic int hifn_sramsize(struct hifn_softc *); 127104477Ssamstatic int hifn_dramsize(struct hifn_softc *); 128104477Ssamstatic int hifn_ramtype(struct hifn_softc *); 129104477Ssamstatic void hifn_sessions(struct hifn_softc *); 130104477Ssamstatic void hifn_intr(void *); 131104477Ssamstatic u_int hifn_write_command(struct hifn_command *, u_int8_t *); 132104477Ssamstatic u_int32_t hifn_next_signature(u_int32_t a, u_int cnt); 133104477Ssamstatic int hifn_newsession(void *, u_int32_t *, struct cryptoini *); 134104477Ssamstatic int hifn_freesession(void *, u_int64_t); 135104477Ssamstatic int hifn_process(void *, struct cryptop *, int); 136104477Ssamstatic void hifn_callback(struct hifn_softc *, struct hifn_command *, u_int8_t *); 137104477Ssamstatic int hifn_crypto(struct hifn_softc *, struct hifn_command *, struct cryptop *, int); 138104477Ssamstatic int hifn_readramaddr(struct hifn_softc *, int, u_int8_t *); 139104477Ssamstatic int hifn_writeramaddr(struct hifn_softc *, int, u_int8_t *); 140104477Ssamstatic int hifn_dmamap_load_src(struct hifn_softc *, struct hifn_command *); 141104477Ssamstatic int hifn_dmamap_load_dst(struct hifn_softc *, struct hifn_command *); 142104477Ssamstatic int hifn_init_pubrng(struct hifn_softc *); 143104477Ssamstatic void hifn_rng(void *); 144104477Ssamstatic void hifn_tick(void *); 145104477Ssamstatic void hifn_abort(struct hifn_softc *); 146104477Ssamstatic void hifn_alloc_slot(struct hifn_softc *, int *, int *, int *, int *); 147104477Ssam 148104477Ssamstatic void hifn_write_reg_0(struct hifn_softc *, bus_size_t, u_int32_t); 149104477Ssamstatic void hifn_write_reg_1(struct hifn_softc *, bus_size_t, u_int32_t); 150104477Ssam 151104477Ssamstatic __inline__ u_int32_t 152104477SsamREAD_REG_0(struct hifn_softc *sc, bus_size_t reg) 153104477Ssam{ 154104477Ssam u_int32_t v = bus_space_read_4(sc->sc_st0, sc->sc_sh0, reg); 155104477Ssam sc->sc_bar0_lastreg = (bus_size_t) -1; 156104477Ssam return (v); 157104477Ssam} 158104477Ssam#define WRITE_REG_0(sc, reg, val) hifn_write_reg_0(sc, reg, val) 159104477Ssam 160104477Ssamstatic __inline__ u_int32_t 161104477SsamREAD_REG_1(struct hifn_softc *sc, bus_size_t reg) 162104477Ssam{ 163104477Ssam u_int32_t v = bus_space_read_4(sc->sc_st1, sc->sc_sh1, reg); 164104477Ssam sc->sc_bar1_lastreg = (bus_size_t) -1; 165104477Ssam return (v); 166104477Ssam} 167104477Ssam#define WRITE_REG_1(sc, reg, val) hifn_write_reg_1(sc, reg, val) 168104477Ssam 169109596SsamSYSCTL_NODE(_hw, OID_AUTO, hifn, CTLFLAG_RD, 0, "Hifn driver parameters"); 170109596Ssam 171104477Ssam#ifdef HIFN_DEBUG 172104477Ssamstatic int hifn_debug = 0; 173109596SsamSYSCTL_INT(_hw_hifn, OID_AUTO, debug, CTLFLAG_RW, &hifn_debug, 174109596Ssam 0, "control debugging msgs"); 175104477Ssam#endif 176104477Ssam 177104477Ssamstatic struct hifn_stats hifnstats; 178109596SsamSYSCTL_STRUCT(_hw_hifn, OID_AUTO, stats, CTLFLAG_RD, &hifnstats, 179109596Ssam hifn_stats, "driver statistics"); 180112121Ssamstatic int hifn_maxbatch = 1; 181109596SsamSYSCTL_INT(_hw_hifn, OID_AUTO, maxbatch, CTLFLAG_RW, &hifn_maxbatch, 182109596Ssam 0, "max ops to batch w/o interrupt"); 183104477Ssam 184104477Ssam/* 185104477Ssam * Probe for a supported device. The PCI vendor and device 186104477Ssam * IDs are used to detect devices we know how to handle. 187104477Ssam */ 188104477Ssamstatic int 189104477Ssamhifn_probe(device_t dev) 190104477Ssam{ 191104477Ssam if (pci_get_vendor(dev) == PCI_VENDOR_INVERTEX && 192104477Ssam pci_get_device(dev) == PCI_PRODUCT_INVERTEX_AEON) 193104477Ssam return (0); 194104477Ssam if (pci_get_vendor(dev) == PCI_VENDOR_HIFN && 195104477Ssam (pci_get_device(dev) == PCI_PRODUCT_HIFN_7751 || 196104477Ssam pci_get_device(dev) == PCI_PRODUCT_HIFN_7951 || 197104477Ssam pci_get_device(dev) == PCI_PRODUCT_HIFN_7811)) 198104477Ssam return (0); 199104477Ssam if (pci_get_vendor(dev) == PCI_VENDOR_NETSEC && 200104477Ssam pci_get_device(dev) == PCI_PRODUCT_NETSEC_7751) 201104477Ssam return (0); 202104477Ssam return (ENXIO); 203104477Ssam} 204104477Ssam 205104477Ssamstatic void 206104477Ssamhifn_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 207104477Ssam{ 208104477Ssam bus_addr_t *paddr = (bus_addr_t*) arg; 209104477Ssam *paddr = segs->ds_addr; 210104477Ssam} 211104477Ssam 212104477Ssamstatic const char* 213104477Ssamhifn_partname(struct hifn_softc *sc) 214104477Ssam{ 215104477Ssam /* XXX sprintf numbers when not decoded */ 216104477Ssam switch (pci_get_vendor(sc->sc_dev)) { 217104477Ssam case PCI_VENDOR_HIFN: 218104477Ssam switch (pci_get_device(sc->sc_dev)) { 219104477Ssam case PCI_PRODUCT_HIFN_6500: return "Hifn 6500"; 220104477Ssam case PCI_PRODUCT_HIFN_7751: return "Hifn 7751"; 221104477Ssam case PCI_PRODUCT_HIFN_7811: return "Hifn 7811"; 222104477Ssam case PCI_PRODUCT_HIFN_7951: return "Hifn 7951"; 223104477Ssam } 224104477Ssam return "Hifn unknown-part"; 225104477Ssam case PCI_VENDOR_INVERTEX: 226104477Ssam switch (pci_get_device(sc->sc_dev)) { 227104477Ssam case PCI_PRODUCT_INVERTEX_AEON: return "Invertex AEON"; 228104477Ssam } 229104477Ssam return "Invertex unknown-part"; 230104477Ssam case PCI_VENDOR_NETSEC: 231104477Ssam switch (pci_get_device(sc->sc_dev)) { 232104477Ssam case PCI_PRODUCT_NETSEC_7751: return "NetSec 7751"; 233104477Ssam } 234104477Ssam return "NetSec unknown-part"; 235104477Ssam } 236104477Ssam return "Unknown-vendor unknown-part"; 237104477Ssam} 238104477Ssam 239112124Ssamstatic void 240112124Ssamdefault_harvest(struct rndtest_state *rsp, void *buf, u_int count) 241112124Ssam{ 242112124Ssam random_harvest(buf, count, count*NBBY, 0, RANDOM_PURE); 243112124Ssam} 244112124Ssam 245104477Ssam/* 246104477Ssam * Attach an interface that successfully probed. 247104477Ssam */ 248104477Ssamstatic int 249104477Ssamhifn_attach(device_t dev) 250104477Ssam{ 251104477Ssam struct hifn_softc *sc = device_get_softc(dev); 252104477Ssam u_int32_t cmd; 253104477Ssam caddr_t kva; 254104477Ssam int rseg, rid; 255104477Ssam char rbase; 256104477Ssam u_int16_t ena, rev; 257104477Ssam 258104477Ssam KASSERT(sc != NULL, ("hifn_attach: null software carrier!")); 259104477Ssam bzero(sc, sizeof (*sc)); 260104477Ssam sc->sc_dev = dev; 261104477Ssam 262115748Ssam mtx_init(&sc->sc_mtx, device_get_nameunit(dev), "hifn driver", MTX_DEF); 263104477Ssam 264104477Ssam /* XXX handle power management */ 265104477Ssam 266104477Ssam /* 267104477Ssam * The 7951 has a random number generator and 268104477Ssam * public key support; note this. 269104477Ssam */ 270104477Ssam if (pci_get_vendor(dev) == PCI_VENDOR_HIFN && 271104477Ssam pci_get_device(dev) == PCI_PRODUCT_HIFN_7951) 272104477Ssam sc->sc_flags = HIFN_HAS_RNG | HIFN_HAS_PUBLIC; 273104477Ssam /* 274104477Ssam * The 7811 has a random number generator and 275104477Ssam * we also note it's identity 'cuz of some quirks. 276104477Ssam */ 277104477Ssam if (pci_get_vendor(dev) == PCI_VENDOR_HIFN && 278104477Ssam pci_get_device(dev) == PCI_PRODUCT_HIFN_7811) 279104477Ssam sc->sc_flags |= HIFN_IS_7811 | HIFN_HAS_RNG; 280104477Ssam 281104477Ssam /* 282104477Ssam * Configure support for memory-mapped access to 283104477Ssam * registers and for DMA operations. 284104477Ssam */ 285104477Ssam#define PCIM_ENA (PCIM_CMD_MEMEN|PCIM_CMD_BUSMASTEREN) 286104477Ssam cmd = pci_read_config(dev, PCIR_COMMAND, 4); 287104477Ssam cmd |= PCIM_ENA; 288104477Ssam pci_write_config(dev, PCIR_COMMAND, cmd, 4); 289104477Ssam cmd = pci_read_config(dev, PCIR_COMMAND, 4); 290104477Ssam if ((cmd & PCIM_ENA) != PCIM_ENA) { 291104477Ssam device_printf(dev, "failed to enable %s\n", 292104477Ssam (cmd & PCIM_ENA) == 0 ? 293104477Ssam "memory mapping & bus mastering" : 294104477Ssam (cmd & PCIM_CMD_MEMEN) == 0 ? 295104477Ssam "memory mapping" : "bus mastering"); 296104477Ssam goto fail_pci; 297104477Ssam } 298104477Ssam#undef PCIM_ENA 299104477Ssam 300104477Ssam /* 301104477Ssam * Setup PCI resources. Note that we record the bus 302104477Ssam * tag and handle for each register mapping, this is 303104477Ssam * used by the READ_REG_0, WRITE_REG_0, READ_REG_1, 304104477Ssam * and WRITE_REG_1 macros throughout the driver. 305104477Ssam */ 306104477Ssam rid = HIFN_BAR0; 307104477Ssam sc->sc_bar0res = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid, 308104477Ssam 0, ~0, 1, RF_ACTIVE); 309104477Ssam if (sc->sc_bar0res == NULL) { 310104477Ssam device_printf(dev, "cannot map bar%d register space\n", 0); 311104477Ssam goto fail_pci; 312104477Ssam } 313104477Ssam sc->sc_st0 = rman_get_bustag(sc->sc_bar0res); 314104477Ssam sc->sc_sh0 = rman_get_bushandle(sc->sc_bar0res); 315104477Ssam sc->sc_bar0_lastreg = (bus_size_t) -1; 316104477Ssam 317104477Ssam rid = HIFN_BAR1; 318104477Ssam sc->sc_bar1res = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid, 319104477Ssam 0, ~0, 1, RF_ACTIVE); 320104477Ssam if (sc->sc_bar1res == NULL) { 321104477Ssam device_printf(dev, "cannot map bar%d register space\n", 1); 322104477Ssam goto fail_io0; 323104477Ssam } 324104477Ssam sc->sc_st1 = rman_get_bustag(sc->sc_bar1res); 325104477Ssam sc->sc_sh1 = rman_get_bushandle(sc->sc_bar1res); 326104477Ssam sc->sc_bar1_lastreg = (bus_size_t) -1; 327104477Ssam 328104477Ssam hifn_set_retry(sc); 329104477Ssam 330104477Ssam /* 331104477Ssam * Setup the area where the Hifn DMA's descriptors 332104477Ssam * and associated data structures. 333104477Ssam */ 334104477Ssam if (bus_dma_tag_create(NULL, /* parent */ 335104477Ssam 1, 0, /* alignment,boundary */ 336104477Ssam BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 337104477Ssam BUS_SPACE_MAXADDR, /* highaddr */ 338104477Ssam NULL, NULL, /* filter, filterarg */ 339104477Ssam HIFN_MAX_DMALEN, /* maxsize */ 340104477Ssam MAX_SCATTER, /* nsegments */ 341104477Ssam HIFN_MAX_SEGLEN, /* maxsegsize */ 342104477Ssam BUS_DMA_ALLOCNOW, /* flags */ 343104477Ssam &sc->sc_dmat)) { 344104477Ssam device_printf(dev, "cannot allocate DMA tag\n"); 345104477Ssam goto fail_io1; 346104477Ssam } 347104477Ssam if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &sc->sc_dmamap)) { 348104477Ssam device_printf(dev, "cannot create dma map\n"); 349104477Ssam bus_dma_tag_destroy(sc->sc_dmat); 350104477Ssam goto fail_io1; 351104477Ssam } 352104477Ssam if (bus_dmamem_alloc(sc->sc_dmat, (void**) &kva, BUS_DMA_NOWAIT, &sc->sc_dmamap)) { 353104477Ssam device_printf(dev, "cannot alloc dma buffer\n"); 354104477Ssam bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap); 355104477Ssam bus_dma_tag_destroy(sc->sc_dmat); 356104477Ssam goto fail_io1; 357104477Ssam } 358104477Ssam if (bus_dmamap_load(sc->sc_dmat, sc->sc_dmamap, kva, 359104477Ssam sizeof (*sc->sc_dma), 360104477Ssam hifn_dmamap_cb, &sc->sc_dma_physaddr, 361104477Ssam BUS_DMA_NOWAIT)) { 362104477Ssam device_printf(dev, "cannot load dma map\n"); 363104477Ssam bus_dmamem_free(sc->sc_dmat, kva, sc->sc_dmamap); 364104477Ssam bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap); 365104477Ssam bus_dma_tag_destroy(sc->sc_dmat); 366104477Ssam goto fail_io1; 367104477Ssam } 368104477Ssam sc->sc_dma = (struct hifn_dma *)kva; 369104477Ssam bzero(sc->sc_dma, sizeof(*sc->sc_dma)); 370104477Ssam 371104477Ssam KASSERT(sc->sc_st0 != NULL, ("hifn_attach: null bar0 tag!")); 372104477Ssam KASSERT(sc->sc_sh0 != NULL, ("hifn_attach: null bar0 handle!")); 373104477Ssam KASSERT(sc->sc_st1 != NULL, ("hifn_attach: null bar1 tag!")); 374104477Ssam KASSERT(sc->sc_sh1 != NULL, ("hifn_attach: null bar1 handle!")); 375104477Ssam 376104477Ssam /* 377104477Ssam * Reset the board and do the ``secret handshake'' 378104477Ssam * to enable the crypto support. Then complete the 379104477Ssam * initialization procedure by setting up the interrupt 380104477Ssam * and hooking in to the system crypto support so we'll 381104477Ssam * get used for system services like the crypto device, 382104477Ssam * IPsec, RNG device, etc. 383104477Ssam */ 384104477Ssam hifn_reset_board(sc, 0); 385104477Ssam 386104477Ssam if (hifn_enable_crypto(sc) != 0) { 387104477Ssam device_printf(dev, "crypto enabling failed\n"); 388104477Ssam goto fail_mem; 389104477Ssam } 390104477Ssam hifn_reset_puc(sc); 391104477Ssam 392104477Ssam hifn_init_dma(sc); 393104477Ssam hifn_init_pci_registers(sc); 394104477Ssam 395104477Ssam if (hifn_ramtype(sc)) 396104477Ssam goto fail_mem; 397104477Ssam 398104477Ssam if (sc->sc_drammodel == 0) 399104477Ssam hifn_sramsize(sc); 400104477Ssam else 401104477Ssam hifn_dramsize(sc); 402104477Ssam 403104477Ssam /* 404104477Ssam * Workaround for NetSec 7751 rev A: half ram size because two 405104477Ssam * of the address lines were left floating 406104477Ssam */ 407104477Ssam if (pci_get_vendor(dev) == PCI_VENDOR_NETSEC && 408104477Ssam pci_get_device(dev) == PCI_PRODUCT_NETSEC_7751 && 409104477Ssam pci_get_revid(dev) == 0x61) /*XXX???*/ 410104477Ssam sc->sc_ramsize >>= 1; 411104477Ssam 412104477Ssam /* 413104477Ssam * Arrange the interrupt line. 414104477Ssam */ 415104477Ssam rid = 0; 416104477Ssam sc->sc_irq = bus_alloc_resource(dev, SYS_RES_IRQ, &rid, 417104477Ssam 0, ~0, 1, RF_SHAREABLE|RF_ACTIVE); 418104477Ssam if (sc->sc_irq == NULL) { 419104477Ssam device_printf(dev, "could not map interrupt\n"); 420104477Ssam goto fail_mem; 421104477Ssam } 422104477Ssam /* 423104477Ssam * NB: Network code assumes we are blocked with splimp() 424104477Ssam * so make sure the IRQ is marked appropriately. 425104477Ssam */ 426115748Ssam if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE, 427104477Ssam hifn_intr, sc, &sc->sc_intrhand)) { 428104477Ssam device_printf(dev, "could not setup interrupt\n"); 429104477Ssam goto fail_intr2; 430104477Ssam } 431104477Ssam 432104477Ssam hifn_sessions(sc); 433104477Ssam 434104477Ssam /* 435104477Ssam * NB: Keep only the low 16 bits; this masks the chip id 436104477Ssam * from the 7951. 437104477Ssam */ 438104477Ssam rev = READ_REG_1(sc, HIFN_1_REVID) & 0xffff; 439104477Ssam 440104477Ssam rseg = sc->sc_ramsize / 1024; 441104477Ssam rbase = 'K'; 442104477Ssam if (sc->sc_ramsize >= (1024 * 1024)) { 443104477Ssam rbase = 'M'; 444104477Ssam rseg /= 1024; 445104477Ssam } 446104477Ssam device_printf(sc->sc_dev, "%s, rev %u, %d%cB %cram, %u sessions\n", 447104477Ssam hifn_partname(sc), rev, 448104477Ssam rseg, rbase, sc->sc_drammodel ? 'd' : 's', 449104477Ssam sc->sc_maxses); 450104477Ssam 451104477Ssam sc->sc_cid = crypto_get_driverid(0); 452104477Ssam if (sc->sc_cid < 0) { 453104477Ssam device_printf(dev, "could not get crypto driver id\n"); 454104477Ssam goto fail_intr; 455104477Ssam } 456104477Ssam 457104477Ssam WRITE_REG_0(sc, HIFN_0_PUCNFG, 458104477Ssam READ_REG_0(sc, HIFN_0_PUCNFG) | HIFN_PUCNFG_CHIPID); 459104477Ssam ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA; 460104477Ssam 461104477Ssam switch (ena) { 462104477Ssam case HIFN_PUSTAT_ENA_2: 463104477Ssam crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0, 464104477Ssam hifn_newsession, hifn_freesession, hifn_process, sc); 465104477Ssam crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0, 466104477Ssam hifn_newsession, hifn_freesession, hifn_process, sc); 467104477Ssam /*FALLTHROUGH*/ 468104477Ssam case HIFN_PUSTAT_ENA_1: 469104477Ssam crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0, 470104477Ssam hifn_newsession, hifn_freesession, hifn_process, sc); 471104477Ssam crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0, 472104477Ssam hifn_newsession, hifn_freesession, hifn_process, sc); 473104477Ssam crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0, 474104477Ssam hifn_newsession, hifn_freesession, hifn_process, sc); 475104477Ssam crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0, 476104477Ssam hifn_newsession, hifn_freesession, hifn_process, sc); 477104477Ssam crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0, 478104477Ssam hifn_newsession, hifn_freesession, hifn_process, sc); 479104477Ssam break; 480104477Ssam } 481104477Ssam 482104477Ssam bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 483104477Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 484104477Ssam 485104477Ssam if (sc->sc_flags & (HIFN_HAS_PUBLIC | HIFN_HAS_RNG)) 486104477Ssam hifn_init_pubrng(sc); 487104477Ssam 488104918Ssam /* NB: 1 means the callout runs w/o Giant locked */ 489104918Ssam callout_init(&sc->sc_tickto, 1); 490104477Ssam callout_reset(&sc->sc_tickto, hz, hifn_tick, sc); 491104477Ssam 492104477Ssam return (0); 493104477Ssam 494104477Ssamfail_intr: 495104477Ssam bus_teardown_intr(dev, sc->sc_irq, sc->sc_intrhand); 496104477Ssamfail_intr2: 497104477Ssam /* XXX don't store rid */ 498104477Ssam bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 499104477Ssamfail_mem: 500104477Ssam bus_dmamap_unload(sc->sc_dmat, sc->sc_dmamap); 501104477Ssam bus_dmamem_free(sc->sc_dmat, sc->sc_dma, sc->sc_dmamap); 502104477Ssam bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap); 503104477Ssam bus_dma_tag_destroy(sc->sc_dmat); 504104477Ssam 505104477Ssam /* Turn off DMA polling */ 506104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 507104477Ssam HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 508104477Ssamfail_io1: 509104477Ssam bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR1, sc->sc_bar1res); 510104477Ssamfail_io0: 511104477Ssam bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR0, sc->sc_bar0res); 512104477Ssamfail_pci: 513104477Ssam mtx_destroy(&sc->sc_mtx); 514104477Ssam return (ENXIO); 515104477Ssam} 516104477Ssam 517104477Ssam/* 518104477Ssam * Detach an interface that successfully probed. 519104477Ssam */ 520104477Ssamstatic int 521104477Ssamhifn_detach(device_t dev) 522104477Ssam{ 523104477Ssam struct hifn_softc *sc = device_get_softc(dev); 524104477Ssam 525104477Ssam KASSERT(sc != NULL, ("hifn_detach: null software carrier!")); 526104477Ssam 527115748Ssam /* disable interrupts */ 528115748Ssam WRITE_REG_1(sc, HIFN_1_DMA_IER, 0); 529104477Ssam 530104477Ssam /*XXX other resources */ 531104477Ssam callout_stop(&sc->sc_tickto); 532104477Ssam callout_stop(&sc->sc_rngto); 533104477Ssam 534104477Ssam /* Turn off DMA polling */ 535104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 536104477Ssam HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 537104477Ssam 538104477Ssam crypto_unregister_all(sc->sc_cid); 539104477Ssam 540104477Ssam bus_generic_detach(dev); /*XXX should be no children, right? */ 541104477Ssam 542104477Ssam bus_teardown_intr(dev, sc->sc_irq, sc->sc_intrhand); 543104477Ssam /* XXX don't store rid */ 544104477Ssam bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 545104477Ssam 546104477Ssam bus_dmamap_unload(sc->sc_dmat, sc->sc_dmamap); 547104477Ssam bus_dmamem_free(sc->sc_dmat, sc->sc_dma, sc->sc_dmamap); 548104477Ssam bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap); 549104477Ssam bus_dma_tag_destroy(sc->sc_dmat); 550104477Ssam 551104477Ssam bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR1, sc->sc_bar1res); 552104477Ssam bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR0, sc->sc_bar0res); 553104477Ssam 554104477Ssam mtx_destroy(&sc->sc_mtx); 555104477Ssam 556104477Ssam return (0); 557104477Ssam} 558104477Ssam 559104477Ssam/* 560104477Ssam * Stop all chip I/O so that the kernel's probe routines don't 561104477Ssam * get confused by errant DMAs when rebooting. 562104477Ssam */ 563104477Ssamstatic void 564104477Ssamhifn_shutdown(device_t dev) 565104477Ssam{ 566104477Ssam#ifdef notyet 567104477Ssam hifn_stop(device_get_softc(dev)); 568104477Ssam#endif 569104477Ssam} 570104477Ssam 571104477Ssam/* 572104477Ssam * Device suspend routine. Stop the interface and save some PCI 573104477Ssam * settings in case the BIOS doesn't restore them properly on 574104477Ssam * resume. 575104477Ssam */ 576104477Ssamstatic int 577104477Ssamhifn_suspend(device_t dev) 578104477Ssam{ 579104477Ssam struct hifn_softc *sc = device_get_softc(dev); 580104477Ssam#ifdef notyet 581104477Ssam int i; 582104477Ssam 583104477Ssam hifn_stop(sc); 584104477Ssam for (i = 0; i < 5; i++) 585104477Ssam sc->saved_maps[i] = pci_read_config(dev, PCIR_MAPS + i * 4, 4); 586104477Ssam sc->saved_biosaddr = pci_read_config(dev, PCIR_BIOS, 4); 587104477Ssam sc->saved_intline = pci_read_config(dev, PCIR_INTLINE, 1); 588104477Ssam sc->saved_cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1); 589104477Ssam sc->saved_lattimer = pci_read_config(dev, PCIR_LATTIMER, 1); 590104477Ssam#endif 591104477Ssam sc->sc_suspended = 1; 592104477Ssam 593104477Ssam return (0); 594104477Ssam} 595104477Ssam 596104477Ssam/* 597104477Ssam * Device resume routine. Restore some PCI settings in case the BIOS 598104477Ssam * doesn't, re-enable busmastering, and restart the interface if 599104477Ssam * appropriate. 600104477Ssam */ 601104477Ssamstatic int 602104477Ssamhifn_resume(device_t dev) 603104477Ssam{ 604104477Ssam struct hifn_softc *sc = device_get_softc(dev); 605104477Ssam#ifdef notyet 606104477Ssam int i; 607104477Ssam 608104477Ssam /* better way to do this? */ 609104477Ssam for (i = 0; i < 5; i++) 610104477Ssam pci_write_config(dev, PCIR_MAPS + i * 4, sc->saved_maps[i], 4); 611104477Ssam pci_write_config(dev, PCIR_BIOS, sc->saved_biosaddr, 4); 612104477Ssam pci_write_config(dev, PCIR_INTLINE, sc->saved_intline, 1); 613104477Ssam pci_write_config(dev, PCIR_CACHELNSZ, sc->saved_cachelnsz, 1); 614104477Ssam pci_write_config(dev, PCIR_LATTIMER, sc->saved_lattimer, 1); 615104477Ssam 616104477Ssam /* reenable busmastering */ 617104477Ssam pci_enable_busmaster(dev); 618104477Ssam pci_enable_io(dev, HIFN_RES); 619104477Ssam 620104477Ssam /* reinitialize interface if necessary */ 621104477Ssam if (ifp->if_flags & IFF_UP) 622104477Ssam rl_init(sc); 623104477Ssam#endif 624104477Ssam sc->sc_suspended = 0; 625104477Ssam 626104477Ssam return (0); 627104477Ssam} 628104477Ssam 629104477Ssamstatic int 630104477Ssamhifn_init_pubrng(struct hifn_softc *sc) 631104477Ssam{ 632104477Ssam u_int32_t r; 633104477Ssam int i; 634104477Ssam 635112124Ssam#ifdef HIFN_RNDTEST 636112124Ssam sc->sc_rndtest = rndtest_attach(sc->sc_dev); 637112124Ssam if (sc->sc_rndtest) 638112124Ssam sc->sc_harvest = rndtest_harvest; 639112124Ssam else 640112124Ssam sc->sc_harvest = default_harvest; 641112124Ssam#else 642112124Ssam sc->sc_harvest = default_harvest; 643112124Ssam#endif 644104477Ssam if ((sc->sc_flags & HIFN_IS_7811) == 0) { 645104477Ssam /* Reset 7951 public key/rng engine */ 646104477Ssam WRITE_REG_1(sc, HIFN_1_PUB_RESET, 647104477Ssam READ_REG_1(sc, HIFN_1_PUB_RESET) | HIFN_PUBRST_RESET); 648104477Ssam 649104477Ssam for (i = 0; i < 100; i++) { 650104477Ssam DELAY(1000); 651104477Ssam if ((READ_REG_1(sc, HIFN_1_PUB_RESET) & 652104477Ssam HIFN_PUBRST_RESET) == 0) 653104477Ssam break; 654104477Ssam } 655104477Ssam 656104477Ssam if (i == 100) { 657104477Ssam device_printf(sc->sc_dev, "public key init failed\n"); 658104477Ssam return (1); 659104477Ssam } 660104477Ssam } 661104477Ssam 662104477Ssam /* Enable the rng, if available */ 663104477Ssam if (sc->sc_flags & HIFN_HAS_RNG) { 664104477Ssam if (sc->sc_flags & HIFN_IS_7811) { 665104477Ssam r = READ_REG_1(sc, HIFN_1_7811_RNGENA); 666104477Ssam if (r & HIFN_7811_RNGENA_ENA) { 667104477Ssam r &= ~HIFN_7811_RNGENA_ENA; 668104477Ssam WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r); 669104477Ssam } 670104477Ssam WRITE_REG_1(sc, HIFN_1_7811_RNGCFG, 671104477Ssam HIFN_7811_RNGCFG_DEFL); 672104477Ssam r |= HIFN_7811_RNGENA_ENA; 673104477Ssam WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r); 674104477Ssam } else 675104477Ssam WRITE_REG_1(sc, HIFN_1_RNG_CONFIG, 676104477Ssam READ_REG_1(sc, HIFN_1_RNG_CONFIG) | 677104477Ssam HIFN_RNGCFG_ENA); 678104477Ssam 679104477Ssam sc->sc_rngfirst = 1; 680104477Ssam if (hz >= 100) 681104477Ssam sc->sc_rnghz = hz / 100; 682104477Ssam else 683104477Ssam sc->sc_rnghz = 1; 684105190Ssam /* NB: 1 means the callout runs w/o Giant locked */ 685105190Ssam callout_init(&sc->sc_rngto, 1); 686104477Ssam callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc); 687104477Ssam } 688104477Ssam 689104477Ssam /* Enable public key engine, if available */ 690104477Ssam if (sc->sc_flags & HIFN_HAS_PUBLIC) { 691104477Ssam WRITE_REG_1(sc, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE); 692104477Ssam sc->sc_dmaier |= HIFN_DMAIER_PUBDONE; 693104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 694104477Ssam } 695104477Ssam 696104477Ssam return (0); 697104477Ssam} 698104477Ssam 699104477Ssamstatic void 700104477Ssamhifn_rng(void *vsc) 701104477Ssam{ 702104477Ssam#define RANDOM_BITS(n) (n)*sizeof (u_int32_t), (n)*sizeof (u_int32_t)*NBBY, 0 703104477Ssam struct hifn_softc *sc = vsc; 704104477Ssam u_int32_t sts, num[2]; 705104477Ssam int i; 706104477Ssam 707104477Ssam if (sc->sc_flags & HIFN_IS_7811) { 708104477Ssam for (i = 0; i < 5; i++) { 709104477Ssam sts = READ_REG_1(sc, HIFN_1_7811_RNGSTS); 710104477Ssam if (sts & HIFN_7811_RNGSTS_UFL) { 711104477Ssam device_printf(sc->sc_dev, 712104477Ssam "RNG underflow: disabling\n"); 713104477Ssam return; 714104477Ssam } 715104477Ssam if ((sts & HIFN_7811_RNGSTS_RDY) == 0) 716104477Ssam break; 717104477Ssam 718104477Ssam /* 719104477Ssam * There are at least two words in the RNG FIFO 720104477Ssam * at this point. 721104477Ssam */ 722104477Ssam num[0] = READ_REG_1(sc, HIFN_1_7811_RNGDAT); 723104477Ssam num[1] = READ_REG_1(sc, HIFN_1_7811_RNGDAT); 724104477Ssam /* NB: discard first data read */ 725104477Ssam if (sc->sc_rngfirst) 726104477Ssam sc->sc_rngfirst = 0; 727104477Ssam else 728112124Ssam (*sc->sc_harvest)(sc->sc_rndtest, 729112124Ssam num, sizeof (num)); 730104477Ssam } 731104477Ssam } else { 732104477Ssam num[0] = READ_REG_1(sc, HIFN_1_RNG_DATA); 733104477Ssam 734104477Ssam /* NB: discard first data read */ 735104477Ssam if (sc->sc_rngfirst) 736104477Ssam sc->sc_rngfirst = 0; 737104477Ssam else 738112124Ssam (*sc->sc_harvest)(sc->sc_rndtest, 739112124Ssam num, sizeof (num[0])); 740104477Ssam } 741104477Ssam 742104477Ssam callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc); 743104477Ssam#undef RANDOM_BITS 744104477Ssam} 745104477Ssam 746104477Ssamstatic void 747104477Ssamhifn_puc_wait(struct hifn_softc *sc) 748104477Ssam{ 749104477Ssam int i; 750104477Ssam 751104477Ssam for (i = 5000; i > 0; i--) { 752104477Ssam DELAY(1); 753104477Ssam if (!(READ_REG_0(sc, HIFN_0_PUCTRL) & HIFN_PUCTRL_RESET)) 754104477Ssam break; 755104477Ssam } 756104477Ssam if (!i) 757104477Ssam device_printf(sc->sc_dev, "proc unit did not reset\n"); 758104477Ssam} 759104477Ssam 760104477Ssam/* 761104477Ssam * Reset the processing unit. 762104477Ssam */ 763104477Ssamstatic void 764104477Ssamhifn_reset_puc(struct hifn_softc *sc) 765104477Ssam{ 766104477Ssam /* Reset processing unit */ 767104477Ssam WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); 768104477Ssam hifn_puc_wait(sc); 769104477Ssam} 770104477Ssam 771104477Ssam/* 772104477Ssam * Set the Retry and TRDY registers; note that we set them to 773104477Ssam * zero because the 7811 locks up when forced to retry (section 774104477Ssam * 3.6 of "Specification Update SU-0014-04". Not clear if we 775104477Ssam * should do this for all Hifn parts, but it doesn't seem to hurt. 776104477Ssam */ 777104477Ssamstatic void 778104477Ssamhifn_set_retry(struct hifn_softc *sc) 779104477Ssam{ 780104477Ssam /* NB: RETRY only responds to 8-bit reads/writes */ 781104477Ssam pci_write_config(sc->sc_dev, HIFN_RETRY_TIMEOUT, 0, 1); 782104477Ssam pci_write_config(sc->sc_dev, HIFN_TRDY_TIMEOUT, 0, 4); 783104477Ssam} 784104477Ssam 785104477Ssam/* 786104477Ssam * Resets the board. Values in the regesters are left as is 787104477Ssam * from the reset (i.e. initial values are assigned elsewhere). 788104477Ssam */ 789104477Ssamstatic void 790104477Ssamhifn_reset_board(struct hifn_softc *sc, int full) 791104477Ssam{ 792104477Ssam u_int32_t reg; 793104477Ssam 794104477Ssam /* 795104477Ssam * Set polling in the DMA configuration register to zero. 0x7 avoids 796104477Ssam * resetting the board and zeros out the other fields. 797104477Ssam */ 798104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 799104477Ssam HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 800104477Ssam 801104477Ssam /* 802104477Ssam * Now that polling has been disabled, we have to wait 1 ms 803104477Ssam * before resetting the board. 804104477Ssam */ 805104477Ssam DELAY(1000); 806104477Ssam 807104477Ssam /* Reset the DMA unit */ 808104477Ssam if (full) { 809104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE); 810104477Ssam DELAY(1000); 811104477Ssam } else { 812104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CNFG, 813104477Ssam HIFN_DMACNFG_MODE | HIFN_DMACNFG_MSTRESET); 814104477Ssam hifn_reset_puc(sc); 815104477Ssam } 816104477Ssam 817104477Ssam KASSERT(sc->sc_dma != NULL, ("hifn_reset_board: null DMA tag!")); 818104477Ssam bzero(sc->sc_dma, sizeof(*sc->sc_dma)); 819104477Ssam 820104477Ssam /* Bring dma unit out of reset */ 821104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 822104477Ssam HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 823104477Ssam 824104477Ssam hifn_puc_wait(sc); 825104477Ssam hifn_set_retry(sc); 826104477Ssam 827104477Ssam if (sc->sc_flags & HIFN_IS_7811) { 828104477Ssam for (reg = 0; reg < 1000; reg++) { 829104477Ssam if (READ_REG_1(sc, HIFN_1_7811_MIPSRST) & 830104477Ssam HIFN_MIPSRST_CRAMINIT) 831104477Ssam break; 832104477Ssam DELAY(1000); 833104477Ssam } 834104477Ssam if (reg == 1000) 835104477Ssam printf(": cram init timeout\n"); 836104477Ssam } 837104477Ssam} 838104477Ssam 839104477Ssamstatic u_int32_t 840104477Ssamhifn_next_signature(u_int32_t a, u_int cnt) 841104477Ssam{ 842104477Ssam int i; 843104477Ssam u_int32_t v; 844104477Ssam 845104477Ssam for (i = 0; i < cnt; i++) { 846104477Ssam 847104477Ssam /* get the parity */ 848104477Ssam v = a & 0x80080125; 849104477Ssam v ^= v >> 16; 850104477Ssam v ^= v >> 8; 851104477Ssam v ^= v >> 4; 852104477Ssam v ^= v >> 2; 853104477Ssam v ^= v >> 1; 854104477Ssam 855104477Ssam a = (v & 1) ^ (a << 1); 856104477Ssam } 857104477Ssam 858104477Ssam return a; 859104477Ssam} 860104477Ssam 861104477Ssamstruct pci2id { 862104477Ssam u_short pci_vendor; 863104477Ssam u_short pci_prod; 864104477Ssam char card_id[13]; 865104477Ssam}; 866104477Ssamstatic struct pci2id pci2id[] = { 867104477Ssam { 868104477Ssam PCI_VENDOR_HIFN, 869104477Ssam PCI_PRODUCT_HIFN_7951, 870104477Ssam { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 871104477Ssam 0x00, 0x00, 0x00, 0x00, 0x00 } 872104477Ssam }, { 873104477Ssam PCI_VENDOR_NETSEC, 874104477Ssam PCI_PRODUCT_NETSEC_7751, 875104477Ssam { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 876104477Ssam 0x00, 0x00, 0x00, 0x00, 0x00 } 877104477Ssam }, { 878104477Ssam PCI_VENDOR_INVERTEX, 879104477Ssam PCI_PRODUCT_INVERTEX_AEON, 880104477Ssam { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 881104477Ssam 0x00, 0x00, 0x00, 0x00, 0x00 } 882104477Ssam }, { 883104477Ssam PCI_VENDOR_HIFN, 884104477Ssam PCI_PRODUCT_HIFN_7811, 885104477Ssam { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 886104477Ssam 0x00, 0x00, 0x00, 0x00, 0x00 } 887104477Ssam }, { 888104477Ssam /* 889104477Ssam * Other vendors share this PCI ID as well, such as 890104477Ssam * http://www.powercrypt.com, and obviously they also 891104477Ssam * use the same key. 892104477Ssam */ 893104477Ssam PCI_VENDOR_HIFN, 894104477Ssam PCI_PRODUCT_HIFN_7751, 895104477Ssam { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 896104477Ssam 0x00, 0x00, 0x00, 0x00, 0x00 } 897104477Ssam }, 898104477Ssam}; 899104477Ssam 900104477Ssam/* 901104477Ssam * Checks to see if crypto is already enabled. If crypto isn't enable, 902104477Ssam * "hifn_enable_crypto" is called to enable it. The check is important, 903104477Ssam * as enabling crypto twice will lock the board. 904104477Ssam */ 905104477Ssamstatic int 906104477Ssamhifn_enable_crypto(struct hifn_softc *sc) 907104477Ssam{ 908104477Ssam u_int32_t dmacfg, ramcfg, encl, addr, i; 909104477Ssam char *offtbl = NULL; 910104477Ssam 911104477Ssam for (i = 0; i < sizeof(pci2id)/sizeof(pci2id[0]); i++) { 912104477Ssam if (pci2id[i].pci_vendor == pci_get_vendor(sc->sc_dev) && 913104477Ssam pci2id[i].pci_prod == pci_get_device(sc->sc_dev)) { 914104477Ssam offtbl = pci2id[i].card_id; 915104477Ssam break; 916104477Ssam } 917104477Ssam } 918104477Ssam if (offtbl == NULL) { 919104477Ssam device_printf(sc->sc_dev, "Unknown card!\n"); 920104477Ssam return (1); 921104477Ssam } 922104477Ssam 923104477Ssam ramcfg = READ_REG_0(sc, HIFN_0_PUCNFG); 924104477Ssam dmacfg = READ_REG_1(sc, HIFN_1_DMA_CNFG); 925104477Ssam 926104477Ssam /* 927104477Ssam * The RAM config register's encrypt level bit needs to be set before 928104477Ssam * every read performed on the encryption level register. 929104477Ssam */ 930104477Ssam WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID); 931104477Ssam 932104477Ssam encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA; 933104477Ssam 934104477Ssam /* 935104477Ssam * Make sure we don't re-unlock. Two unlocks kills chip until the 936104477Ssam * next reboot. 937104477Ssam */ 938104477Ssam if (encl == HIFN_PUSTAT_ENA_1 || encl == HIFN_PUSTAT_ENA_2) { 939104477Ssam#ifdef HIFN_DEBUG 940104477Ssam if (hifn_debug) 941104477Ssam device_printf(sc->sc_dev, 942104477Ssam "Strong crypto already enabled!\n"); 943104477Ssam#endif 944104477Ssam goto report; 945104477Ssam } 946104477Ssam 947104477Ssam if (encl != 0 && encl != HIFN_PUSTAT_ENA_0) { 948104477Ssam#ifdef HIFN_DEBUG 949104477Ssam if (hifn_debug) 950104477Ssam device_printf(sc->sc_dev, 951104477Ssam "Unknown encryption level 0x%x\n", encl); 952104477Ssam#endif 953104477Ssam return 1; 954104477Ssam } 955104477Ssam 956104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_UNLOCK | 957104477Ssam HIFN_DMACNFG_MSTRESET | HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 958104477Ssam DELAY(1000); 959104477Ssam addr = READ_REG_1(sc, HIFN_UNLOCK_SECRET1); 960104477Ssam DELAY(1000); 961104477Ssam WRITE_REG_1(sc, HIFN_UNLOCK_SECRET2, 0); 962104477Ssam DELAY(1000); 963104477Ssam 964104477Ssam for (i = 0; i <= 12; i++) { 965104477Ssam addr = hifn_next_signature(addr, offtbl[i] + 0x101); 966104477Ssam WRITE_REG_1(sc, HIFN_UNLOCK_SECRET2, addr); 967104477Ssam 968104477Ssam DELAY(1000); 969104477Ssam } 970104477Ssam 971104477Ssam WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID); 972104477Ssam encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA; 973104477Ssam 974104477Ssam#ifdef HIFN_DEBUG 975104477Ssam if (hifn_debug) { 976104477Ssam if (encl != HIFN_PUSTAT_ENA_1 && encl != HIFN_PUSTAT_ENA_2) 977104477Ssam device_printf(sc->sc_dev, "Engine is permanently " 978104477Ssam "locked until next system reset!\n"); 979104477Ssam else 980104477Ssam device_printf(sc->sc_dev, "Engine enabled " 981104477Ssam "successfully!\n"); 982104477Ssam } 983104477Ssam#endif 984104477Ssam 985104477Ssamreport: 986104477Ssam WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg); 987104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CNFG, dmacfg); 988104477Ssam 989104477Ssam switch (encl) { 990104477Ssam case HIFN_PUSTAT_ENA_1: 991104477Ssam case HIFN_PUSTAT_ENA_2: 992104477Ssam break; 993104477Ssam case HIFN_PUSTAT_ENA_0: 994104477Ssam default: 995104477Ssam device_printf(sc->sc_dev, "disabled"); 996104477Ssam break; 997104477Ssam } 998104477Ssam 999104477Ssam return 0; 1000104477Ssam} 1001104477Ssam 1002104477Ssam/* 1003104477Ssam * Give initial values to the registers listed in the "Register Space" 1004104477Ssam * section of the HIFN Software Development reference manual. 1005104477Ssam */ 1006104477Ssamstatic void 1007104477Ssamhifn_init_pci_registers(struct hifn_softc *sc) 1008104477Ssam{ 1009104477Ssam /* write fixed values needed by the Initialization registers */ 1010104477Ssam WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); 1011104477Ssam WRITE_REG_0(sc, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD); 1012104477Ssam WRITE_REG_0(sc, HIFN_0_PUIER, HIFN_PUIER_DSTOVER); 1013104477Ssam 1014104477Ssam /* write all 4 ring address registers */ 1015104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CRAR, sc->sc_dma_physaddr + 1016104477Ssam offsetof(struct hifn_dma, cmdr[0])); 1017104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_SRAR, sc->sc_dma_physaddr + 1018104477Ssam offsetof(struct hifn_dma, srcr[0])); 1019104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_DRAR, sc->sc_dma_physaddr + 1020104477Ssam offsetof(struct hifn_dma, dstr[0])); 1021104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_RRAR, sc->sc_dma_physaddr + 1022104477Ssam offsetof(struct hifn_dma, resr[0])); 1023104477Ssam 1024104477Ssam DELAY(2000); 1025104477Ssam 1026104477Ssam /* write status register */ 1027104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1028104477Ssam HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS | 1029104477Ssam HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS | 1030104477Ssam HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST | 1031104477Ssam HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER | 1032104477Ssam HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST | 1033104477Ssam HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER | 1034104477Ssam HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST | 1035104477Ssam HIFN_DMACSR_S_WAIT | 1036104477Ssam HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST | 1037104477Ssam HIFN_DMACSR_C_WAIT | 1038104477Ssam HIFN_DMACSR_ENGINE | 1039104477Ssam ((sc->sc_flags & HIFN_HAS_PUBLIC) ? 1040104477Ssam HIFN_DMACSR_PUBDONE : 0) | 1041104477Ssam ((sc->sc_flags & HIFN_IS_7811) ? 1042104477Ssam HIFN_DMACSR_ILLW | HIFN_DMACSR_ILLR : 0)); 1043104477Ssam 1044104477Ssam sc->sc_d_busy = sc->sc_r_busy = sc->sc_s_busy = sc->sc_c_busy = 0; 1045104477Ssam sc->sc_dmaier |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT | 1046104477Ssam HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER | 1047104477Ssam HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT | 1048104477Ssam ((sc->sc_flags & HIFN_IS_7811) ? 1049104477Ssam HIFN_DMAIER_ILLW | HIFN_DMAIER_ILLR : 0); 1050104477Ssam sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT; 1051104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 1052104477Ssam 1053104477Ssam WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING | 1054104477Ssam HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES | 1055104477Ssam HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 | 1056104477Ssam (sc->sc_drammodel ? HIFN_PUCNFG_DRAM : HIFN_PUCNFG_SRAM)); 1057104477Ssam 1058104477Ssam WRITE_REG_0(sc, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); 1059104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 1060104477Ssam HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST | 1061104477Ssam ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) | 1062104477Ssam ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL)); 1063104477Ssam} 1064104477Ssam 1065104477Ssam/* 1066104477Ssam * The maximum number of sessions supported by the card 1067104477Ssam * is dependent on the amount of context ram, which 1068104477Ssam * encryption algorithms are enabled, and how compression 1069104477Ssam * is configured. This should be configured before this 1070104477Ssam * routine is called. 1071104477Ssam */ 1072104477Ssamstatic void 1073104477Ssamhifn_sessions(struct hifn_softc *sc) 1074104477Ssam{ 1075104477Ssam u_int32_t pucnfg; 1076104477Ssam int ctxsize; 1077104477Ssam 1078104477Ssam pucnfg = READ_REG_0(sc, HIFN_0_PUCNFG); 1079104477Ssam 1080104477Ssam if (pucnfg & HIFN_PUCNFG_COMPSING) { 1081104477Ssam if (pucnfg & HIFN_PUCNFG_ENCCNFG) 1082104477Ssam ctxsize = 128; 1083104477Ssam else 1084104477Ssam ctxsize = 512; 1085104477Ssam sc->sc_maxses = 1 + 1086104477Ssam ((sc->sc_ramsize - 32768) / ctxsize); 1087104477Ssam } else 1088104477Ssam sc->sc_maxses = sc->sc_ramsize / 16384; 1089104477Ssam 1090104477Ssam if (sc->sc_maxses > 2048) 1091104477Ssam sc->sc_maxses = 2048; 1092104477Ssam} 1093104477Ssam 1094104477Ssam/* 1095104477Ssam * Determine ram type (sram or dram). Board should be just out of a reset 1096104477Ssam * state when this is called. 1097104477Ssam */ 1098104477Ssamstatic int 1099104477Ssamhifn_ramtype(struct hifn_softc *sc) 1100104477Ssam{ 1101104477Ssam u_int8_t data[8], dataexpect[8]; 1102104477Ssam int i; 1103104477Ssam 1104104477Ssam for (i = 0; i < sizeof(data); i++) 1105104477Ssam data[i] = dataexpect[i] = 0x55; 1106104477Ssam if (hifn_writeramaddr(sc, 0, data)) 1107104477Ssam return (-1); 1108104477Ssam if (hifn_readramaddr(sc, 0, data)) 1109104477Ssam return (-1); 1110104477Ssam if (bcmp(data, dataexpect, sizeof(data)) != 0) { 1111104477Ssam sc->sc_drammodel = 1; 1112104477Ssam return (0); 1113104477Ssam } 1114104477Ssam 1115104477Ssam for (i = 0; i < sizeof(data); i++) 1116104477Ssam data[i] = dataexpect[i] = 0xaa; 1117104477Ssam if (hifn_writeramaddr(sc, 0, data)) 1118104477Ssam return (-1); 1119104477Ssam if (hifn_readramaddr(sc, 0, data)) 1120104477Ssam return (-1); 1121104477Ssam if (bcmp(data, dataexpect, sizeof(data)) != 0) { 1122104477Ssam sc->sc_drammodel = 1; 1123104477Ssam return (0); 1124104477Ssam } 1125104477Ssam 1126104477Ssam return (0); 1127104477Ssam} 1128104477Ssam 1129104477Ssam#define HIFN_SRAM_MAX (32 << 20) 1130104477Ssam#define HIFN_SRAM_STEP_SIZE 16384 1131104477Ssam#define HIFN_SRAM_GRANULARITY (HIFN_SRAM_MAX / HIFN_SRAM_STEP_SIZE) 1132104477Ssam 1133104477Ssamstatic int 1134104477Ssamhifn_sramsize(struct hifn_softc *sc) 1135104477Ssam{ 1136104477Ssam u_int32_t a; 1137104477Ssam u_int8_t data[8]; 1138104477Ssam u_int8_t dataexpect[sizeof(data)]; 1139104477Ssam int32_t i; 1140104477Ssam 1141104477Ssam for (i = 0; i < sizeof(data); i++) 1142104477Ssam data[i] = dataexpect[i] = i ^ 0x5a; 1143104477Ssam 1144104477Ssam for (i = HIFN_SRAM_GRANULARITY - 1; i >= 0; i--) { 1145104477Ssam a = i * HIFN_SRAM_STEP_SIZE; 1146104477Ssam bcopy(&i, data, sizeof(i)); 1147104477Ssam hifn_writeramaddr(sc, a, data); 1148104477Ssam } 1149104477Ssam 1150104477Ssam for (i = 0; i < HIFN_SRAM_GRANULARITY; i++) { 1151104477Ssam a = i * HIFN_SRAM_STEP_SIZE; 1152104477Ssam bcopy(&i, dataexpect, sizeof(i)); 1153104477Ssam if (hifn_readramaddr(sc, a, data) < 0) 1154104477Ssam return (0); 1155104477Ssam if (bcmp(data, dataexpect, sizeof(data)) != 0) 1156104477Ssam return (0); 1157104477Ssam sc->sc_ramsize = a + HIFN_SRAM_STEP_SIZE; 1158104477Ssam } 1159104477Ssam 1160104477Ssam return (0); 1161104477Ssam} 1162104477Ssam 1163104477Ssam/* 1164104477Ssam * XXX For dram boards, one should really try all of the 1165104477Ssam * HIFN_PUCNFG_DSZ_*'s. This just assumes that PUCNFG 1166104477Ssam * is already set up correctly. 1167104477Ssam */ 1168104477Ssamstatic int 1169104477Ssamhifn_dramsize(struct hifn_softc *sc) 1170104477Ssam{ 1171104477Ssam u_int32_t cnfg; 1172104477Ssam 1173104477Ssam cnfg = READ_REG_0(sc, HIFN_0_PUCNFG) & 1174104477Ssam HIFN_PUCNFG_DRAMMASK; 1175104477Ssam sc->sc_ramsize = 1 << ((cnfg >> 13) + 18); 1176104477Ssam return (0); 1177104477Ssam} 1178104477Ssam 1179104477Ssamstatic void 1180104477Ssamhifn_alloc_slot(struct hifn_softc *sc, int *cmdp, int *srcp, int *dstp, int *resp) 1181104477Ssam{ 1182104477Ssam struct hifn_dma *dma = sc->sc_dma; 1183104477Ssam 1184104477Ssam if (dma->cmdi == HIFN_D_CMD_RSIZE) { 1185104477Ssam dma->cmdi = 0; 1186104477Ssam dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID | 1187104477Ssam HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1188104477Ssam HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE, 1189104477Ssam BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1190104477Ssam } 1191104477Ssam *cmdp = dma->cmdi++; 1192104477Ssam dma->cmdk = dma->cmdi; 1193104477Ssam 1194104477Ssam if (dma->srci == HIFN_D_SRC_RSIZE) { 1195104477Ssam dma->srci = 0; 1196104477Ssam dma->srcr[HIFN_D_SRC_RSIZE].l = htole32(HIFN_D_VALID | 1197104477Ssam HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1198104477Ssam HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE, 1199104477Ssam BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1200104477Ssam } 1201104477Ssam *srcp = dma->srci++; 1202104477Ssam dma->srck = dma->srci; 1203104477Ssam 1204104477Ssam if (dma->dsti == HIFN_D_DST_RSIZE) { 1205104477Ssam dma->dsti = 0; 1206104477Ssam dma->dstr[HIFN_D_DST_RSIZE].l = htole32(HIFN_D_VALID | 1207104477Ssam HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1208104477Ssam HIFN_DSTR_SYNC(sc, HIFN_D_DST_RSIZE, 1209104477Ssam BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1210104477Ssam } 1211104477Ssam *dstp = dma->dsti++; 1212104477Ssam dma->dstk = dma->dsti; 1213104477Ssam 1214104477Ssam if (dma->resi == HIFN_D_RES_RSIZE) { 1215104477Ssam dma->resi = 0; 1216104477Ssam dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID | 1217104477Ssam HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1218104477Ssam HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE, 1219104477Ssam BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1220104477Ssam } 1221104477Ssam *resp = dma->resi++; 1222104477Ssam dma->resk = dma->resi; 1223104477Ssam} 1224104477Ssam 1225104477Ssamstatic int 1226104477Ssamhifn_writeramaddr(struct hifn_softc *sc, int addr, u_int8_t *data) 1227104477Ssam{ 1228104477Ssam struct hifn_dma *dma = sc->sc_dma; 1229104477Ssam hifn_base_command_t wc; 1230104477Ssam const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ; 1231104477Ssam int r, cmdi, resi, srci, dsti; 1232104477Ssam 1233104477Ssam wc.masks = htole16(3 << 13); 1234104477Ssam wc.session_num = htole16(addr >> 14); 1235104477Ssam wc.total_source_count = htole16(8); 1236104477Ssam wc.total_dest_count = htole16(addr & 0x3fff); 1237104477Ssam 1238104477Ssam hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi); 1239104477Ssam 1240104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1241104477Ssam HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA | 1242104477Ssam HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA); 1243104477Ssam 1244104477Ssam /* build write command */ 1245104477Ssam bzero(dma->command_bufs[cmdi], HIFN_MAX_COMMAND); 1246104477Ssam *(hifn_base_command_t *)dma->command_bufs[cmdi] = wc; 1247104477Ssam bcopy(data, &dma->test_src, sizeof(dma->test_src)); 1248104477Ssam 1249104477Ssam dma->srcr[srci].p = htole32(sc->sc_dma_physaddr 1250104477Ssam + offsetof(struct hifn_dma, test_src)); 1251104477Ssam dma->dstr[dsti].p = htole32(sc->sc_dma_physaddr 1252104477Ssam + offsetof(struct hifn_dma, test_dst)); 1253104477Ssam 1254104477Ssam dma->cmdr[cmdi].l = htole32(16 | masks); 1255104477Ssam dma->srcr[srci].l = htole32(8 | masks); 1256104477Ssam dma->dstr[dsti].l = htole32(4 | masks); 1257104477Ssam dma->resr[resi].l = htole32(4 | masks); 1258104477Ssam 1259104477Ssam bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1260104477Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1261104477Ssam 1262104477Ssam for (r = 10000; r >= 0; r--) { 1263104477Ssam DELAY(10); 1264104477Ssam bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1265104477Ssam BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1266104477Ssam if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0) 1267104477Ssam break; 1268104477Ssam bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1269104477Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1270104477Ssam } 1271104477Ssam if (r == 0) { 1272104477Ssam device_printf(sc->sc_dev, "writeramaddr -- " 1273104477Ssam "result[%d](addr %d) still valid\n", resi, addr); 1274104477Ssam r = -1; 1275104477Ssam return (-1); 1276104477Ssam } else 1277104477Ssam r = 0; 1278104477Ssam 1279104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1280104477Ssam HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS | 1281104477Ssam HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS); 1282104477Ssam 1283104477Ssam return (r); 1284104477Ssam} 1285104477Ssam 1286104477Ssamstatic int 1287104477Ssamhifn_readramaddr(struct hifn_softc *sc, int addr, u_int8_t *data) 1288104477Ssam{ 1289104477Ssam struct hifn_dma *dma = sc->sc_dma; 1290104477Ssam hifn_base_command_t rc; 1291104477Ssam const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ; 1292104477Ssam int r, cmdi, srci, dsti, resi; 1293104477Ssam 1294104477Ssam rc.masks = htole16(2 << 13); 1295104477Ssam rc.session_num = htole16(addr >> 14); 1296104477Ssam rc.total_source_count = htole16(addr & 0x3fff); 1297104477Ssam rc.total_dest_count = htole16(8); 1298104477Ssam 1299104477Ssam hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi); 1300104477Ssam 1301104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1302104477Ssam HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA | 1303104477Ssam HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA); 1304104477Ssam 1305104477Ssam bzero(dma->command_bufs[cmdi], HIFN_MAX_COMMAND); 1306104477Ssam *(hifn_base_command_t *)dma->command_bufs[cmdi] = rc; 1307104477Ssam 1308104477Ssam dma->srcr[srci].p = htole32(sc->sc_dma_physaddr + 1309104477Ssam offsetof(struct hifn_dma, test_src)); 1310104477Ssam dma->test_src = 0; 1311104477Ssam dma->dstr[dsti].p = htole32(sc->sc_dma_physaddr + 1312104477Ssam offsetof(struct hifn_dma, test_dst)); 1313104477Ssam dma->test_dst = 0; 1314104477Ssam dma->cmdr[cmdi].l = htole32(8 | masks); 1315104477Ssam dma->srcr[srci].l = htole32(8 | masks); 1316104477Ssam dma->dstr[dsti].l = htole32(8 | masks); 1317104477Ssam dma->resr[resi].l = htole32(HIFN_MAX_RESULT | masks); 1318104477Ssam 1319104477Ssam bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1320104477Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1321104477Ssam 1322104477Ssam for (r = 10000; r >= 0; r--) { 1323104477Ssam DELAY(10); 1324104477Ssam bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1325104477Ssam BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1326104477Ssam if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0) 1327104477Ssam break; 1328104477Ssam bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1329104477Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1330104477Ssam } 1331104477Ssam if (r == 0) { 1332104477Ssam device_printf(sc->sc_dev, "readramaddr -- " 1333104477Ssam "result[%d](addr %d) still valid\n", resi, addr); 1334104477Ssam r = -1; 1335104477Ssam } else { 1336104477Ssam r = 0; 1337104477Ssam bcopy(&dma->test_dst, data, sizeof(dma->test_dst)); 1338104477Ssam } 1339104477Ssam 1340104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1341104477Ssam HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS | 1342104477Ssam HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS); 1343104477Ssam 1344104477Ssam return (r); 1345104477Ssam} 1346104477Ssam 1347104477Ssam/* 1348104477Ssam * Initialize the descriptor rings. 1349104477Ssam */ 1350104477Ssamstatic void 1351104477Ssamhifn_init_dma(struct hifn_softc *sc) 1352104477Ssam{ 1353104477Ssam struct hifn_dma *dma = sc->sc_dma; 1354104477Ssam int i; 1355104477Ssam 1356104477Ssam hifn_set_retry(sc); 1357104477Ssam 1358104477Ssam /* initialize static pointer values */ 1359104477Ssam for (i = 0; i < HIFN_D_CMD_RSIZE; i++) 1360104477Ssam dma->cmdr[i].p = htole32(sc->sc_dma_physaddr + 1361104477Ssam offsetof(struct hifn_dma, command_bufs[i][0])); 1362104477Ssam for (i = 0; i < HIFN_D_RES_RSIZE; i++) 1363104477Ssam dma->resr[i].p = htole32(sc->sc_dma_physaddr + 1364104477Ssam offsetof(struct hifn_dma, result_bufs[i][0])); 1365104477Ssam 1366104477Ssam dma->cmdr[HIFN_D_CMD_RSIZE].p = 1367104477Ssam htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, cmdr[0])); 1368104477Ssam dma->srcr[HIFN_D_SRC_RSIZE].p = 1369104477Ssam htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, srcr[0])); 1370104477Ssam dma->dstr[HIFN_D_DST_RSIZE].p = 1371104477Ssam htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, dstr[0])); 1372104477Ssam dma->resr[HIFN_D_RES_RSIZE].p = 1373104477Ssam htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, resr[0])); 1374104477Ssam 1375104477Ssam dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0; 1376104477Ssam dma->cmdi = dma->srci = dma->dsti = dma->resi = 0; 1377104477Ssam dma->cmdk = dma->srck = dma->dstk = dma->resk = 0; 1378104477Ssam} 1379104477Ssam 1380104477Ssam/* 1381104477Ssam * Writes out the raw command buffer space. Returns the 1382104477Ssam * command buffer size. 1383104477Ssam */ 1384104477Ssamstatic u_int 1385104477Ssamhifn_write_command(struct hifn_command *cmd, u_int8_t *buf) 1386104477Ssam{ 1387104477Ssam u_int8_t *buf_pos; 1388104477Ssam hifn_base_command_t *base_cmd; 1389104477Ssam hifn_mac_command_t *mac_cmd; 1390104477Ssam hifn_crypt_command_t *cry_cmd; 1391104477Ssam int using_mac, using_crypt, len; 1392104477Ssam u_int32_t dlen, slen; 1393104477Ssam 1394104477Ssam buf_pos = buf; 1395104477Ssam using_mac = cmd->base_masks & HIFN_BASE_CMD_MAC; 1396104477Ssam using_crypt = cmd->base_masks & HIFN_BASE_CMD_CRYPT; 1397104477Ssam 1398104477Ssam base_cmd = (hifn_base_command_t *)buf_pos; 1399104477Ssam base_cmd->masks = htole16(cmd->base_masks); 1400104477Ssam slen = cmd->src_mapsize; 1401104477Ssam if (cmd->sloplen) 1402104477Ssam dlen = cmd->dst_mapsize - cmd->sloplen + sizeof(u_int32_t); 1403104477Ssam else 1404104477Ssam dlen = cmd->dst_mapsize; 1405104477Ssam base_cmd->total_source_count = htole16(slen & HIFN_BASE_CMD_LENMASK_LO); 1406104477Ssam base_cmd->total_dest_count = htole16(dlen & HIFN_BASE_CMD_LENMASK_LO); 1407104477Ssam dlen >>= 16; 1408104477Ssam slen >>= 16; 1409104477Ssam base_cmd->session_num = htole16(cmd->session_num | 1410104477Ssam ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) | 1411104477Ssam ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M)); 1412104477Ssam buf_pos += sizeof(hifn_base_command_t); 1413104477Ssam 1414104477Ssam if (using_mac) { 1415104477Ssam mac_cmd = (hifn_mac_command_t *)buf_pos; 1416104477Ssam dlen = cmd->maccrd->crd_len; 1417104477Ssam mac_cmd->source_count = htole16(dlen & 0xffff); 1418104477Ssam dlen >>= 16; 1419104477Ssam mac_cmd->masks = htole16(cmd->mac_masks | 1420104477Ssam ((dlen << HIFN_MAC_CMD_SRCLEN_S) & HIFN_MAC_CMD_SRCLEN_M)); 1421104477Ssam mac_cmd->header_skip = htole16(cmd->maccrd->crd_skip); 1422104477Ssam mac_cmd->reserved = 0; 1423104477Ssam buf_pos += sizeof(hifn_mac_command_t); 1424104477Ssam } 1425104477Ssam 1426104477Ssam if (using_crypt) { 1427104477Ssam cry_cmd = (hifn_crypt_command_t *)buf_pos; 1428104477Ssam dlen = cmd->enccrd->crd_len; 1429104477Ssam cry_cmd->source_count = htole16(dlen & 0xffff); 1430104477Ssam dlen >>= 16; 1431104477Ssam cry_cmd->masks = htole16(cmd->cry_masks | 1432104477Ssam ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & HIFN_CRYPT_CMD_SRCLEN_M)); 1433104477Ssam cry_cmd->header_skip = htole16(cmd->enccrd->crd_skip); 1434104477Ssam cry_cmd->reserved = 0; 1435104477Ssam buf_pos += sizeof(hifn_crypt_command_t); 1436104477Ssam } 1437104477Ssam 1438104477Ssam if (using_mac && cmd->mac_masks & HIFN_MAC_CMD_NEW_KEY) { 1439104477Ssam bcopy(cmd->mac, buf_pos, HIFN_MAC_KEY_LENGTH); 1440104477Ssam buf_pos += HIFN_MAC_KEY_LENGTH; 1441104477Ssam } 1442104477Ssam 1443104477Ssam if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_KEY) { 1444104477Ssam switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) { 1445104477Ssam case HIFN_CRYPT_CMD_ALG_3DES: 1446104477Ssam bcopy(cmd->ck, buf_pos, HIFN_3DES_KEY_LENGTH); 1447104477Ssam buf_pos += HIFN_3DES_KEY_LENGTH; 1448104477Ssam break; 1449104477Ssam case HIFN_CRYPT_CMD_ALG_DES: 1450104477Ssam bcopy(cmd->ck, buf_pos, HIFN_DES_KEY_LENGTH); 1451104477Ssam buf_pos += cmd->cklen; 1452104477Ssam break; 1453104477Ssam case HIFN_CRYPT_CMD_ALG_RC4: 1454104477Ssam len = 256; 1455104477Ssam do { 1456104477Ssam int clen; 1457104477Ssam 1458104477Ssam clen = MIN(cmd->cklen, len); 1459104477Ssam bcopy(cmd->ck, buf_pos, clen); 1460104477Ssam len -= clen; 1461104477Ssam buf_pos += clen; 1462104477Ssam } while (len > 0); 1463104477Ssam bzero(buf_pos, 4); 1464104477Ssam buf_pos += 4; 1465104477Ssam break; 1466104477Ssam } 1467104477Ssam } 1468104477Ssam 1469104477Ssam if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_IV) { 1470104477Ssam bcopy(cmd->iv, buf_pos, HIFN_IV_LENGTH); 1471104477Ssam buf_pos += HIFN_IV_LENGTH; 1472104477Ssam } 1473104477Ssam 1474104477Ssam if ((cmd->base_masks & (HIFN_BASE_CMD_MAC|HIFN_BASE_CMD_CRYPT)) == 0) { 1475104477Ssam bzero(buf_pos, 8); 1476104477Ssam buf_pos += 8; 1477104477Ssam } 1478104477Ssam 1479104477Ssam return (buf_pos - buf); 1480104477Ssam} 1481104477Ssam 1482104477Ssamstatic int 1483104477Ssamhifn_dmamap_aligned(struct hifn_operand *op) 1484104477Ssam{ 1485104477Ssam int i; 1486104477Ssam 1487104477Ssam for (i = 0; i < op->nsegs; i++) { 1488104477Ssam if (op->segs[i].ds_addr & 3) 1489104477Ssam return (0); 1490104477Ssam if ((i != (op->nsegs - 1)) && (op->segs[i].ds_len & 3)) 1491104477Ssam return (0); 1492104477Ssam } 1493104477Ssam return (1); 1494104477Ssam} 1495104477Ssam 1496104477Ssamstatic int 1497104477Ssamhifn_dmamap_load_dst(struct hifn_softc *sc, struct hifn_command *cmd) 1498104477Ssam{ 1499104477Ssam struct hifn_dma *dma = sc->sc_dma; 1500104477Ssam struct hifn_operand *dst = &cmd->dst; 1501104477Ssam u_int32_t p, l; 1502104477Ssam int idx, used = 0, i; 1503104477Ssam 1504104477Ssam idx = dma->dsti; 1505104477Ssam for (i = 0; i < dst->nsegs - 1; i++) { 1506104477Ssam dma->dstr[idx].p = htole32(dst->segs[i].ds_addr); 1507104477Ssam dma->dstr[idx].l = htole32(HIFN_D_VALID | 1508104477Ssam HIFN_D_MASKDONEIRQ | dst->segs[i].ds_len); 1509104477Ssam HIFN_DSTR_SYNC(sc, idx, 1510104477Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1511104477Ssam used++; 1512104477Ssam 1513104477Ssam if (++idx == HIFN_D_DST_RSIZE) { 1514104477Ssam dma->dstr[idx].l = htole32(HIFN_D_VALID | 1515104477Ssam HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1516104477Ssam HIFN_DSTR_SYNC(sc, idx, 1517104477Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1518104477Ssam idx = 0; 1519104477Ssam } 1520104477Ssam } 1521104477Ssam 1522104477Ssam if (cmd->sloplen == 0) { 1523104477Ssam p = dst->segs[i].ds_addr; 1524104477Ssam l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST | 1525104477Ssam dst->segs[i].ds_len; 1526104477Ssam } else { 1527104477Ssam p = sc->sc_dma_physaddr + 1528104477Ssam offsetof(struct hifn_dma, slop[cmd->slopidx]); 1529104477Ssam l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST | 1530104477Ssam sizeof(u_int32_t); 1531104477Ssam 1532104477Ssam if ((dst->segs[i].ds_len - cmd->sloplen) != 0) { 1533104477Ssam dma->dstr[idx].p = htole32(dst->segs[i].ds_addr); 1534104477Ssam dma->dstr[idx].l = htole32(HIFN_D_VALID | 1535104477Ssam HIFN_D_MASKDONEIRQ | 1536104477Ssam (dst->segs[i].ds_len - cmd->sloplen)); 1537104477Ssam HIFN_DSTR_SYNC(sc, idx, 1538104477Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1539104477Ssam used++; 1540104477Ssam 1541104477Ssam if (++idx == HIFN_D_DST_RSIZE) { 1542104477Ssam dma->dstr[idx].l = htole32(HIFN_D_VALID | 1543104477Ssam HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1544104477Ssam HIFN_DSTR_SYNC(sc, idx, 1545104477Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1546104477Ssam idx = 0; 1547104477Ssam } 1548104477Ssam } 1549104477Ssam } 1550104477Ssam dma->dstr[idx].p = htole32(p); 1551104477Ssam dma->dstr[idx].l = htole32(l); 1552104477Ssam HIFN_DSTR_SYNC(sc, idx, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1553104477Ssam used++; 1554104477Ssam 1555104477Ssam if (++idx == HIFN_D_DST_RSIZE) { 1556104477Ssam dma->dstr[idx].l = htole32(HIFN_D_VALID | HIFN_D_JUMP | 1557104477Ssam HIFN_D_MASKDONEIRQ); 1558104477Ssam HIFN_DSTR_SYNC(sc, idx, 1559104477Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1560104477Ssam idx = 0; 1561104477Ssam } 1562104477Ssam 1563104477Ssam dma->dsti = idx; 1564104477Ssam dma->dstu += used; 1565104477Ssam return (idx); 1566104477Ssam} 1567104477Ssam 1568104477Ssamstatic int 1569104477Ssamhifn_dmamap_load_src(struct hifn_softc *sc, struct hifn_command *cmd) 1570104477Ssam{ 1571104477Ssam struct hifn_dma *dma = sc->sc_dma; 1572104477Ssam struct hifn_operand *src = &cmd->src; 1573104477Ssam int idx, i; 1574104477Ssam u_int32_t last = 0; 1575104477Ssam 1576104477Ssam idx = dma->srci; 1577104477Ssam for (i = 0; i < src->nsegs; i++) { 1578104477Ssam if (i == src->nsegs - 1) 1579104477Ssam last = HIFN_D_LAST; 1580104477Ssam 1581104477Ssam dma->srcr[idx].p = htole32(src->segs[i].ds_addr); 1582104477Ssam dma->srcr[idx].l = htole32(src->segs[i].ds_len | 1583104477Ssam HIFN_D_VALID | HIFN_D_MASKDONEIRQ | last); 1584104477Ssam HIFN_SRCR_SYNC(sc, idx, 1585104477Ssam BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1586104477Ssam 1587104477Ssam if (++idx == HIFN_D_SRC_RSIZE) { 1588104477Ssam dma->srcr[idx].l = htole32(HIFN_D_VALID | 1589104477Ssam HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1590104477Ssam HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE, 1591104477Ssam BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1592104477Ssam idx = 0; 1593104477Ssam } 1594104477Ssam } 1595104477Ssam dma->srci = idx; 1596104477Ssam dma->srcu += src->nsegs; 1597104477Ssam return (idx); 1598104477Ssam} 1599104477Ssam 1600104477Ssamstatic void 1601104477Ssamhifn_op_cb(void* arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error) 1602104477Ssam{ 1603104477Ssam struct hifn_operand *op = arg; 1604104477Ssam 1605104477Ssam KASSERT(nsegs <= MAX_SCATTER, 1606104477Ssam ("hifn_op_cb: too many DMA segments (%u > %u) " 1607104477Ssam "returned when mapping operand", nsegs, MAX_SCATTER)); 1608104477Ssam op->mapsize = mapsize; 1609104477Ssam op->nsegs = nsegs; 1610104477Ssam bcopy(seg, op->segs, nsegs * sizeof (seg[0])); 1611104477Ssam} 1612104477Ssam 1613104477Ssamstatic int 1614104477Ssamhifn_crypto( 1615104477Ssam struct hifn_softc *sc, 1616104477Ssam struct hifn_command *cmd, 1617104477Ssam struct cryptop *crp, 1618104477Ssam int hint) 1619104477Ssam{ 1620104477Ssam struct hifn_dma *dma = sc->sc_dma; 1621104477Ssam u_int32_t cmdlen; 1622104477Ssam int cmdi, resi, err = 0; 1623104477Ssam 1624104477Ssam /* 1625104477Ssam * need 1 cmd, and 1 res 1626104477Ssam * 1627104477Ssam * NB: check this first since it's easy. 1628104477Ssam */ 1629115748Ssam HIFN_LOCK(sc); 1630104477Ssam if ((dma->cmdu + 1) > HIFN_D_CMD_RSIZE || 1631104477Ssam (dma->resu + 1) > HIFN_D_RES_RSIZE) { 1632104477Ssam#ifdef HIFN_DEBUG 1633104477Ssam if (hifn_debug) { 1634104477Ssam device_printf(sc->sc_dev, 1635104477Ssam "cmd/result exhaustion, cmdu %u resu %u\n", 1636104477Ssam dma->cmdu, dma->resu); 1637104477Ssam } 1638104477Ssam#endif 1639104477Ssam hifnstats.hst_nomem_cr++; 1640115748Ssam HIFN_UNLOCK(sc); 1641104477Ssam return (ERESTART); 1642104477Ssam } 1643104477Ssam 1644104477Ssam if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->src_map)) { 1645104477Ssam hifnstats.hst_nomem_map++; 1646115748Ssam HIFN_UNLOCK(sc); 1647104477Ssam return (ENOMEM); 1648104477Ssam } 1649104477Ssam 1650104477Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) { 1651104477Ssam if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map, 1652104477Ssam cmd->src_m, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) { 1653104477Ssam hifnstats.hst_nomem_load++; 1654104477Ssam err = ENOMEM; 1655104477Ssam goto err_srcmap1; 1656104477Ssam } 1657104477Ssam } else if (crp->crp_flags & CRYPTO_F_IOV) { 1658104477Ssam if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map, 1659104477Ssam cmd->src_io, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) { 1660104477Ssam hifnstats.hst_nomem_load++; 1661104477Ssam err = ENOMEM; 1662104477Ssam goto err_srcmap1; 1663104477Ssam } 1664104477Ssam } else { 1665104477Ssam err = EINVAL; 1666104477Ssam goto err_srcmap1; 1667104477Ssam } 1668104477Ssam 1669104477Ssam if (hifn_dmamap_aligned(&cmd->src)) { 1670104477Ssam cmd->sloplen = cmd->src_mapsize & 3; 1671104477Ssam cmd->dst = cmd->src; 1672104477Ssam } else { 1673104477Ssam if (crp->crp_flags & CRYPTO_F_IOV) { 1674104477Ssam err = EINVAL; 1675104477Ssam goto err_srcmap; 1676104477Ssam } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1677104477Ssam int totlen, len; 1678104477Ssam struct mbuf *m, *m0, *mlast; 1679104477Ssam 1680104477Ssam KASSERT(cmd->dst_m == cmd->src_m, 1681104477Ssam ("hifn_crypto: dst_m initialized improperly")); 1682104477Ssam hifnstats.hst_unaligned++; 1683104477Ssam /* 1684104477Ssam * Source is not aligned on a longword boundary. 1685104477Ssam * Copy the data to insure alignment. If we fail 1686104477Ssam * to allocate mbufs or clusters while doing this 1687104477Ssam * we return ERESTART so the operation is requeued 1688104477Ssam * at the crypto later, but only if there are 1689104477Ssam * ops already posted to the hardware; otherwise we 1690104477Ssam * have no guarantee that we'll be re-entered. 1691104477Ssam */ 1692104477Ssam totlen = cmd->src_mapsize; 1693104477Ssam if (cmd->src_m->m_flags & M_PKTHDR) { 1694104477Ssam len = MHLEN; 1695111119Simp MGETHDR(m0, M_DONTWAIT, MT_DATA); 1696111119Simp if (m0 && !m_dup_pkthdr(m0, cmd->src_m, M_DONTWAIT)) { 1697108466Ssam m_free(m0); 1698108466Ssam m0 = NULL; 1699108466Ssam } 1700104477Ssam } else { 1701104477Ssam len = MLEN; 1702111119Simp MGET(m0, M_DONTWAIT, MT_DATA); 1703104477Ssam } 1704104477Ssam if (m0 == NULL) { 1705104477Ssam hifnstats.hst_nomem_mbuf++; 1706104477Ssam err = dma->cmdu ? ERESTART : ENOMEM; 1707104477Ssam goto err_srcmap; 1708104477Ssam } 1709104477Ssam if (totlen >= MINCLSIZE) { 1710111119Simp MCLGET(m0, M_DONTWAIT); 1711104477Ssam if ((m0->m_flags & M_EXT) == 0) { 1712104477Ssam hifnstats.hst_nomem_mcl++; 1713104477Ssam err = dma->cmdu ? ERESTART : ENOMEM; 1714104477Ssam m_freem(m0); 1715104477Ssam goto err_srcmap; 1716104477Ssam } 1717104477Ssam len = MCLBYTES; 1718104477Ssam } 1719104477Ssam totlen -= len; 1720104477Ssam m0->m_pkthdr.len = m0->m_len = len; 1721104477Ssam mlast = m0; 1722104477Ssam 1723104477Ssam while (totlen > 0) { 1724111119Simp MGET(m, M_DONTWAIT, MT_DATA); 1725104477Ssam if (m == NULL) { 1726104477Ssam hifnstats.hst_nomem_mbuf++; 1727104477Ssam err = dma->cmdu ? ERESTART : ENOMEM; 1728104477Ssam m_freem(m0); 1729104477Ssam goto err_srcmap; 1730104477Ssam } 1731104477Ssam len = MLEN; 1732104477Ssam if (totlen >= MINCLSIZE) { 1733111119Simp MCLGET(m, M_DONTWAIT); 1734104477Ssam if ((m->m_flags & M_EXT) == 0) { 1735104477Ssam hifnstats.hst_nomem_mcl++; 1736104477Ssam err = dma->cmdu ? ERESTART : ENOMEM; 1737104477Ssam mlast->m_next = m; 1738104477Ssam m_freem(m0); 1739104477Ssam goto err_srcmap; 1740104477Ssam } 1741104477Ssam len = MCLBYTES; 1742104477Ssam } 1743104477Ssam 1744104477Ssam m->m_len = len; 1745104477Ssam m0->m_pkthdr.len += len; 1746104477Ssam totlen -= len; 1747104477Ssam 1748104477Ssam mlast->m_next = m; 1749104477Ssam mlast = m; 1750104477Ssam } 1751104477Ssam cmd->dst_m = m0; 1752104477Ssam } 1753104477Ssam } 1754104477Ssam 1755104477Ssam if (cmd->dst_map == NULL) { 1756104477Ssam if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->dst_map)) { 1757104477Ssam hifnstats.hst_nomem_map++; 1758104477Ssam err = ENOMEM; 1759104477Ssam goto err_srcmap; 1760104477Ssam } 1761104477Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) { 1762104477Ssam if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map, 1763104477Ssam cmd->dst_m, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) { 1764104477Ssam hifnstats.hst_nomem_map++; 1765104477Ssam err = ENOMEM; 1766104477Ssam goto err_dstmap1; 1767104477Ssam } 1768104477Ssam } else if (crp->crp_flags & CRYPTO_F_IOV) { 1769104477Ssam if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map, 1770104477Ssam cmd->dst_io, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) { 1771104477Ssam hifnstats.hst_nomem_load++; 1772104477Ssam err = ENOMEM; 1773104477Ssam goto err_dstmap1; 1774104477Ssam } 1775104477Ssam } 1776104477Ssam } 1777104477Ssam 1778104477Ssam#ifdef HIFN_DEBUG 1779104477Ssam if (hifn_debug) { 1780104477Ssam device_printf(sc->sc_dev, 1781104477Ssam "Entering cmd: stat %8x ien %8x u %d/%d/%d/%d n %d/%d\n", 1782104477Ssam READ_REG_1(sc, HIFN_1_DMA_CSR), 1783104477Ssam READ_REG_1(sc, HIFN_1_DMA_IER), 1784104477Ssam dma->cmdu, dma->srcu, dma->dstu, dma->resu, 1785104477Ssam cmd->src_nsegs, cmd->dst_nsegs); 1786104477Ssam } 1787104477Ssam#endif 1788104477Ssam 1789104477Ssam if (cmd->src_map == cmd->dst_map) { 1790104477Ssam bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 1791104477Ssam BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD); 1792104477Ssam } else { 1793104477Ssam bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 1794104477Ssam BUS_DMASYNC_PREWRITE); 1795104477Ssam bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 1796104477Ssam BUS_DMASYNC_PREREAD); 1797104477Ssam } 1798104477Ssam 1799104477Ssam /* 1800104477Ssam * need N src, and N dst 1801104477Ssam */ 1802104477Ssam if ((dma->srcu + cmd->src_nsegs) > HIFN_D_SRC_RSIZE || 1803104477Ssam (dma->dstu + cmd->dst_nsegs + 1) > HIFN_D_DST_RSIZE) { 1804104477Ssam#ifdef HIFN_DEBUG 1805104477Ssam if (hifn_debug) { 1806104477Ssam device_printf(sc->sc_dev, 1807104477Ssam "src/dst exhaustion, srcu %u+%u dstu %u+%u\n", 1808104477Ssam dma->srcu, cmd->src_nsegs, 1809104477Ssam dma->dstu, cmd->dst_nsegs); 1810104477Ssam } 1811104477Ssam#endif 1812104477Ssam hifnstats.hst_nomem_sd++; 1813104477Ssam err = ERESTART; 1814104477Ssam goto err_dstmap; 1815104477Ssam } 1816104477Ssam 1817104477Ssam if (dma->cmdi == HIFN_D_CMD_RSIZE) { 1818104477Ssam dma->cmdi = 0; 1819104477Ssam dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID | 1820104477Ssam HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1821104477Ssam HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE, 1822104477Ssam BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1823104477Ssam } 1824104477Ssam cmdi = dma->cmdi++; 1825104477Ssam cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]); 1826104477Ssam HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE); 1827104477Ssam 1828104477Ssam /* .p for command/result already set */ 1829104477Ssam dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST | 1830104477Ssam HIFN_D_MASKDONEIRQ); 1831104477Ssam HIFN_CMDR_SYNC(sc, cmdi, 1832104477Ssam BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1833104477Ssam dma->cmdu++; 1834104477Ssam if (sc->sc_c_busy == 0) { 1835104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA); 1836104477Ssam sc->sc_c_busy = 1; 1837104477Ssam } 1838104477Ssam 1839104477Ssam /* 1840104477Ssam * We don't worry about missing an interrupt (which a "command wait" 1841104477Ssam * interrupt salvages us from), unless there is more than one command 1842104477Ssam * in the queue. 1843104477Ssam */ 1844104477Ssam if (dma->cmdu > 1) { 1845104477Ssam sc->sc_dmaier |= HIFN_DMAIER_C_WAIT; 1846104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 1847104477Ssam } 1848104477Ssam 1849104477Ssam hifnstats.hst_ipackets++; 1850104477Ssam hifnstats.hst_ibytes += cmd->src_mapsize; 1851104477Ssam 1852104477Ssam hifn_dmamap_load_src(sc, cmd); 1853104477Ssam if (sc->sc_s_busy == 0) { 1854104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA); 1855104477Ssam sc->sc_s_busy = 1; 1856104477Ssam } 1857104477Ssam 1858104477Ssam /* 1859104477Ssam * Unlike other descriptors, we don't mask done interrupt from 1860104477Ssam * result descriptor. 1861104477Ssam */ 1862104477Ssam#ifdef HIFN_DEBUG 1863104477Ssam if (hifn_debug) 1864104477Ssam printf("load res\n"); 1865104477Ssam#endif 1866104477Ssam if (dma->resi == HIFN_D_RES_RSIZE) { 1867104477Ssam dma->resi = 0; 1868104477Ssam dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID | 1869104477Ssam HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1870104477Ssam HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE, 1871104477Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1872104477Ssam } 1873104477Ssam resi = dma->resi++; 1874104477Ssam KASSERT(dma->hifn_commands[resi] == NULL, 1875104477Ssam ("hifn_crypto: command slot %u busy", resi)); 1876104477Ssam dma->hifn_commands[resi] = cmd; 1877104477Ssam HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD); 1878104477Ssam if ((hint & CRYPTO_HINT_MORE) && sc->sc_curbatch < hifn_maxbatch) { 1879104477Ssam dma->resr[resi].l = htole32(HIFN_MAX_RESULT | 1880104477Ssam HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ); 1881104477Ssam sc->sc_curbatch++; 1882104477Ssam if (sc->sc_curbatch > hifnstats.hst_maxbatch) 1883104477Ssam hifnstats.hst_maxbatch = sc->sc_curbatch; 1884104477Ssam hifnstats.hst_totbatch++; 1885104477Ssam } else { 1886104477Ssam dma->resr[resi].l = htole32(HIFN_MAX_RESULT | 1887104477Ssam HIFN_D_VALID | HIFN_D_LAST); 1888104477Ssam sc->sc_curbatch = 0; 1889104477Ssam } 1890104477Ssam HIFN_RESR_SYNC(sc, resi, 1891104477Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1892104477Ssam dma->resu++; 1893104477Ssam if (sc->sc_r_busy == 0) { 1894104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA); 1895104477Ssam sc->sc_r_busy = 1; 1896104477Ssam } 1897104477Ssam 1898104477Ssam if (cmd->sloplen) 1899104477Ssam cmd->slopidx = resi; 1900104477Ssam 1901104477Ssam hifn_dmamap_load_dst(sc, cmd); 1902104477Ssam 1903104477Ssam if (sc->sc_d_busy == 0) { 1904104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA); 1905104477Ssam sc->sc_d_busy = 1; 1906104477Ssam } 1907104477Ssam 1908104477Ssam#ifdef HIFN_DEBUG 1909104477Ssam if (hifn_debug) { 1910104477Ssam device_printf(sc->sc_dev, "command: stat %8x ier %8x\n", 1911104477Ssam READ_REG_1(sc, HIFN_1_DMA_CSR), 1912104477Ssam READ_REG_1(sc, HIFN_1_DMA_IER)); 1913104477Ssam } 1914104477Ssam#endif 1915104477Ssam 1916104477Ssam sc->sc_active = 5; 1917115748Ssam HIFN_UNLOCK(sc); 1918104477Ssam KASSERT(err == 0, ("hifn_crypto: success with error %u", err)); 1919104477Ssam return (err); /* success */ 1920104477Ssam 1921104477Ssamerr_dstmap: 1922104477Ssam if (cmd->src_map != cmd->dst_map) 1923104477Ssam bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 1924104477Ssamerr_dstmap1: 1925104477Ssam if (cmd->src_map != cmd->dst_map) 1926104477Ssam bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 1927104477Ssamerr_srcmap: 1928104477Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) { 1929104477Ssam if (cmd->src_m != cmd->dst_m) 1930104477Ssam m_freem(cmd->dst_m); 1931104477Ssam } 1932104477Ssam bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 1933104477Ssamerr_srcmap1: 1934104477Ssam bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 1935115748Ssam HIFN_UNLOCK(sc); 1936104477Ssam return (err); 1937104477Ssam} 1938104477Ssam 1939104477Ssamstatic void 1940104477Ssamhifn_tick(void* vsc) 1941104477Ssam{ 1942104477Ssam struct hifn_softc *sc = vsc; 1943104477Ssam 1944104477Ssam HIFN_LOCK(sc); 1945104477Ssam if (sc->sc_active == 0) { 1946104477Ssam struct hifn_dma *dma = sc->sc_dma; 1947104477Ssam u_int32_t r = 0; 1948104477Ssam 1949104477Ssam if (dma->cmdu == 0 && sc->sc_c_busy) { 1950104477Ssam sc->sc_c_busy = 0; 1951104477Ssam r |= HIFN_DMACSR_C_CTRL_DIS; 1952104477Ssam } 1953104477Ssam if (dma->srcu == 0 && sc->sc_s_busy) { 1954104477Ssam sc->sc_s_busy = 0; 1955104477Ssam r |= HIFN_DMACSR_S_CTRL_DIS; 1956104477Ssam } 1957104477Ssam if (dma->dstu == 0 && sc->sc_d_busy) { 1958104477Ssam sc->sc_d_busy = 0; 1959104477Ssam r |= HIFN_DMACSR_D_CTRL_DIS; 1960104477Ssam } 1961104477Ssam if (dma->resu == 0 && sc->sc_r_busy) { 1962104477Ssam sc->sc_r_busy = 0; 1963104477Ssam r |= HIFN_DMACSR_R_CTRL_DIS; 1964104477Ssam } 1965104477Ssam if (r) 1966104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CSR, r); 1967104477Ssam } else 1968104477Ssam sc->sc_active--; 1969104477Ssam HIFN_UNLOCK(sc); 1970104477Ssam callout_reset(&sc->sc_tickto, hz, hifn_tick, sc); 1971104477Ssam} 1972104477Ssam 1973104477Ssamstatic void 1974104477Ssamhifn_intr(void *arg) 1975104477Ssam{ 1976104477Ssam struct hifn_softc *sc = arg; 1977104477Ssam struct hifn_dma *dma; 1978104477Ssam u_int32_t dmacsr, restart; 1979104477Ssam int i, u; 1980104477Ssam 1981115748Ssam dmacsr = READ_REG_1(sc, HIFN_1_DMA_CSR); 1982115748Ssam 1983115748Ssam /* Nothing in the DMA unit interrupted */ 1984115748Ssam if ((dmacsr & sc->sc_dmaier) == 0) 1985115748Ssam return; 1986115748Ssam 1987104477Ssam HIFN_LOCK(sc); 1988115748Ssam 1989104477Ssam dma = sc->sc_dma; 1990104477Ssam 1991104477Ssam#ifdef HIFN_DEBUG 1992104477Ssam if (hifn_debug) { 1993104477Ssam device_printf(sc->sc_dev, 1994104477Ssam "irq: stat %08x ien %08x damier %08x i %d/%d/%d/%d k %d/%d/%d/%d u %d/%d/%d/%d\n", 1995104477Ssam dmacsr, READ_REG_1(sc, HIFN_1_DMA_IER), sc->sc_dmaier, 1996104477Ssam dma->cmdi, dma->srci, dma->dsti, dma->resi, 1997104477Ssam dma->cmdk, dma->srck, dma->dstk, dma->resk, 1998104477Ssam dma->cmdu, dma->srcu, dma->dstu, dma->resu); 1999104477Ssam } 2000104477Ssam#endif 2001104477Ssam 2002104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_CSR, dmacsr & sc->sc_dmaier); 2003104477Ssam 2004104477Ssam if ((sc->sc_flags & HIFN_HAS_PUBLIC) && 2005104477Ssam (dmacsr & HIFN_DMACSR_PUBDONE)) 2006104477Ssam WRITE_REG_1(sc, HIFN_1_PUB_STATUS, 2007104477Ssam READ_REG_1(sc, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE); 2008104477Ssam 2009104477Ssam restart = dmacsr & (HIFN_DMACSR_D_OVER | HIFN_DMACSR_R_OVER); 2010104477Ssam if (restart) 2011104477Ssam device_printf(sc->sc_dev, "overrun %x\n", dmacsr); 2012104477Ssam 2013104477Ssam if (sc->sc_flags & HIFN_IS_7811) { 2014104477Ssam if (dmacsr & HIFN_DMACSR_ILLR) 2015104477Ssam device_printf(sc->sc_dev, "illegal read\n"); 2016104477Ssam if (dmacsr & HIFN_DMACSR_ILLW) 2017104477Ssam device_printf(sc->sc_dev, "illegal write\n"); 2018104477Ssam } 2019104477Ssam 2020104477Ssam restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT | 2021104477Ssam HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT); 2022104477Ssam if (restart) { 2023104477Ssam device_printf(sc->sc_dev, "abort, resetting.\n"); 2024104477Ssam hifnstats.hst_abort++; 2025104477Ssam hifn_abort(sc); 2026104477Ssam HIFN_UNLOCK(sc); 2027104477Ssam return; 2028104477Ssam } 2029104477Ssam 2030104477Ssam if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) { 2031104477Ssam /* 2032104477Ssam * If no slots to process and we receive a "waiting on 2033104477Ssam * command" interrupt, we disable the "waiting on command" 2034104477Ssam * (by clearing it). 2035104477Ssam */ 2036104477Ssam sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT; 2037104477Ssam WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 2038104477Ssam } 2039104477Ssam 2040104477Ssam /* clear the rings */ 2041104477Ssam i = dma->resk; u = dma->resu; 2042104477Ssam while (u != 0) { 2043104477Ssam HIFN_RESR_SYNC(sc, i, 2044104477Ssam BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2045104477Ssam if (dma->resr[i].l & htole32(HIFN_D_VALID)) { 2046104477Ssam HIFN_RESR_SYNC(sc, i, 2047104477Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2048104477Ssam break; 2049104477Ssam } 2050104477Ssam 2051104477Ssam if (i != HIFN_D_RES_RSIZE) { 2052104477Ssam struct hifn_command *cmd; 2053104477Ssam u_int8_t *macbuf = NULL; 2054104477Ssam 2055104477Ssam HIFN_RES_SYNC(sc, i, BUS_DMASYNC_POSTREAD); 2056104477Ssam cmd = dma->hifn_commands[i]; 2057104477Ssam KASSERT(cmd != NULL, 2058104477Ssam ("hifn_intr: null command slot %u", i)); 2059104477Ssam dma->hifn_commands[i] = NULL; 2060104477Ssam 2061104477Ssam if (cmd->base_masks & HIFN_BASE_CMD_MAC) { 2062104477Ssam macbuf = dma->result_bufs[i]; 2063104477Ssam macbuf += 12; 2064104477Ssam } 2065104477Ssam 2066104477Ssam hifn_callback(sc, cmd, macbuf); 2067104477Ssam hifnstats.hst_opackets++; 2068104477Ssam u--; 2069104477Ssam } 2070104477Ssam 2071104477Ssam if (++i == (HIFN_D_RES_RSIZE + 1)) 2072104477Ssam i = 0; 2073104477Ssam } 2074104477Ssam dma->resk = i; dma->resu = u; 2075104477Ssam 2076104477Ssam i = dma->srck; u = dma->srcu; 2077104477Ssam while (u != 0) { 2078104477Ssam if (i == HIFN_D_SRC_RSIZE) 2079104477Ssam i = 0; 2080104477Ssam HIFN_SRCR_SYNC(sc, i, 2081104477Ssam BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2082104477Ssam if (dma->srcr[i].l & htole32(HIFN_D_VALID)) { 2083104477Ssam HIFN_SRCR_SYNC(sc, i, 2084104477Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2085104477Ssam break; 2086104477Ssam } 2087104477Ssam i++, u--; 2088104477Ssam } 2089104477Ssam dma->srck = i; dma->srcu = u; 2090104477Ssam 2091104477Ssam i = dma->cmdk; u = dma->cmdu; 2092104477Ssam while (u != 0) { 2093104477Ssam HIFN_CMDR_SYNC(sc, i, 2094104477Ssam BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2095104477Ssam if (dma->cmdr[i].l & htole32(HIFN_D_VALID)) { 2096104477Ssam HIFN_CMDR_SYNC(sc, i, 2097104477Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2098104477Ssam break; 2099104477Ssam } 2100104477Ssam if (i != HIFN_D_CMD_RSIZE) { 2101104477Ssam u--; 2102104477Ssam HIFN_CMD_SYNC(sc, i, BUS_DMASYNC_POSTWRITE); 2103104477Ssam } 2104104477Ssam if (++i == (HIFN_D_CMD_RSIZE + 1)) 2105104477Ssam i = 0; 2106104477Ssam } 2107104477Ssam dma->cmdk = i; dma->cmdu = u; 2108104477Ssam 2109115748Ssam HIFN_UNLOCK(sc); 2110115748Ssam 2111104477Ssam if (sc->sc_needwakeup) { /* XXX check high watermark */ 2112104477Ssam int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ); 2113104477Ssam#ifdef HIFN_DEBUG 2114104477Ssam if (hifn_debug) 2115104477Ssam device_printf(sc->sc_dev, 2116104477Ssam "wakeup crypto (%x) u %d/%d/%d/%d\n", 2117104477Ssam sc->sc_needwakeup, 2118104477Ssam dma->cmdu, dma->srcu, dma->dstu, dma->resu); 2119104477Ssam#endif 2120104477Ssam sc->sc_needwakeup &= ~wakeup; 2121104477Ssam crypto_unblock(sc->sc_cid, wakeup); 2122104477Ssam } 2123104477Ssam} 2124104477Ssam 2125104477Ssam/* 2126104477Ssam * Allocate a new 'session' and return an encoded session id. 'sidp' 2127104477Ssam * contains our registration id, and should contain an encoded session 2128104477Ssam * id on successful allocation. 2129104477Ssam */ 2130104477Ssamstatic int 2131104477Ssamhifn_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri) 2132104477Ssam{ 2133104477Ssam struct cryptoini *c; 2134104477Ssam struct hifn_softc *sc = arg; 2135104477Ssam int i, mac = 0, cry = 0; 2136104477Ssam 2137104477Ssam KASSERT(sc != NULL, ("hifn_newsession: null softc")); 2138104477Ssam if (sidp == NULL || cri == NULL || sc == NULL) 2139104477Ssam return (EINVAL); 2140104477Ssam 2141104477Ssam for (i = 0; i < sc->sc_maxses; i++) 2142104477Ssam if (sc->sc_sessions[i].hs_state == HS_STATE_FREE) 2143104477Ssam break; 2144104477Ssam if (i == sc->sc_maxses) 2145104477Ssam return (ENOMEM); 2146104477Ssam 2147104477Ssam for (c = cri; c != NULL; c = c->cri_next) { 2148104477Ssam switch (c->cri_alg) { 2149104477Ssam case CRYPTO_MD5: 2150104477Ssam case CRYPTO_SHA1: 2151104477Ssam case CRYPTO_MD5_HMAC: 2152104477Ssam case CRYPTO_SHA1_HMAC: 2153104477Ssam if (mac) 2154104477Ssam return (EINVAL); 2155104477Ssam mac = 1; 2156104477Ssam break; 2157104477Ssam case CRYPTO_DES_CBC: 2158104477Ssam case CRYPTO_3DES_CBC: 2159104477Ssam /* XXX this may read fewer, does it matter? */ 2160104477Ssam read_random(sc->sc_sessions[i].hs_iv, HIFN_IV_LENGTH); 2161104477Ssam /*FALLTHROUGH*/ 2162104477Ssam case CRYPTO_ARC4: 2163104477Ssam if (cry) 2164104477Ssam return (EINVAL); 2165104477Ssam cry = 1; 2166104477Ssam break; 2167104477Ssam default: 2168104477Ssam return (EINVAL); 2169104477Ssam } 2170104477Ssam } 2171104477Ssam if (mac == 0 && cry == 0) 2172104477Ssam return (EINVAL); 2173104477Ssam 2174104477Ssam *sidp = HIFN_SID(device_get_unit(sc->sc_dev), i); 2175104477Ssam sc->sc_sessions[i].hs_state = HS_STATE_USED; 2176104477Ssam 2177104477Ssam return (0); 2178104477Ssam} 2179104477Ssam 2180104477Ssam/* 2181104477Ssam * Deallocate a session. 2182104477Ssam * XXX this routine should run a zero'd mac/encrypt key into context ram. 2183104477Ssam * XXX to blow away any keys already stored there. 2184104477Ssam */ 2185104477Ssamstatic int 2186104477Ssamhifn_freesession(void *arg, u_int64_t tid) 2187104477Ssam{ 2188104477Ssam struct hifn_softc *sc = arg; 2189104477Ssam int session; 2190104477Ssam u_int32_t sid = ((u_int32_t) tid) & 0xffffffff; 2191104477Ssam 2192104477Ssam KASSERT(sc != NULL, ("hifn_freesession: null softc")); 2193104477Ssam if (sc == NULL) 2194104477Ssam return (EINVAL); 2195104477Ssam 2196104477Ssam session = HIFN_SESSION(sid); 2197104477Ssam if (session >= sc->sc_maxses) 2198104477Ssam return (EINVAL); 2199104477Ssam 2200104477Ssam bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session])); 2201104477Ssam return (0); 2202104477Ssam} 2203104477Ssam 2204104477Ssamstatic int 2205104477Ssamhifn_process(void *arg, struct cryptop *crp, int hint) 2206104477Ssam{ 2207104477Ssam struct hifn_softc *sc = arg; 2208104477Ssam struct hifn_command *cmd = NULL; 2209104477Ssam int session, err; 2210104477Ssam struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 2211104477Ssam 2212104477Ssam if (crp == NULL || crp->crp_callback == NULL) { 2213104477Ssam hifnstats.hst_invalid++; 2214104477Ssam return (EINVAL); 2215104477Ssam } 2216104477Ssam session = HIFN_SESSION(crp->crp_sid); 2217104477Ssam 2218104477Ssam if (sc == NULL || session >= sc->sc_maxses) { 2219104477Ssam err = EINVAL; 2220104477Ssam goto errout; 2221104477Ssam } 2222104477Ssam 2223104477Ssam cmd = malloc(sizeof(struct hifn_command), M_DEVBUF, M_NOWAIT | M_ZERO); 2224104477Ssam if (cmd == NULL) { 2225104477Ssam hifnstats.hst_nomem++; 2226104477Ssam err = ENOMEM; 2227104477Ssam goto errout; 2228104477Ssam } 2229104477Ssam 2230104477Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) { 2231104477Ssam cmd->src_m = (struct mbuf *)crp->crp_buf; 2232104477Ssam cmd->dst_m = (struct mbuf *)crp->crp_buf; 2233104477Ssam } else if (crp->crp_flags & CRYPTO_F_IOV) { 2234104477Ssam cmd->src_io = (struct uio *)crp->crp_buf; 2235104477Ssam cmd->dst_io = (struct uio *)crp->crp_buf; 2236104477Ssam } else { 2237104477Ssam err = EINVAL; 2238104477Ssam goto errout; /* XXX we don't handle contiguous buffers! */ 2239104477Ssam } 2240104477Ssam 2241104477Ssam crd1 = crp->crp_desc; 2242104477Ssam if (crd1 == NULL) { 2243104477Ssam err = EINVAL; 2244104477Ssam goto errout; 2245104477Ssam } 2246104477Ssam crd2 = crd1->crd_next; 2247104477Ssam 2248104477Ssam if (crd2 == NULL) { 2249104477Ssam if (crd1->crd_alg == CRYPTO_MD5_HMAC || 2250104477Ssam crd1->crd_alg == CRYPTO_SHA1_HMAC || 2251104477Ssam crd1->crd_alg == CRYPTO_SHA1 || 2252104477Ssam crd1->crd_alg == CRYPTO_MD5) { 2253104477Ssam maccrd = crd1; 2254104477Ssam enccrd = NULL; 2255104477Ssam } else if (crd1->crd_alg == CRYPTO_DES_CBC || 2256104477Ssam crd1->crd_alg == CRYPTO_3DES_CBC || 2257104477Ssam crd1->crd_alg == CRYPTO_ARC4) { 2258104477Ssam if ((crd1->crd_flags & CRD_F_ENCRYPT) == 0) 2259104477Ssam cmd->base_masks |= HIFN_BASE_CMD_DECODE; 2260104477Ssam maccrd = NULL; 2261104477Ssam enccrd = crd1; 2262104477Ssam } else { 2263104477Ssam err = EINVAL; 2264104477Ssam goto errout; 2265104477Ssam } 2266104477Ssam } else { 2267104477Ssam if ((crd1->crd_alg == CRYPTO_MD5_HMAC || 2268104477Ssam crd1->crd_alg == CRYPTO_SHA1_HMAC || 2269104477Ssam crd1->crd_alg == CRYPTO_MD5 || 2270104477Ssam crd1->crd_alg == CRYPTO_SHA1) && 2271104477Ssam (crd2->crd_alg == CRYPTO_DES_CBC || 2272104477Ssam crd2->crd_alg == CRYPTO_3DES_CBC || 2273104477Ssam crd2->crd_alg == CRYPTO_ARC4) && 2274104477Ssam ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 2275104477Ssam cmd->base_masks = HIFN_BASE_CMD_DECODE; 2276104477Ssam maccrd = crd1; 2277104477Ssam enccrd = crd2; 2278104477Ssam } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 2279104477Ssam crd1->crd_alg == CRYPTO_ARC4 || 2280104477Ssam crd1->crd_alg == CRYPTO_3DES_CBC) && 2281104477Ssam (crd2->crd_alg == CRYPTO_MD5_HMAC || 2282104477Ssam crd2->crd_alg == CRYPTO_SHA1_HMAC || 2283104477Ssam crd2->crd_alg == CRYPTO_MD5 || 2284104477Ssam crd2->crd_alg == CRYPTO_SHA1) && 2285104477Ssam (crd1->crd_flags & CRD_F_ENCRYPT)) { 2286104477Ssam enccrd = crd1; 2287104477Ssam maccrd = crd2; 2288104477Ssam } else { 2289104477Ssam /* 2290104477Ssam * We cannot order the 7751 as requested 2291104477Ssam */ 2292104477Ssam err = EINVAL; 2293104477Ssam goto errout; 2294104477Ssam } 2295104477Ssam } 2296104477Ssam 2297104477Ssam if (enccrd) { 2298104477Ssam cmd->enccrd = enccrd; 2299104477Ssam cmd->base_masks |= HIFN_BASE_CMD_CRYPT; 2300104477Ssam switch (enccrd->crd_alg) { 2301104477Ssam case CRYPTO_ARC4: 2302104477Ssam cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_RC4; 2303104477Ssam if ((enccrd->crd_flags & CRD_F_ENCRYPT) 2304104477Ssam != sc->sc_sessions[session].hs_prev_op) 2305104477Ssam sc->sc_sessions[session].hs_state = 2306104477Ssam HS_STATE_USED; 2307104477Ssam break; 2308104477Ssam case CRYPTO_DES_CBC: 2309104477Ssam cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_DES | 2310104477Ssam HIFN_CRYPT_CMD_MODE_CBC | 2311104477Ssam HIFN_CRYPT_CMD_NEW_IV; 2312104477Ssam break; 2313104477Ssam case CRYPTO_3DES_CBC: 2314104477Ssam cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_3DES | 2315104477Ssam HIFN_CRYPT_CMD_MODE_CBC | 2316104477Ssam HIFN_CRYPT_CMD_NEW_IV; 2317104477Ssam break; 2318104477Ssam default: 2319104477Ssam err = EINVAL; 2320104477Ssam goto errout; 2321104477Ssam } 2322104477Ssam if (enccrd->crd_alg != CRYPTO_ARC4) { 2323104477Ssam if (enccrd->crd_flags & CRD_F_ENCRYPT) { 2324104477Ssam if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 2325104477Ssam bcopy(enccrd->crd_iv, cmd->iv, 2326104477Ssam HIFN_IV_LENGTH); 2327104477Ssam else 2328104477Ssam bcopy(sc->sc_sessions[session].hs_iv, 2329104477Ssam cmd->iv, HIFN_IV_LENGTH); 2330104477Ssam 2331104477Ssam if ((enccrd->crd_flags & CRD_F_IV_PRESENT) 2332104477Ssam == 0) { 2333104477Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) 2334104477Ssam m_copyback(cmd->src_m, 2335104477Ssam enccrd->crd_inject, 2336104477Ssam HIFN_IV_LENGTH, cmd->iv); 2337104477Ssam else if (crp->crp_flags & CRYPTO_F_IOV) 2338104477Ssam cuio_copyback(cmd->src_io, 2339104477Ssam enccrd->crd_inject, 2340104477Ssam HIFN_IV_LENGTH, cmd->iv); 2341104477Ssam } 2342104477Ssam } else { 2343104477Ssam if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 2344104477Ssam bcopy(enccrd->crd_iv, cmd->iv, 2345104477Ssam HIFN_IV_LENGTH); 2346104477Ssam else if (crp->crp_flags & CRYPTO_F_IMBUF) 2347104477Ssam m_copydata(cmd->src_m, 2348104477Ssam enccrd->crd_inject, 2349104477Ssam HIFN_IV_LENGTH, cmd->iv); 2350104477Ssam else if (crp->crp_flags & CRYPTO_F_IOV) 2351104477Ssam cuio_copydata(cmd->src_io, 2352104477Ssam enccrd->crd_inject, 2353104477Ssam HIFN_IV_LENGTH, cmd->iv); 2354104477Ssam } 2355104477Ssam } 2356104477Ssam 2357104477Ssam cmd->ck = enccrd->crd_key; 2358104477Ssam cmd->cklen = enccrd->crd_klen >> 3; 2359104477Ssam 2360104477Ssam if (sc->sc_sessions[session].hs_state == HS_STATE_USED) 2361104477Ssam cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY; 2362104477Ssam } 2363104477Ssam 2364104477Ssam if (maccrd) { 2365104477Ssam cmd->maccrd = maccrd; 2366104477Ssam cmd->base_masks |= HIFN_BASE_CMD_MAC; 2367104477Ssam 2368104477Ssam switch (maccrd->crd_alg) { 2369104477Ssam case CRYPTO_MD5: 2370104477Ssam cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 | 2371104477Ssam HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH | 2372104477Ssam HIFN_MAC_CMD_POS_IPSEC; 2373104477Ssam break; 2374104477Ssam case CRYPTO_MD5_HMAC: 2375104477Ssam cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 | 2376104477Ssam HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC | 2377104477Ssam HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC; 2378104477Ssam break; 2379104477Ssam case CRYPTO_SHA1: 2380104477Ssam cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 | 2381104477Ssam HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH | 2382104477Ssam HIFN_MAC_CMD_POS_IPSEC; 2383104477Ssam break; 2384104477Ssam case CRYPTO_SHA1_HMAC: 2385104477Ssam cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 | 2386104477Ssam HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC | 2387104477Ssam HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC; 2388104477Ssam break; 2389104477Ssam } 2390104477Ssam 2391104477Ssam if ((maccrd->crd_alg == CRYPTO_SHA1_HMAC || 2392104477Ssam maccrd->crd_alg == CRYPTO_MD5_HMAC) && 2393104477Ssam sc->sc_sessions[session].hs_state == HS_STATE_USED) { 2394104477Ssam cmd->mac_masks |= HIFN_MAC_CMD_NEW_KEY; 2395104477Ssam bcopy(maccrd->crd_key, cmd->mac, maccrd->crd_klen >> 3); 2396104477Ssam bzero(cmd->mac + (maccrd->crd_klen >> 3), 2397104477Ssam HIFN_MAC_KEY_LENGTH - (maccrd->crd_klen >> 3)); 2398104477Ssam } 2399104477Ssam } 2400104477Ssam 2401104477Ssam cmd->crp = crp; 2402104477Ssam cmd->session_num = session; 2403104477Ssam cmd->softc = sc; 2404104477Ssam 2405104477Ssam err = hifn_crypto(sc, cmd, crp, hint); 2406104477Ssam if (!err) { 2407104477Ssam if (enccrd) 2408104477Ssam sc->sc_sessions[session].hs_prev_op = 2409104477Ssam enccrd->crd_flags & CRD_F_ENCRYPT; 2410104477Ssam if (sc->sc_sessions[session].hs_state == HS_STATE_USED) 2411104477Ssam sc->sc_sessions[session].hs_state = HS_STATE_KEY; 2412104477Ssam return 0; 2413104477Ssam } else if (err == ERESTART) { 2414104477Ssam /* 2415104477Ssam * There weren't enough resources to dispatch the request 2416104477Ssam * to the part. Notify the caller so they'll requeue this 2417104477Ssam * request and resubmit it again soon. 2418104477Ssam */ 2419104477Ssam#ifdef HIFN_DEBUG 2420104477Ssam if (hifn_debug) 2421104477Ssam device_printf(sc->sc_dev, "requeue request\n"); 2422104477Ssam#endif 2423104477Ssam free(cmd, M_DEVBUF); 2424104477Ssam sc->sc_needwakeup |= CRYPTO_SYMQ; 2425104477Ssam return (err); 2426104477Ssam } 2427104477Ssam 2428104477Ssamerrout: 2429104477Ssam if (cmd != NULL) 2430104477Ssam free(cmd, M_DEVBUF); 2431104477Ssam if (err == EINVAL) 2432104477Ssam hifnstats.hst_invalid++; 2433104477Ssam else 2434104477Ssam hifnstats.hst_nomem++; 2435104477Ssam crp->crp_etype = err; 2436104477Ssam crypto_done(crp); 2437104477Ssam return (err); 2438104477Ssam} 2439104477Ssam 2440104477Ssamstatic void 2441104477Ssamhifn_abort(struct hifn_softc *sc) 2442104477Ssam{ 2443104477Ssam struct hifn_dma *dma = sc->sc_dma; 2444104477Ssam struct hifn_command *cmd; 2445104477Ssam struct cryptop *crp; 2446104477Ssam int i, u; 2447104477Ssam 2448104477Ssam i = dma->resk; u = dma->resu; 2449104477Ssam while (u != 0) { 2450104477Ssam cmd = dma->hifn_commands[i]; 2451104477Ssam KASSERT(cmd != NULL, ("hifn_abort: null command slot %u", i)); 2452104477Ssam dma->hifn_commands[i] = NULL; 2453104477Ssam crp = cmd->crp; 2454104477Ssam 2455104477Ssam if ((dma->resr[i].l & htole32(HIFN_D_VALID)) == 0) { 2456104477Ssam /* Salvage what we can. */ 2457104477Ssam u_int8_t *macbuf; 2458104477Ssam 2459104477Ssam if (cmd->base_masks & HIFN_BASE_CMD_MAC) { 2460104477Ssam macbuf = dma->result_bufs[i]; 2461104477Ssam macbuf += 12; 2462104477Ssam } else 2463104477Ssam macbuf = NULL; 2464104477Ssam hifnstats.hst_opackets++; 2465104477Ssam hifn_callback(sc, cmd, macbuf); 2466104477Ssam } else { 2467104477Ssam if (cmd->src_map == cmd->dst_map) { 2468104477Ssam bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2469104477Ssam BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 2470104477Ssam } else { 2471104477Ssam bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2472104477Ssam BUS_DMASYNC_POSTWRITE); 2473104477Ssam bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2474104477Ssam BUS_DMASYNC_POSTREAD); 2475104477Ssam } 2476104477Ssam 2477104477Ssam if (cmd->src_m != cmd->dst_m) { 2478104477Ssam m_freem(cmd->src_m); 2479104477Ssam crp->crp_buf = (caddr_t)cmd->dst_m; 2480104477Ssam } 2481104477Ssam 2482104477Ssam /* non-shared buffers cannot be restarted */ 2483104477Ssam if (cmd->src_map != cmd->dst_map) { 2484104477Ssam /* 2485104477Ssam * XXX should be EAGAIN, delayed until 2486104477Ssam * after the reset. 2487104477Ssam */ 2488104477Ssam crp->crp_etype = ENOMEM; 2489104477Ssam bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 2490104477Ssam bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2491104477Ssam } else 2492104477Ssam crp->crp_etype = ENOMEM; 2493104477Ssam 2494104477Ssam bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2495104477Ssam bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2496104477Ssam 2497104477Ssam free(cmd, M_DEVBUF); 2498104477Ssam if (crp->crp_etype != EAGAIN) 2499104477Ssam crypto_done(crp); 2500104477Ssam } 2501104477Ssam 2502104477Ssam if (++i == HIFN_D_RES_RSIZE) 2503104477Ssam i = 0; 2504104477Ssam u--; 2505104477Ssam } 2506104477Ssam dma->resk = i; dma->resu = u; 2507104477Ssam 2508104477Ssam /* Force upload of key next time */ 2509104477Ssam for (i = 0; i < sc->sc_maxses; i++) 2510104477Ssam if (sc->sc_sessions[i].hs_state == HS_STATE_KEY) 2511104477Ssam sc->sc_sessions[i].hs_state = HS_STATE_USED; 2512104477Ssam 2513104477Ssam hifn_reset_board(sc, 1); 2514104477Ssam hifn_init_dma(sc); 2515104477Ssam hifn_init_pci_registers(sc); 2516104477Ssam} 2517104477Ssam 2518104477Ssamstatic void 2519104477Ssamhifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *macbuf) 2520104477Ssam{ 2521104477Ssam struct hifn_dma *dma = sc->sc_dma; 2522104477Ssam struct cryptop *crp = cmd->crp; 2523104477Ssam struct cryptodesc *crd; 2524104477Ssam struct mbuf *m; 2525104477Ssam int totlen, i, u; 2526104477Ssam 2527104477Ssam if (cmd->src_map == cmd->dst_map) { 2528104477Ssam bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2529104477Ssam BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 2530104477Ssam } else { 2531104477Ssam bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2532104477Ssam BUS_DMASYNC_POSTWRITE); 2533104477Ssam bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2534104477Ssam BUS_DMASYNC_POSTREAD); 2535104477Ssam } 2536104477Ssam 2537104477Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) { 2538104477Ssam if (cmd->src_m != cmd->dst_m) { 2539104477Ssam crp->crp_buf = (caddr_t)cmd->dst_m; 2540104477Ssam totlen = cmd->src_mapsize; 2541104477Ssam for (m = cmd->dst_m; m != NULL; m = m->m_next) { 2542104477Ssam if (totlen < m->m_len) { 2543104477Ssam m->m_len = totlen; 2544104477Ssam totlen = 0; 2545104477Ssam } else 2546104477Ssam totlen -= m->m_len; 2547104477Ssam } 2548104477Ssam cmd->dst_m->m_pkthdr.len = cmd->src_m->m_pkthdr.len; 2549104477Ssam m_freem(cmd->src_m); 2550104477Ssam } 2551104477Ssam } 2552104477Ssam 2553104477Ssam if (cmd->sloplen != 0) { 2554104477Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) 2555104477Ssam m_copyback((struct mbuf *)crp->crp_buf, 2556104477Ssam cmd->src_mapsize - cmd->sloplen, 2557104477Ssam cmd->sloplen, (caddr_t)&dma->slop[cmd->slopidx]); 2558104477Ssam else if (crp->crp_flags & CRYPTO_F_IOV) 2559104477Ssam cuio_copyback((struct uio *)crp->crp_buf, 2560104477Ssam cmd->src_mapsize - cmd->sloplen, 2561104477Ssam cmd->sloplen, (caddr_t)&dma->slop[cmd->slopidx]); 2562104477Ssam } 2563104477Ssam 2564104477Ssam i = dma->dstk; u = dma->dstu; 2565104477Ssam while (u != 0) { 2566104477Ssam if (i == HIFN_D_DST_RSIZE) 2567104477Ssam i = 0; 2568104477Ssam bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 2569104477Ssam BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2570104477Ssam if (dma->dstr[i].l & htole32(HIFN_D_VALID)) { 2571104477Ssam bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 2572104477Ssam BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2573104477Ssam break; 2574104477Ssam } 2575104477Ssam i++, u--; 2576104477Ssam } 2577104477Ssam dma->dstk = i; dma->dstu = u; 2578104477Ssam 2579104477Ssam hifnstats.hst_obytes += cmd->dst_mapsize; 2580104477Ssam 2581104477Ssam if ((cmd->base_masks & (HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE)) == 2582104477Ssam HIFN_BASE_CMD_CRYPT) { 2583104477Ssam for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 2584104477Ssam if (crd->crd_alg != CRYPTO_DES_CBC && 2585104477Ssam crd->crd_alg != CRYPTO_3DES_CBC) 2586104477Ssam continue; 2587104477Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) 2588104477Ssam m_copydata((struct mbuf *)crp->crp_buf, 2589104477Ssam crd->crd_skip + crd->crd_len - HIFN_IV_LENGTH, 2590104477Ssam HIFN_IV_LENGTH, 2591104477Ssam cmd->softc->sc_sessions[cmd->session_num].hs_iv); 2592104477Ssam else if (crp->crp_flags & CRYPTO_F_IOV) { 2593104477Ssam cuio_copydata((struct uio *)crp->crp_buf, 2594104477Ssam crd->crd_skip + crd->crd_len - HIFN_IV_LENGTH, 2595104477Ssam HIFN_IV_LENGTH, 2596104477Ssam cmd->softc->sc_sessions[cmd->session_num].hs_iv); 2597104477Ssam } 2598104477Ssam break; 2599104477Ssam } 2600104477Ssam } 2601104477Ssam 2602104477Ssam if (macbuf != NULL) { 2603104477Ssam for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 2604105275Ssam int len; 2605104477Ssam 2606105275Ssam if (crd->crd_alg == CRYPTO_MD5) 2607105275Ssam len = 16; 2608105275Ssam else if (crd->crd_alg == CRYPTO_SHA1) 2609105275Ssam len = 20; 2610105275Ssam else if (crd->crd_alg == CRYPTO_MD5_HMAC || 2611105275Ssam crd->crd_alg == CRYPTO_SHA1_HMAC) 2612105275Ssam len = 12; 2613105275Ssam else 2614104477Ssam continue; 2615104477Ssam 2616104477Ssam if (crp->crp_flags & CRYPTO_F_IMBUF) 2617104477Ssam m_copyback((struct mbuf *)crp->crp_buf, 2618104477Ssam crd->crd_inject, len, macbuf); 2619104477Ssam else if ((crp->crp_flags & CRYPTO_F_IOV) && crp->crp_mac) 2620104477Ssam bcopy((caddr_t)macbuf, crp->crp_mac, len); 2621104477Ssam break; 2622104477Ssam } 2623104477Ssam } 2624104477Ssam 2625104477Ssam if (cmd->src_map != cmd->dst_map) { 2626104477Ssam bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 2627104477Ssam bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2628104477Ssam } 2629104477Ssam bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2630104477Ssam bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2631104477Ssam free(cmd, M_DEVBUF); 2632104477Ssam crypto_done(crp); 2633104477Ssam} 2634104477Ssam 2635104477Ssam/* 2636104477Ssam * 7811 PB3 rev/2 parts lock-up on burst writes to Group 0 2637104477Ssam * and Group 1 registers; avoid conditions that could create 2638104477Ssam * burst writes by doing a read in between the writes. 2639104477Ssam * 2640104477Ssam * NB: The read we interpose is always to the same register; 2641104477Ssam * we do this because reading from an arbitrary (e.g. last) 2642104477Ssam * register may not always work. 2643104477Ssam */ 2644104477Ssamstatic void 2645104477Ssamhifn_write_reg_0(struct hifn_softc *sc, bus_size_t reg, u_int32_t val) 2646104477Ssam{ 2647104477Ssam if (sc->sc_flags & HIFN_IS_7811) { 2648104477Ssam if (sc->sc_bar0_lastreg == reg - 4) 2649104477Ssam bus_space_read_4(sc->sc_st0, sc->sc_sh0, HIFN_0_PUCNFG); 2650104477Ssam sc->sc_bar0_lastreg = reg; 2651104477Ssam } 2652104477Ssam bus_space_write_4(sc->sc_st0, sc->sc_sh0, reg, val); 2653104477Ssam} 2654104477Ssam 2655104477Ssamstatic void 2656104477Ssamhifn_write_reg_1(struct hifn_softc *sc, bus_size_t reg, u_int32_t val) 2657104477Ssam{ 2658104477Ssam if (sc->sc_flags & HIFN_IS_7811) { 2659104477Ssam if (sc->sc_bar1_lastreg == reg - 4) 2660104477Ssam bus_space_read_4(sc->sc_st1, sc->sc_sh1, HIFN_1_REVID); 2661104477Ssam sc->sc_bar1_lastreg = reg; 2662104477Ssam } 2663104477Ssam bus_space_write_4(sc->sc_st1, sc->sc_sh1, reg, val); 2664104477Ssam} 2665