1/* $FreeBSD: head/sys/dev/hifn/hifn7751.c 119280 2003-08-22 06:00:27Z imp $ */
2/* $OpenBSD: hifn7751.c,v 1.120 2002/05/17 00:33:34 deraadt Exp $ */
3
4/*
5 * Invertex AEON / Hifn 7751 driver
6 * Copyright (c) 1999 Invertex Inc. All rights reserved.
7 * Copyright (c) 1999 Theo de Raadt
8 * Copyright (c) 2000-2001 Network Security Technologies, Inc.
9 * http://www.netsec.net
10 *
11 * This driver is based on a previous driver by Invertex, for which they
12 * requested: Please send any comments, feedback, bug-fixes, or feature
13 * requests to software@invertex.com.
14 *
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
17 * are met:
18 *
19 * 1. Redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer.
21 * 2. Redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution.
24 * 3. The name of the author may not be used to endorse or promote products
25 * derived from this software without specific prior written permission.
26 *
27 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
28 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
29 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
30 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
31 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
32 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
36 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 *
38 * Effort sponsored in part by the Defense Advanced Research Projects
39 * Agency (DARPA) and Air Force Research Laboratory, Air Force
40 * Materiel Command, USAF, under agreement number F30602-01-2-0537.
41 *
42 */
43
44/*
45 * Driver for the Hifn 7751 encryption processor.
46 */
47#include "opt_hifn.h"
48
49#include <sys/param.h>
50#include <sys/systm.h>
51#include <sys/proc.h>
52#include <sys/errno.h>
53#include <sys/malloc.h>
54#include <sys/kernel.h>
55#include <sys/mbuf.h>
56#include <sys/lock.h>
57#include <sys/mutex.h>
58#include <sys/sysctl.h>
59
60#include <vm/vm.h>
61#include <vm/pmap.h>
62
63#include <machine/clock.h>
64#include <machine/bus.h>
65#include <machine/resource.h>
66#include <sys/bus.h>
67#include <sys/rman.h>
68
69#include <opencrypto/cryptodev.h>
70#include <sys/random.h>
71
72#include <dev/pci/pcivar.h>
73#include <dev/pci/pcireg.h>
74
75#ifdef HIFN_RNDTEST
76#include <dev/rndtest/rndtest.h>
77#endif
78#include <dev/hifn/hifn7751reg.h>
79#include <dev/hifn/hifn7751var.h>
80
81/*
82 * Prototypes and count for the pci_device structure
83 */
84static int hifn_probe(device_t);
85static int hifn_attach(device_t);
86static int hifn_detach(device_t);
87static int hifn_suspend(device_t);
88static int hifn_resume(device_t);
89static void hifn_shutdown(device_t);
90
91static device_method_t hifn_methods[] = {
92 /* Device interface */
93 DEVMETHOD(device_probe, hifn_probe),
94 DEVMETHOD(device_attach, hifn_attach),
95 DEVMETHOD(device_detach, hifn_detach),
96 DEVMETHOD(device_suspend, hifn_suspend),
97 DEVMETHOD(device_resume, hifn_resume),
98 DEVMETHOD(device_shutdown, hifn_shutdown),
99
100 /* bus interface */
101 DEVMETHOD(bus_print_child, bus_generic_print_child),
102 DEVMETHOD(bus_driver_added, bus_generic_driver_added),
103
104 { 0, 0 }
105};
106static driver_t hifn_driver = {
107 "hifn",
108 hifn_methods,
109 sizeof (struct hifn_softc)
110};
111static devclass_t hifn_devclass;
112
113DRIVER_MODULE(hifn, pci, hifn_driver, hifn_devclass, 0, 0);
114MODULE_DEPEND(hifn, crypto, 1, 1, 1);
115#ifdef HIFN_RNDTEST
116MODULE_DEPEND(hifn, rndtest, 1, 1, 1);
117#endif
118
119static void hifn_reset_board(struct hifn_softc *, int);
120static void hifn_reset_puc(struct hifn_softc *);
121static void hifn_puc_wait(struct hifn_softc *);
122static int hifn_enable_crypto(struct hifn_softc *);
123static void hifn_set_retry(struct hifn_softc *sc);
124static void hifn_init_dma(struct hifn_softc *);
125static void hifn_init_pci_registers(struct hifn_softc *);
126static int hifn_sramsize(struct hifn_softc *);
127static int hifn_dramsize(struct hifn_softc *);
128static int hifn_ramtype(struct hifn_softc *);
129static void hifn_sessions(struct hifn_softc *);
130static void hifn_intr(void *);
131static u_int hifn_write_command(struct hifn_command *, u_int8_t *);
132static u_int32_t hifn_next_signature(u_int32_t a, u_int cnt);
133static int hifn_newsession(void *, u_int32_t *, struct cryptoini *);
134static int hifn_freesession(void *, u_int64_t);
135static int hifn_process(void *, struct cryptop *, int);
136static void hifn_callback(struct hifn_softc *, struct hifn_command *, u_int8_t *);
137static int hifn_crypto(struct hifn_softc *, struct hifn_command *, struct cryptop *, int);
138static int hifn_readramaddr(struct hifn_softc *, int, u_int8_t *);
139static int hifn_writeramaddr(struct hifn_softc *, int, u_int8_t *);
140static int hifn_dmamap_load_src(struct hifn_softc *, struct hifn_command *);
141static int hifn_dmamap_load_dst(struct hifn_softc *, struct hifn_command *);
142static int hifn_init_pubrng(struct hifn_softc *);
143static void hifn_rng(void *);
144static void hifn_tick(void *);
145static void hifn_abort(struct hifn_softc *);
146static void hifn_alloc_slot(struct hifn_softc *, int *, int *, int *, int *);
147
148static void hifn_write_reg_0(struct hifn_softc *, bus_size_t, u_int32_t);
149static void hifn_write_reg_1(struct hifn_softc *, bus_size_t, u_int32_t);
150
151static __inline__ u_int32_t
152READ_REG_0(struct hifn_softc *sc, bus_size_t reg)
153{
154 u_int32_t v = bus_space_read_4(sc->sc_st0, sc->sc_sh0, reg);
155 sc->sc_bar0_lastreg = (bus_size_t) -1;
156 return (v);
157}
158#define WRITE_REG_0(sc, reg, val) hifn_write_reg_0(sc, reg, val)
159
160static __inline__ u_int32_t
161READ_REG_1(struct hifn_softc *sc, bus_size_t reg)
162{
163 u_int32_t v = bus_space_read_4(sc->sc_st1, sc->sc_sh1, reg);
164 sc->sc_bar1_lastreg = (bus_size_t) -1;
165 return (v);
166}
167#define WRITE_REG_1(sc, reg, val) hifn_write_reg_1(sc, reg, val)
168
169SYSCTL_NODE(_hw, OID_AUTO, hifn, CTLFLAG_RD, 0, "Hifn driver parameters");
170
171#ifdef HIFN_DEBUG
172static int hifn_debug = 0;
173SYSCTL_INT(_hw_hifn, OID_AUTO, debug, CTLFLAG_RW, &hifn_debug,
174 0, "control debugging msgs");
175#endif
176
177static struct hifn_stats hifnstats;
178SYSCTL_STRUCT(_hw_hifn, OID_AUTO, stats, CTLFLAG_RD, &hifnstats,
179 hifn_stats, "driver statistics");
180static int hifn_maxbatch = 1;
181SYSCTL_INT(_hw_hifn, OID_AUTO, maxbatch, CTLFLAG_RW, &hifn_maxbatch,
182 0, "max ops to batch w/o interrupt");
183
184/*
185 * Probe for a supported device. The PCI vendor and device
186 * IDs are used to detect devices we know how to handle.
187 */
188static int
189hifn_probe(device_t dev)
190{
191 if (pci_get_vendor(dev) == PCI_VENDOR_INVERTEX &&
192 pci_get_device(dev) == PCI_PRODUCT_INVERTEX_AEON)
193 return (0);
194 if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
195 (pci_get_device(dev) == PCI_PRODUCT_HIFN_7751 ||
196 pci_get_device(dev) == PCI_PRODUCT_HIFN_7951 ||
197 pci_get_device(dev) == PCI_PRODUCT_HIFN_7811))
198 return (0);
199 if (pci_get_vendor(dev) == PCI_VENDOR_NETSEC &&
200 pci_get_device(dev) == PCI_PRODUCT_NETSEC_7751)
201 return (0);
202 return (ENXIO);
203}
204
205static void
206hifn_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
207{
208 bus_addr_t *paddr = (bus_addr_t*) arg;
209 *paddr = segs->ds_addr;
210}
211
212static const char*
213hifn_partname(struct hifn_softc *sc)
214{
215 /* XXX sprintf numbers when not decoded */
216 switch (pci_get_vendor(sc->sc_dev)) {
217 case PCI_VENDOR_HIFN:
218 switch (pci_get_device(sc->sc_dev)) {
219 case PCI_PRODUCT_HIFN_6500: return "Hifn 6500";
220 case PCI_PRODUCT_HIFN_7751: return "Hifn 7751";
221 case PCI_PRODUCT_HIFN_7811: return "Hifn 7811";
222 case PCI_PRODUCT_HIFN_7951: return "Hifn 7951";
223 }
224 return "Hifn unknown-part";
225 case PCI_VENDOR_INVERTEX:
226 switch (pci_get_device(sc->sc_dev)) {
227 case PCI_PRODUCT_INVERTEX_AEON: return "Invertex AEON";
228 }
229 return "Invertex unknown-part";
230 case PCI_VENDOR_NETSEC:
231 switch (pci_get_device(sc->sc_dev)) {
232 case PCI_PRODUCT_NETSEC_7751: return "NetSec 7751";
233 }
234 return "NetSec unknown-part";
235 }
236 return "Unknown-vendor unknown-part";
237}
238
239static void
240default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
241{
242 random_harvest(buf, count, count*NBBY, 0, RANDOM_PURE);
243}
244
245/*
246 * Attach an interface that successfully probed.
247 */
248static int
249hifn_attach(device_t dev)
250{
251 struct hifn_softc *sc = device_get_softc(dev);
252 u_int32_t cmd;
253 caddr_t kva;
254 int rseg, rid;
255 char rbase;
256 u_int16_t ena, rev;
257
258 KASSERT(sc != NULL, ("hifn_attach: null software carrier!"));
259 bzero(sc, sizeof (*sc));
260 sc->sc_dev = dev;
261
262 mtx_init(&sc->sc_mtx, device_get_nameunit(dev), "hifn driver", MTX_DEF);
263
264 /* XXX handle power management */
265
266 /*
267 * The 7951 has a random number generator and
268 * public key support; note this.
269 */
270 if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
271 pci_get_device(dev) == PCI_PRODUCT_HIFN_7951)
272 sc->sc_flags = HIFN_HAS_RNG | HIFN_HAS_PUBLIC;
273 /*
274 * The 7811 has a random number generator and
275 * we also note it's identity 'cuz of some quirks.
276 */
277 if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
278 pci_get_device(dev) == PCI_PRODUCT_HIFN_7811)
279 sc->sc_flags |= HIFN_IS_7811 | HIFN_HAS_RNG;
280
281 /*
282 * Configure support for memory-mapped access to
283 * registers and for DMA operations.
284 */
285#define PCIM_ENA (PCIM_CMD_MEMEN|PCIM_CMD_BUSMASTEREN)
286 cmd = pci_read_config(dev, PCIR_COMMAND, 4);
287 cmd |= PCIM_ENA;
288 pci_write_config(dev, PCIR_COMMAND, cmd, 4);
289 cmd = pci_read_config(dev, PCIR_COMMAND, 4);
290 if ((cmd & PCIM_ENA) != PCIM_ENA) {
291 device_printf(dev, "failed to enable %s\n",
292 (cmd & PCIM_ENA) == 0 ?
293 "memory mapping & bus mastering" :
294 (cmd & PCIM_CMD_MEMEN) == 0 ?
295 "memory mapping" : "bus mastering");
296 goto fail_pci;
297 }
298#undef PCIM_ENA
299
300 /*
301 * Setup PCI resources. Note that we record the bus
302 * tag and handle for each register mapping, this is
303 * used by the READ_REG_0, WRITE_REG_0, READ_REG_1,
304 * and WRITE_REG_1 macros throughout the driver.
305 */
306 rid = HIFN_BAR0;
307 sc->sc_bar0res = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid,
308 0, ~0, 1, RF_ACTIVE);
309 if (sc->sc_bar0res == NULL) {
310 device_printf(dev, "cannot map bar%d register space\n", 0);
311 goto fail_pci;
312 }
313 sc->sc_st0 = rman_get_bustag(sc->sc_bar0res);
314 sc->sc_sh0 = rman_get_bushandle(sc->sc_bar0res);
315 sc->sc_bar0_lastreg = (bus_size_t) -1;
316
317 rid = HIFN_BAR1;
318 sc->sc_bar1res = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid,
319 0, ~0, 1, RF_ACTIVE);
320 if (sc->sc_bar1res == NULL) {
321 device_printf(dev, "cannot map bar%d register space\n", 1);
322 goto fail_io0;
323 }
324 sc->sc_st1 = rman_get_bustag(sc->sc_bar1res);
325 sc->sc_sh1 = rman_get_bushandle(sc->sc_bar1res);
326 sc->sc_bar1_lastreg = (bus_size_t) -1;
327
328 hifn_set_retry(sc);
329
330 /*
331 * Setup the area where the Hifn DMA's descriptors
332 * and associated data structures.
333 */
334 if (bus_dma_tag_create(NULL, /* parent */
335 1, 0, /* alignment,boundary */
336 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
337 BUS_SPACE_MAXADDR, /* highaddr */
338 NULL, NULL, /* filter, filterarg */
339 HIFN_MAX_DMALEN, /* maxsize */
340 MAX_SCATTER, /* nsegments */
341 HIFN_MAX_SEGLEN, /* maxsegsize */
342 BUS_DMA_ALLOCNOW, /* flags */
343 NULL, /* lockfunc */
344 NULL, /* lockarg */
345 &sc->sc_dmat)) {
346 device_printf(dev, "cannot allocate DMA tag\n");
347 goto fail_io1;
348 }
349 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &sc->sc_dmamap)) {
350 device_printf(dev, "cannot create dma map\n");
351 bus_dma_tag_destroy(sc->sc_dmat);
352 goto fail_io1;
353 }
354 if (bus_dmamem_alloc(sc->sc_dmat, (void**) &kva, BUS_DMA_NOWAIT, &sc->sc_dmamap)) {
355 device_printf(dev, "cannot alloc dma buffer\n");
356 bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
357 bus_dma_tag_destroy(sc->sc_dmat);
358 goto fail_io1;
359 }
360 if (bus_dmamap_load(sc->sc_dmat, sc->sc_dmamap, kva,
361 sizeof (*sc->sc_dma),
362 hifn_dmamap_cb, &sc->sc_dma_physaddr,
363 BUS_DMA_NOWAIT)) {
364 device_printf(dev, "cannot load dma map\n");
365 bus_dmamem_free(sc->sc_dmat, kva, sc->sc_dmamap);
366 bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
367 bus_dma_tag_destroy(sc->sc_dmat);
368 goto fail_io1;
369 }
370 sc->sc_dma = (struct hifn_dma *)kva;
371 bzero(sc->sc_dma, sizeof(*sc->sc_dma));
372
373 KASSERT(sc->sc_st0 != NULL, ("hifn_attach: null bar0 tag!"));
374 KASSERT(sc->sc_sh0 != NULL, ("hifn_attach: null bar0 handle!"));
375 KASSERT(sc->sc_st1 != NULL, ("hifn_attach: null bar1 tag!"));
376 KASSERT(sc->sc_sh1 != NULL, ("hifn_attach: null bar1 handle!"));
377
378 /*
379 * Reset the board and do the ``secret handshake''
380 * to enable the crypto support. Then complete the
381 * initialization procedure by setting up the interrupt
382 * and hooking in to the system crypto support so we'll
383 * get used for system services like the crypto device,
384 * IPsec, RNG device, etc.
385 */
386 hifn_reset_board(sc, 0);
387
388 if (hifn_enable_crypto(sc) != 0) {
389 device_printf(dev, "crypto enabling failed\n");
390 goto fail_mem;
391 }
392 hifn_reset_puc(sc);
393
394 hifn_init_dma(sc);
395 hifn_init_pci_registers(sc);
396
397 if (hifn_ramtype(sc))
398 goto fail_mem;
399
400 if (sc->sc_drammodel == 0)
401 hifn_sramsize(sc);
402 else
403 hifn_dramsize(sc);
404
405 /*
406 * Workaround for NetSec 7751 rev A: half ram size because two
407 * of the address lines were left floating
408 */
409 if (pci_get_vendor(dev) == PCI_VENDOR_NETSEC &&
410 pci_get_device(dev) == PCI_PRODUCT_NETSEC_7751 &&
411 pci_get_revid(dev) == 0x61) /*XXX???*/
412 sc->sc_ramsize >>= 1;
413
414 /*
415 * Arrange the interrupt line.
416 */
417 rid = 0;
418 sc->sc_irq = bus_alloc_resource(dev, SYS_RES_IRQ, &rid,
419 0, ~0, 1, RF_SHAREABLE|RF_ACTIVE);
420 if (sc->sc_irq == NULL) {
421 device_printf(dev, "could not map interrupt\n");
422 goto fail_mem;
423 }
424 /*
425 * NB: Network code assumes we are blocked with splimp()
426 * so make sure the IRQ is marked appropriately.
427 */
428 if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
429 hifn_intr, sc, &sc->sc_intrhand)) {
430 device_printf(dev, "could not setup interrupt\n");
431 goto fail_intr2;
432 }
433
434 hifn_sessions(sc);
435
436 /*
437 * NB: Keep only the low 16 bits; this masks the chip id
438 * from the 7951.
439 */
440 rev = READ_REG_1(sc, HIFN_1_REVID) & 0xffff;
441
442 rseg = sc->sc_ramsize / 1024;
443 rbase = 'K';
444 if (sc->sc_ramsize >= (1024 * 1024)) {
445 rbase = 'M';
446 rseg /= 1024;
447 }
448 device_printf(sc->sc_dev, "%s, rev %u, %d%cB %cram, %u sessions\n",
449 hifn_partname(sc), rev,
450 rseg, rbase, sc->sc_drammodel ? 'd' : 's',
451 sc->sc_maxses);
452
453 sc->sc_cid = crypto_get_driverid(0);
454 if (sc->sc_cid < 0) {
455 device_printf(dev, "could not get crypto driver id\n");
456 goto fail_intr;
457 }
458
459 WRITE_REG_0(sc, HIFN_0_PUCNFG,
460 READ_REG_0(sc, HIFN_0_PUCNFG) | HIFN_PUCNFG_CHIPID);
461 ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
462
463 switch (ena) {
464 case HIFN_PUSTAT_ENA_2:
465 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0,
466 hifn_newsession, hifn_freesession, hifn_process, sc);
467 crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0,
468 hifn_newsession, hifn_freesession, hifn_process, sc);
469 /*FALLTHROUGH*/
470 case HIFN_PUSTAT_ENA_1:
471 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0,
472 hifn_newsession, hifn_freesession, hifn_process, sc);
473 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0,
474 hifn_newsession, hifn_freesession, hifn_process, sc);
475 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0,
476 hifn_newsession, hifn_freesession, hifn_process, sc);
477 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0,
478 hifn_newsession, hifn_freesession, hifn_process, sc);
479 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0,
480 hifn_newsession, hifn_freesession, hifn_process, sc);
481 break;
482 }
483
484 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
485 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
486
487 if (sc->sc_flags & (HIFN_HAS_PUBLIC | HIFN_HAS_RNG))
488 hifn_init_pubrng(sc);
489
490 callout_init(&sc->sc_tickto, CALLOUT_MPSAFE);
491 callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
492
493 return (0);
494
495fail_intr:
496 bus_teardown_intr(dev, sc->sc_irq, sc->sc_intrhand);
497fail_intr2:
498 /* XXX don't store rid */
499 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
500fail_mem:
501 bus_dmamap_unload(sc->sc_dmat, sc->sc_dmamap);
502 bus_dmamem_free(sc->sc_dmat, sc->sc_dma, sc->sc_dmamap);
503 bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
504 bus_dma_tag_destroy(sc->sc_dmat);
505
506 /* Turn off DMA polling */
507 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
508 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
509fail_io1:
510 bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR1, sc->sc_bar1res);
511fail_io0:
512 bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR0, sc->sc_bar0res);
513fail_pci:
514 mtx_destroy(&sc->sc_mtx);
515 return (ENXIO);
516}
517
518/*
519 * Detach an interface that successfully probed.
520 */
521static int
522hifn_detach(device_t dev)
523{
524 struct hifn_softc *sc = device_get_softc(dev);
525
526 KASSERT(sc != NULL, ("hifn_detach: null software carrier!"));
527
528 /* disable interrupts */
529 WRITE_REG_1(sc, HIFN_1_DMA_IER, 0);
530
531 /*XXX other resources */
532 callout_stop(&sc->sc_tickto);
533 callout_stop(&sc->sc_rngto);
534#ifdef HIFN_RNDTEST
535 if (sc->sc_rndtest)
536 rndtest_detach(sc->sc_rndtest);
537#endif
538
539 /* Turn off DMA polling */
540 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
541 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
542
543 crypto_unregister_all(sc->sc_cid);
544
545 bus_generic_detach(dev); /*XXX should be no children, right? */
546
547 bus_teardown_intr(dev, sc->sc_irq, sc->sc_intrhand);
548 /* XXX don't store rid */
549 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
550
551 bus_dmamap_unload(sc->sc_dmat, sc->sc_dmamap);
552 bus_dmamem_free(sc->sc_dmat, sc->sc_dma, sc->sc_dmamap);
553 bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
554 bus_dma_tag_destroy(sc->sc_dmat);
555
556 bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR1, sc->sc_bar1res);
557 bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR0, sc->sc_bar0res);
558
559 mtx_destroy(&sc->sc_mtx);
560
561 return (0);
562}
563
564/*
565 * Stop all chip I/O so that the kernel's probe routines don't
566 * get confused by errant DMAs when rebooting.
567 */
568static void
569hifn_shutdown(device_t dev)
570{
571#ifdef notyet
572 hifn_stop(device_get_softc(dev));
573#endif
574}
575
576/*
577 * Device suspend routine. Stop the interface and save some PCI
578 * settings in case the BIOS doesn't restore them properly on
579 * resume.
580 */
581static int
582hifn_suspend(device_t dev)
583{
584 struct hifn_softc *sc = device_get_softc(dev);
585#ifdef notyet
586 int i;
587
588 hifn_stop(sc);
589 for (i = 0; i < 5; i++)
590 sc->saved_maps[i] = pci_read_config(dev, PCIR_MAPS + i * 4, 4);
591 sc->saved_biosaddr = pci_read_config(dev, PCIR_BIOS, 4);
592 sc->saved_intline = pci_read_config(dev, PCIR_INTLINE, 1);
593 sc->saved_cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1);
594 sc->saved_lattimer = pci_read_config(dev, PCIR_LATTIMER, 1);
595#endif
596 sc->sc_suspended = 1;
597
598 return (0);
599}
600
601/*
602 * Device resume routine. Restore some PCI settings in case the BIOS
603 * doesn't, re-enable busmastering, and restart the interface if
604 * appropriate.
605 */
606static int
607hifn_resume(device_t dev)
608{
609 struct hifn_softc *sc = device_get_softc(dev);
610#ifdef notyet
611 int i;
612
613 /* better way to do this? */
614 for (i = 0; i < 5; i++)
615 pci_write_config(dev, PCIR_MAPS + i * 4, sc->saved_maps[i], 4);
616 pci_write_config(dev, PCIR_BIOS, sc->saved_biosaddr, 4);
617 pci_write_config(dev, PCIR_INTLINE, sc->saved_intline, 1);
618 pci_write_config(dev, PCIR_CACHELNSZ, sc->saved_cachelnsz, 1);
619 pci_write_config(dev, PCIR_LATTIMER, sc->saved_lattimer, 1);
620
621 /* reenable busmastering */
622 pci_enable_busmaster(dev);
623 pci_enable_io(dev, HIFN_RES);
624
625 /* reinitialize interface if necessary */
626 if (ifp->if_flags & IFF_UP)
627 rl_init(sc);
628#endif
629 sc->sc_suspended = 0;
630
631 return (0);
632}
633
634static int
635hifn_init_pubrng(struct hifn_softc *sc)
636{
637 u_int32_t r;
638 int i;
639
640#ifdef HIFN_RNDTEST
641 sc->sc_rndtest = rndtest_attach(sc->sc_dev);
642 if (sc->sc_rndtest)
643 sc->sc_harvest = rndtest_harvest;
644 else
645 sc->sc_harvest = default_harvest;
646#else
647 sc->sc_harvest = default_harvest;
648#endif
649 if ((sc->sc_flags & HIFN_IS_7811) == 0) {
650 /* Reset 7951 public key/rng engine */
651 WRITE_REG_1(sc, HIFN_1_PUB_RESET,
652 READ_REG_1(sc, HIFN_1_PUB_RESET) | HIFN_PUBRST_RESET);
653
654 for (i = 0; i < 100; i++) {
655 DELAY(1000);
656 if ((READ_REG_1(sc, HIFN_1_PUB_RESET) &
657 HIFN_PUBRST_RESET) == 0)
658 break;
659 }
660
661 if (i == 100) {
662 device_printf(sc->sc_dev, "public key init failed\n");
663 return (1);
664 }
665 }
666
667 /* Enable the rng, if available */
668 if (sc->sc_flags & HIFN_HAS_RNG) {
669 if (sc->sc_flags & HIFN_IS_7811) {
670 r = READ_REG_1(sc, HIFN_1_7811_RNGENA);
671 if (r & HIFN_7811_RNGENA_ENA) {
672 r &= ~HIFN_7811_RNGENA_ENA;
673 WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
674 }
675 WRITE_REG_1(sc, HIFN_1_7811_RNGCFG,
676 HIFN_7811_RNGCFG_DEFL);
677 r |= HIFN_7811_RNGENA_ENA;
678 WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
679 } else
680 WRITE_REG_1(sc, HIFN_1_RNG_CONFIG,
681 READ_REG_1(sc, HIFN_1_RNG_CONFIG) |
682 HIFN_RNGCFG_ENA);
683
684 sc->sc_rngfirst = 1;
685 if (hz >= 100)
686 sc->sc_rnghz = hz / 100;
687 else
688 sc->sc_rnghz = 1;
689 callout_init(&sc->sc_rngto, CALLOUT_MPSAFE);
690 callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc);
691 }
692
693 /* Enable public key engine, if available */
694 if (sc->sc_flags & HIFN_HAS_PUBLIC) {
695 WRITE_REG_1(sc, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE);
696 sc->sc_dmaier |= HIFN_DMAIER_PUBDONE;
697 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
698 }
699
700 return (0);
701}
702
703static void
704hifn_rng(void *vsc)
705{
706#define RANDOM_BITS(n) (n)*sizeof (u_int32_t), (n)*sizeof (u_int32_t)*NBBY, 0
707 struct hifn_softc *sc = vsc;
708 u_int32_t sts, num[2];
709 int i;
710
711 if (sc->sc_flags & HIFN_IS_7811) {
712 for (i = 0; i < 5; i++) {
713 sts = READ_REG_1(sc, HIFN_1_7811_RNGSTS);
714 if (sts & HIFN_7811_RNGSTS_UFL) {
715 device_printf(sc->sc_dev,
716 "RNG underflow: disabling\n");
717 return;
718 }
719 if ((sts & HIFN_7811_RNGSTS_RDY) == 0)
720 break;
721
722 /*
723 * There are at least two words in the RNG FIFO
724 * at this point.
725 */
726 num[0] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
727 num[1] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
728 /* NB: discard first data read */
729 if (sc->sc_rngfirst)
730 sc->sc_rngfirst = 0;
731 else
732 (*sc->sc_harvest)(sc->sc_rndtest,
733 num, sizeof (num));
734 }
735 } else {
736 num[0] = READ_REG_1(sc, HIFN_1_RNG_DATA);
737
738 /* NB: discard first data read */
739 if (sc->sc_rngfirst)
740 sc->sc_rngfirst = 0;
741 else
742 (*sc->sc_harvest)(sc->sc_rndtest,
743 num, sizeof (num[0]));
744 }
745
746 callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc);
747#undef RANDOM_BITS
748}
749
750static void
751hifn_puc_wait(struct hifn_softc *sc)
752{
753 int i;
754
755 for (i = 5000; i > 0; i--) {
756 DELAY(1);
757 if (!(READ_REG_0(sc, HIFN_0_PUCTRL) & HIFN_PUCTRL_RESET))
758 break;
759 }
760 if (!i)
761 device_printf(sc->sc_dev, "proc unit did not reset\n");
762}
763
764/*
765 * Reset the processing unit.
766 */
767static void
768hifn_reset_puc(struct hifn_softc *sc)
769{
770 /* Reset processing unit */
771 WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
772 hifn_puc_wait(sc);
773}
774
775/*
776 * Set the Retry and TRDY registers; note that we set them to
777 * zero because the 7811 locks up when forced to retry (section
778 * 3.6 of "Specification Update SU-0014-04". Not clear if we
779 * should do this for all Hifn parts, but it doesn't seem to hurt.
780 */
781static void
782hifn_set_retry(struct hifn_softc *sc)
783{
784 /* NB: RETRY only responds to 8-bit reads/writes */
785 pci_write_config(sc->sc_dev, HIFN_RETRY_TIMEOUT, 0, 1);
786 pci_write_config(sc->sc_dev, HIFN_TRDY_TIMEOUT, 0, 4);
787}
788
789/*
790 * Resets the board. Values in the regesters are left as is
791 * from the reset (i.e. initial values are assigned elsewhere).
792 */
793static void
794hifn_reset_board(struct hifn_softc *sc, int full)
795{
796 u_int32_t reg;
797
798 /*
799 * Set polling in the DMA configuration register to zero. 0x7 avoids
800 * resetting the board and zeros out the other fields.
801 */
802 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
803 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
804
805 /*
806 * Now that polling has been disabled, we have to wait 1 ms
807 * before resetting the board.
808 */
809 DELAY(1000);
810
811 /* Reset the DMA unit */
812 if (full) {
813 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE);
814 DELAY(1000);
815 } else {
816 WRITE_REG_1(sc, HIFN_1_DMA_CNFG,
817 HIFN_DMACNFG_MODE | HIFN_DMACNFG_MSTRESET);
818 hifn_reset_puc(sc);
819 }
820
821 KASSERT(sc->sc_dma != NULL, ("hifn_reset_board: null DMA tag!"));
822 bzero(sc->sc_dma, sizeof(*sc->sc_dma));
823
824 /* Bring dma unit out of reset */
825 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
826 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
827
828 hifn_puc_wait(sc);
829 hifn_set_retry(sc);
830
831 if (sc->sc_flags & HIFN_IS_7811) {
832 for (reg = 0; reg < 1000; reg++) {
833 if (READ_REG_1(sc, HIFN_1_7811_MIPSRST) &
834 HIFN_MIPSRST_CRAMINIT)
835 break;
836 DELAY(1000);
837 }
838 if (reg == 1000)
839 printf(": cram init timeout\n");
840 }
841}
842
843static u_int32_t
844hifn_next_signature(u_int32_t a, u_int cnt)
845{
846 int i;
847 u_int32_t v;
848
849 for (i = 0; i < cnt; i++) {
850
851 /* get the parity */
852 v = a & 0x80080125;
853 v ^= v >> 16;
854 v ^= v >> 8;
855 v ^= v >> 4;
856 v ^= v >> 2;
857 v ^= v >> 1;
858
859 a = (v & 1) ^ (a << 1);
860 }
861
862 return a;
863}
864
865struct pci2id {
866 u_short pci_vendor;
867 u_short pci_prod;
868 char card_id[13];
869};
870static struct pci2id pci2id[] = {
871 {
872 PCI_VENDOR_HIFN,
873 PCI_PRODUCT_HIFN_7951,
874 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
875 0x00, 0x00, 0x00, 0x00, 0x00 }
876 }, {
877 PCI_VENDOR_NETSEC,
878 PCI_PRODUCT_NETSEC_7751,
879 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
880 0x00, 0x00, 0x00, 0x00, 0x00 }
881 }, {
882 PCI_VENDOR_INVERTEX,
883 PCI_PRODUCT_INVERTEX_AEON,
884 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
885 0x00, 0x00, 0x00, 0x00, 0x00 }
886 }, {
887 PCI_VENDOR_HIFN,
888 PCI_PRODUCT_HIFN_7811,
889 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
890 0x00, 0x00, 0x00, 0x00, 0x00 }
891 }, {
892 /*
893 * Other vendors share this PCI ID as well, such as
894 * http://www.powercrypt.com, and obviously they also
895 * use the same key.
896 */
897 PCI_VENDOR_HIFN,
898 PCI_PRODUCT_HIFN_7751,
899 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
900 0x00, 0x00, 0x00, 0x00, 0x00 }
901 },
902};
903
904/*
905 * Checks to see if crypto is already enabled. If crypto isn't enable,
906 * "hifn_enable_crypto" is called to enable it. The check is important,
907 * as enabling crypto twice will lock the board.
908 */
909static int
910hifn_enable_crypto(struct hifn_softc *sc)
911{
912 u_int32_t dmacfg, ramcfg, encl, addr, i;
913 char *offtbl = NULL;
914
915 for (i = 0; i < sizeof(pci2id)/sizeof(pci2id[0]); i++) {
916 if (pci2id[i].pci_vendor == pci_get_vendor(sc->sc_dev) &&
917 pci2id[i].pci_prod == pci_get_device(sc->sc_dev)) {
918 offtbl = pci2id[i].card_id;
919 break;
920 }
921 }
922 if (offtbl == NULL) {
923 device_printf(sc->sc_dev, "Unknown card!\n");
924 return (1);
925 }
926
927 ramcfg = READ_REG_0(sc, HIFN_0_PUCNFG);
928 dmacfg = READ_REG_1(sc, HIFN_1_DMA_CNFG);
929
930 /*
931 * The RAM config register's encrypt level bit needs to be set before
932 * every read performed on the encryption level register.
933 */
934 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
935
936 encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
937
938 /*
939 * Make sure we don't re-unlock. Two unlocks kills chip until the
940 * next reboot.
941 */
942 if (encl == HIFN_PUSTAT_ENA_1 || encl == HIFN_PUSTAT_ENA_2) {
943#ifdef HIFN_DEBUG
944 if (hifn_debug)
945 device_printf(sc->sc_dev,
946 "Strong crypto already enabled!\n");
947#endif
948 goto report;
949 }
950
951 if (encl != 0 && encl != HIFN_PUSTAT_ENA_0) {
952#ifdef HIFN_DEBUG
953 if (hifn_debug)
954 device_printf(sc->sc_dev,
955 "Unknown encryption level 0x%x\n", encl);
956#endif
957 return 1;
958 }
959
960 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_UNLOCK |
961 HIFN_DMACNFG_MSTRESET | HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
962 DELAY(1000);
963 addr = READ_REG_1(sc, HIFN_UNLOCK_SECRET1);
964 DELAY(1000);
965 WRITE_REG_1(sc, HIFN_UNLOCK_SECRET2, 0);
966 DELAY(1000);
967
968 for (i = 0; i <= 12; i++) {
969 addr = hifn_next_signature(addr, offtbl[i] + 0x101);
970 WRITE_REG_1(sc, HIFN_UNLOCK_SECRET2, addr);
971
972 DELAY(1000);
973 }
974
975 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
976 encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
977
978#ifdef HIFN_DEBUG
979 if (hifn_debug) {
980 if (encl != HIFN_PUSTAT_ENA_1 && encl != HIFN_PUSTAT_ENA_2)
981 device_printf(sc->sc_dev, "Engine is permanently "
982 "locked until next system reset!\n");
983 else
984 device_printf(sc->sc_dev, "Engine enabled "
985 "successfully!\n");
986 }
987#endif
988
989report:
990 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg);
991 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, dmacfg);
992
993 switch (encl) {
994 case HIFN_PUSTAT_ENA_1:
995 case HIFN_PUSTAT_ENA_2:
996 break;
997 case HIFN_PUSTAT_ENA_0:
998 default:
999 device_printf(sc->sc_dev, "disabled");
1000 break;
1001 }
1002
1003 return 0;
1004}
1005
1006/*
1007 * Give initial values to the registers listed in the "Register Space"
1008 * section of the HIFN Software Development reference manual.
1009 */
1010static void
1011hifn_init_pci_registers(struct hifn_softc *sc)
1012{
1013 /* write fixed values needed by the Initialization registers */
1014 WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
1015 WRITE_REG_0(sc, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD);
1016 WRITE_REG_0(sc, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
1017
1018 /* write all 4 ring address registers */
1019 WRITE_REG_1(sc, HIFN_1_DMA_CRAR, sc->sc_dma_physaddr +
1020 offsetof(struct hifn_dma, cmdr[0]));
1021 WRITE_REG_1(sc, HIFN_1_DMA_SRAR, sc->sc_dma_physaddr +
1022 offsetof(struct hifn_dma, srcr[0]));
1023 WRITE_REG_1(sc, HIFN_1_DMA_DRAR, sc->sc_dma_physaddr +
1024 offsetof(struct hifn_dma, dstr[0]));
1025 WRITE_REG_1(sc, HIFN_1_DMA_RRAR, sc->sc_dma_physaddr +
1026 offsetof(struct hifn_dma, resr[0]));
1027
1028 DELAY(2000);
1029
1030 /* write status register */
1031 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1032 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
1033 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS |
1034 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
1035 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
1036 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
1037 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
1038 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
1039 HIFN_DMACSR_S_WAIT |
1040 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
1041 HIFN_DMACSR_C_WAIT |
1042 HIFN_DMACSR_ENGINE |
1043 ((sc->sc_flags & HIFN_HAS_PUBLIC) ?
1044 HIFN_DMACSR_PUBDONE : 0) |
1045 ((sc->sc_flags & HIFN_IS_7811) ?
1046 HIFN_DMACSR_ILLW | HIFN_DMACSR_ILLR : 0));
1047
1048 sc->sc_d_busy = sc->sc_r_busy = sc->sc_s_busy = sc->sc_c_busy = 0;
1049 sc->sc_dmaier |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT |
1050 HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER |
1051 HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT |
1052 ((sc->sc_flags & HIFN_IS_7811) ?
1053 HIFN_DMAIER_ILLW | HIFN_DMAIER_ILLR : 0);
1054 sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
1055 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
1056
1057 WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING |
1058 HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES |
1059 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 |
1060 (sc->sc_drammodel ? HIFN_PUCNFG_DRAM : HIFN_PUCNFG_SRAM));
1061
1062 WRITE_REG_0(sc, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
1063 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
1064 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST |
1065 ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) |
1066 ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL));
1067}
1068
1069/*
1070 * The maximum number of sessions supported by the card
1071 * is dependent on the amount of context ram, which
1072 * encryption algorithms are enabled, and how compression
1073 * is configured. This should be configured before this
1074 * routine is called.
1075 */
1076static void
1077hifn_sessions(struct hifn_softc *sc)
1078{
1079 u_int32_t pucnfg;
1080 int ctxsize;
1081
1082 pucnfg = READ_REG_0(sc, HIFN_0_PUCNFG);
1083
1084 if (pucnfg & HIFN_PUCNFG_COMPSING) {
1085 if (pucnfg & HIFN_PUCNFG_ENCCNFG)
1086 ctxsize = 128;
1087 else
1088 ctxsize = 512;
1089 sc->sc_maxses = 1 +
1090 ((sc->sc_ramsize - 32768) / ctxsize);
1091 } else
1092 sc->sc_maxses = sc->sc_ramsize / 16384;
1093
1094 if (sc->sc_maxses > 2048)
1095 sc->sc_maxses = 2048;
1096}
1097
1098/*
1099 * Determine ram type (sram or dram). Board should be just out of a reset
1100 * state when this is called.
1101 */
1102static int
1103hifn_ramtype(struct hifn_softc *sc)
1104{
1105 u_int8_t data[8], dataexpect[8];
1106 int i;
1107
1108 for (i = 0; i < sizeof(data); i++)
1109 data[i] = dataexpect[i] = 0x55;
1110 if (hifn_writeramaddr(sc, 0, data))
1111 return (-1);
1112 if (hifn_readramaddr(sc, 0, data))
1113 return (-1);
1114 if (bcmp(data, dataexpect, sizeof(data)) != 0) {
1115 sc->sc_drammodel = 1;
1116 return (0);
1117 }
1118
1119 for (i = 0; i < sizeof(data); i++)
1120 data[i] = dataexpect[i] = 0xaa;
1121 if (hifn_writeramaddr(sc, 0, data))
1122 return (-1);
1123 if (hifn_readramaddr(sc, 0, data))
1124 return (-1);
1125 if (bcmp(data, dataexpect, sizeof(data)) != 0) {
1126 sc->sc_drammodel = 1;
1127 return (0);
1128 }
1129
1130 return (0);
1131}
1132
1133#define HIFN_SRAM_MAX (32 << 20)
1134#define HIFN_SRAM_STEP_SIZE 16384
1135#define HIFN_SRAM_GRANULARITY (HIFN_SRAM_MAX / HIFN_SRAM_STEP_SIZE)
1136
1137static int
1138hifn_sramsize(struct hifn_softc *sc)
1139{
1140 u_int32_t a;
1141 u_int8_t data[8];
1142 u_int8_t dataexpect[sizeof(data)];
1143 int32_t i;
1144
1145 for (i = 0; i < sizeof(data); i++)
1146 data[i] = dataexpect[i] = i ^ 0x5a;
1147
1148 for (i = HIFN_SRAM_GRANULARITY - 1; i >= 0; i--) {
1149 a = i * HIFN_SRAM_STEP_SIZE;
1150 bcopy(&i, data, sizeof(i));
1151 hifn_writeramaddr(sc, a, data);
1152 }
1153
1154 for (i = 0; i < HIFN_SRAM_GRANULARITY; i++) {
1155 a = i * HIFN_SRAM_STEP_SIZE;
1156 bcopy(&i, dataexpect, sizeof(i));
1157 if (hifn_readramaddr(sc, a, data) < 0)
1158 return (0);
1159 if (bcmp(data, dataexpect, sizeof(data)) != 0)
1160 return (0);
1161 sc->sc_ramsize = a + HIFN_SRAM_STEP_SIZE;
1162 }
1163
1164 return (0);
1165}
1166
1167/*
1168 * XXX For dram boards, one should really try all of the
1169 * HIFN_PUCNFG_DSZ_*'s. This just assumes that PUCNFG
1170 * is already set up correctly.
1171 */
1172static int
1173hifn_dramsize(struct hifn_softc *sc)
1174{
1175 u_int32_t cnfg;
1176
1177 cnfg = READ_REG_0(sc, HIFN_0_PUCNFG) &
1178 HIFN_PUCNFG_DRAMMASK;
1179 sc->sc_ramsize = 1 << ((cnfg >> 13) + 18);
1180 return (0);
1181}
1182
1183static void
1184hifn_alloc_slot(struct hifn_softc *sc, int *cmdp, int *srcp, int *dstp, int *resp)
1185{
1186 struct hifn_dma *dma = sc->sc_dma;
1187
1188 if (dma->cmdi == HIFN_D_CMD_RSIZE) {
1189 dma->cmdi = 0;
1190 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
1191 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1192 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
1193 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1194 }
1195 *cmdp = dma->cmdi++;
1196 dma->cmdk = dma->cmdi;
1197
1198 if (dma->srci == HIFN_D_SRC_RSIZE) {
1199 dma->srci = 0;
1200 dma->srcr[HIFN_D_SRC_RSIZE].l = htole32(HIFN_D_VALID |
1201 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1202 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
1203 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1204 }
1205 *srcp = dma->srci++;
1206 dma->srck = dma->srci;
1207
1208 if (dma->dsti == HIFN_D_DST_RSIZE) {
1209 dma->dsti = 0;
1210 dma->dstr[HIFN_D_DST_RSIZE].l = htole32(HIFN_D_VALID |
1211 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1212 HIFN_DSTR_SYNC(sc, HIFN_D_DST_RSIZE,
1213 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1214 }
1215 *dstp = dma->dsti++;
1216 dma->dstk = dma->dsti;
1217
1218 if (dma->resi == HIFN_D_RES_RSIZE) {
1219 dma->resi = 0;
1220 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
1221 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1222 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
1223 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1224 }
1225 *resp = dma->resi++;
1226 dma->resk = dma->resi;
1227}
1228
1229static int
1230hifn_writeramaddr(struct hifn_softc *sc, int addr, u_int8_t *data)
1231{
1232 struct hifn_dma *dma = sc->sc_dma;
1233 hifn_base_command_t wc;
1234 const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
1235 int r, cmdi, resi, srci, dsti;
1236
1237 wc.masks = htole16(3 << 13);
1238 wc.session_num = htole16(addr >> 14);
1239 wc.total_source_count = htole16(8);
1240 wc.total_dest_count = htole16(addr & 0x3fff);
1241
1242 hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
1243
1244 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1245 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
1246 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
1247
1248 /* build write command */
1249 bzero(dma->command_bufs[cmdi], HIFN_MAX_COMMAND);
1250 *(hifn_base_command_t *)dma->command_bufs[cmdi] = wc;
1251 bcopy(data, &dma->test_src, sizeof(dma->test_src));
1252
1253 dma->srcr[srci].p = htole32(sc->sc_dma_physaddr
1254 + offsetof(struct hifn_dma, test_src));
1255 dma->dstr[dsti].p = htole32(sc->sc_dma_physaddr
1256 + offsetof(struct hifn_dma, test_dst));
1257
1258 dma->cmdr[cmdi].l = htole32(16 | masks);
1259 dma->srcr[srci].l = htole32(8 | masks);
1260 dma->dstr[dsti].l = htole32(4 | masks);
1261 dma->resr[resi].l = htole32(4 | masks);
1262
1263 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1264 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1265
1266 for (r = 10000; r >= 0; r--) {
1267 DELAY(10);
1268 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1269 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1270 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
1271 break;
1272 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1273 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1274 }
1275 if (r == 0) {
1276 device_printf(sc->sc_dev, "writeramaddr -- "
1277 "result[%d](addr %d) still valid\n", resi, addr);
1278 r = -1;
1279 return (-1);
1280 } else
1281 r = 0;
1282
1283 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1284 HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
1285 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
1286
1287 return (r);
1288}
1289
1290static int
1291hifn_readramaddr(struct hifn_softc *sc, int addr, u_int8_t *data)
1292{
1293 struct hifn_dma *dma = sc->sc_dma;
1294 hifn_base_command_t rc;
1295 const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
1296 int r, cmdi, srci, dsti, resi;
1297
1298 rc.masks = htole16(2 << 13);
1299 rc.session_num = htole16(addr >> 14);
1300 rc.total_source_count = htole16(addr & 0x3fff);
1301 rc.total_dest_count = htole16(8);
1302
1303 hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
1304
1305 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1306 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
1307 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
1308
1309 bzero(dma->command_bufs[cmdi], HIFN_MAX_COMMAND);
1310 *(hifn_base_command_t *)dma->command_bufs[cmdi] = rc;
1311
1312 dma->srcr[srci].p = htole32(sc->sc_dma_physaddr +
1313 offsetof(struct hifn_dma, test_src));
1314 dma->test_src = 0;
1315 dma->dstr[dsti].p = htole32(sc->sc_dma_physaddr +
1316 offsetof(struct hifn_dma, test_dst));
1317 dma->test_dst = 0;
1318 dma->cmdr[cmdi].l = htole32(8 | masks);
1319 dma->srcr[srci].l = htole32(8 | masks);
1320 dma->dstr[dsti].l = htole32(8 | masks);
1321 dma->resr[resi].l = htole32(HIFN_MAX_RESULT | masks);
1322
1323 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1324 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1325
1326 for (r = 10000; r >= 0; r--) {
1327 DELAY(10);
1328 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1329 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1330 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
1331 break;
1332 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1333 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1334 }
1335 if (r == 0) {
1336 device_printf(sc->sc_dev, "readramaddr -- "
1337 "result[%d](addr %d) still valid\n", resi, addr);
1338 r = -1;
1339 } else {
1340 r = 0;
1341 bcopy(&dma->test_dst, data, sizeof(dma->test_dst));
1342 }
1343
1344 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1345 HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
1346 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
1347
1348 return (r);
1349}
1350
1351/*
1352 * Initialize the descriptor rings.
1353 */
1354static void
1355hifn_init_dma(struct hifn_softc *sc)
1356{
1357 struct hifn_dma *dma = sc->sc_dma;
1358 int i;
1359
1360 hifn_set_retry(sc);
1361
1362 /* initialize static pointer values */
1363 for (i = 0; i < HIFN_D_CMD_RSIZE; i++)
1364 dma->cmdr[i].p = htole32(sc->sc_dma_physaddr +
1365 offsetof(struct hifn_dma, command_bufs[i][0]));
1366 for (i = 0; i < HIFN_D_RES_RSIZE; i++)
1367 dma->resr[i].p = htole32(sc->sc_dma_physaddr +
1368 offsetof(struct hifn_dma, result_bufs[i][0]));
1369
1370 dma->cmdr[HIFN_D_CMD_RSIZE].p =
1371 htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, cmdr[0]));
1372 dma->srcr[HIFN_D_SRC_RSIZE].p =
1373 htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, srcr[0]));
1374 dma->dstr[HIFN_D_DST_RSIZE].p =
1375 htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, dstr[0]));
1376 dma->resr[HIFN_D_RES_RSIZE].p =
1377 htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, resr[0]));
1378
1379 dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0;
1380 dma->cmdi = dma->srci = dma->dsti = dma->resi = 0;
1381 dma->cmdk = dma->srck = dma->dstk = dma->resk = 0;
1382}
1383
1384/*
1385 * Writes out the raw command buffer space. Returns the
1386 * command buffer size.
1387 */
1388static u_int
1389hifn_write_command(struct hifn_command *cmd, u_int8_t *buf)
1390{
1391 u_int8_t *buf_pos;
1392 hifn_base_command_t *base_cmd;
1393 hifn_mac_command_t *mac_cmd;
1394 hifn_crypt_command_t *cry_cmd;
1395 int using_mac, using_crypt, len;
1396 u_int32_t dlen, slen;
1397
1398 buf_pos = buf;
1399 using_mac = cmd->base_masks & HIFN_BASE_CMD_MAC;
1400 using_crypt = cmd->base_masks & HIFN_BASE_CMD_CRYPT;
1401
1402 base_cmd = (hifn_base_command_t *)buf_pos;
1403 base_cmd->masks = htole16(cmd->base_masks);
1404 slen = cmd->src_mapsize;
1405 if (cmd->sloplen)
1406 dlen = cmd->dst_mapsize - cmd->sloplen + sizeof(u_int32_t);
1407 else
1408 dlen = cmd->dst_mapsize;
1409 base_cmd->total_source_count = htole16(slen & HIFN_BASE_CMD_LENMASK_LO);
1410 base_cmd->total_dest_count = htole16(dlen & HIFN_BASE_CMD_LENMASK_LO);
1411 dlen >>= 16;
1412 slen >>= 16;
1413 base_cmd->session_num = htole16(cmd->session_num |
1414 ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) |
1415 ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M));
1416 buf_pos += sizeof(hifn_base_command_t);
1417
1418 if (using_mac) {
1419 mac_cmd = (hifn_mac_command_t *)buf_pos;
1420 dlen = cmd->maccrd->crd_len;
1421 mac_cmd->source_count = htole16(dlen & 0xffff);
1422 dlen >>= 16;
1423 mac_cmd->masks = htole16(cmd->mac_masks |
1424 ((dlen << HIFN_MAC_CMD_SRCLEN_S) & HIFN_MAC_CMD_SRCLEN_M));
1425 mac_cmd->header_skip = htole16(cmd->maccrd->crd_skip);
1426 mac_cmd->reserved = 0;
1427 buf_pos += sizeof(hifn_mac_command_t);
1428 }
1429
1430 if (using_crypt) {
1431 cry_cmd = (hifn_crypt_command_t *)buf_pos;
1432 dlen = cmd->enccrd->crd_len;
1433 cry_cmd->source_count = htole16(dlen & 0xffff);
1434 dlen >>= 16;
1435 cry_cmd->masks = htole16(cmd->cry_masks |
1436 ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & HIFN_CRYPT_CMD_SRCLEN_M));
1437 cry_cmd->header_skip = htole16(cmd->enccrd->crd_skip);
1438 cry_cmd->reserved = 0;
1439 buf_pos += sizeof(hifn_crypt_command_t);
1440 }
1441
1442 if (using_mac && cmd->mac_masks & HIFN_MAC_CMD_NEW_KEY) {
1443 bcopy(cmd->mac, buf_pos, HIFN_MAC_KEY_LENGTH);
1444 buf_pos += HIFN_MAC_KEY_LENGTH;
1445 }
1446
1447 if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_KEY) {
1448 switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) {
1449 case HIFN_CRYPT_CMD_ALG_3DES:
1450 bcopy(cmd->ck, buf_pos, HIFN_3DES_KEY_LENGTH);
1451 buf_pos += HIFN_3DES_KEY_LENGTH;
1452 break;
1453 case HIFN_CRYPT_CMD_ALG_DES:
1454 bcopy(cmd->ck, buf_pos, HIFN_DES_KEY_LENGTH);
1455 buf_pos += cmd->cklen;
1456 break;
1457 case HIFN_CRYPT_CMD_ALG_RC4:
1458 len = 256;
1459 do {
1460 int clen;
1461
1462 clen = MIN(cmd->cklen, len);
1463 bcopy(cmd->ck, buf_pos, clen);
1464 len -= clen;
1465 buf_pos += clen;
1466 } while (len > 0);
1467 bzero(buf_pos, 4);
1468 buf_pos += 4;
1469 break;
1470 }
1471 }
1472
1473 if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_IV) {
1474 bcopy(cmd->iv, buf_pos, HIFN_IV_LENGTH);
1475 buf_pos += HIFN_IV_LENGTH;
1476 }
1477
1478 if ((cmd->base_masks & (HIFN_BASE_CMD_MAC|HIFN_BASE_CMD_CRYPT)) == 0) {
1479 bzero(buf_pos, 8);
1480 buf_pos += 8;
1481 }
1482
1483 return (buf_pos - buf);
1484}
1485
1486static int
1487hifn_dmamap_aligned(struct hifn_operand *op)
1488{
1489 int i;
1490
1491 for (i = 0; i < op->nsegs; i++) {
1492 if (op->segs[i].ds_addr & 3)
1493 return (0);
1494 if ((i != (op->nsegs - 1)) && (op->segs[i].ds_len & 3))
1495 return (0);
1496 }
1497 return (1);
1498}
1499
1500static int
1501hifn_dmamap_load_dst(struct hifn_softc *sc, struct hifn_command *cmd)
1502{
1503 struct hifn_dma *dma = sc->sc_dma;
1504 struct hifn_operand *dst = &cmd->dst;
1505 u_int32_t p, l;
1506 int idx, used = 0, i;
1507
1508 idx = dma->dsti;
1509 for (i = 0; i < dst->nsegs - 1; i++) {
1510 dma->dstr[idx].p = htole32(dst->segs[i].ds_addr);
1511 dma->dstr[idx].l = htole32(HIFN_D_VALID |
1512 HIFN_D_MASKDONEIRQ | dst->segs[i].ds_len);
1513 HIFN_DSTR_SYNC(sc, idx,
1514 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1515 used++;
1516
1517 if (++idx == HIFN_D_DST_RSIZE) {
1518 dma->dstr[idx].l = htole32(HIFN_D_VALID |
1519 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1520 HIFN_DSTR_SYNC(sc, idx,
1521 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1522 idx = 0;
1523 }
1524 }
1525
1526 if (cmd->sloplen == 0) {
1527 p = dst->segs[i].ds_addr;
1528 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
1529 dst->segs[i].ds_len;
1530 } else {
1531 p = sc->sc_dma_physaddr +
1532 offsetof(struct hifn_dma, slop[cmd->slopidx]);
1533 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
1534 sizeof(u_int32_t);
1535
1536 if ((dst->segs[i].ds_len - cmd->sloplen) != 0) {
1537 dma->dstr[idx].p = htole32(dst->segs[i].ds_addr);
1538 dma->dstr[idx].l = htole32(HIFN_D_VALID |
1539 HIFN_D_MASKDONEIRQ |
1540 (dst->segs[i].ds_len - cmd->sloplen));
1541 HIFN_DSTR_SYNC(sc, idx,
1542 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1543 used++;
1544
1545 if (++idx == HIFN_D_DST_RSIZE) {
1546 dma->dstr[idx].l = htole32(HIFN_D_VALID |
1547 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1548 HIFN_DSTR_SYNC(sc, idx,
1549 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1550 idx = 0;
1551 }
1552 }
1553 }
1554 dma->dstr[idx].p = htole32(p);
1555 dma->dstr[idx].l = htole32(l);
1556 HIFN_DSTR_SYNC(sc, idx, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1557 used++;
1558
1559 if (++idx == HIFN_D_DST_RSIZE) {
1560 dma->dstr[idx].l = htole32(HIFN_D_VALID | HIFN_D_JUMP |
1561 HIFN_D_MASKDONEIRQ);
1562 HIFN_DSTR_SYNC(sc, idx,
1563 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1564 idx = 0;
1565 }
1566
1567 dma->dsti = idx;
1568 dma->dstu += used;
1569 return (idx);
1570}
1571
1572static int
1573hifn_dmamap_load_src(struct hifn_softc *sc, struct hifn_command *cmd)
1574{
1575 struct hifn_dma *dma = sc->sc_dma;
1576 struct hifn_operand *src = &cmd->src;
1577 int idx, i;
1578 u_int32_t last = 0;
1579
1580 idx = dma->srci;
1581 for (i = 0; i < src->nsegs; i++) {
1582 if (i == src->nsegs - 1)
1583 last = HIFN_D_LAST;
1584
1585 dma->srcr[idx].p = htole32(src->segs[i].ds_addr);
1586 dma->srcr[idx].l = htole32(src->segs[i].ds_len |
1587 HIFN_D_VALID | HIFN_D_MASKDONEIRQ | last);
1588 HIFN_SRCR_SYNC(sc, idx,
1589 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1590
1591 if (++idx == HIFN_D_SRC_RSIZE) {
1592 dma->srcr[idx].l = htole32(HIFN_D_VALID |
1593 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1594 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
1595 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1596 idx = 0;
1597 }
1598 }
1599 dma->srci = idx;
1600 dma->srcu += src->nsegs;
1601 return (idx);
1602}
1603
1604static void
1605hifn_op_cb(void* arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
1606{
1607 struct hifn_operand *op = arg;
1608
1609 KASSERT(nsegs <= MAX_SCATTER,
1610 ("hifn_op_cb: too many DMA segments (%u > %u) "
1611 "returned when mapping operand", nsegs, MAX_SCATTER));
1612 op->mapsize = mapsize;
1613 op->nsegs = nsegs;
1614 bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
1615}
1616
1617static int
1618hifn_crypto(
1619 struct hifn_softc *sc,
1620 struct hifn_command *cmd,
1621 struct cryptop *crp,
1622 int hint)
1623{
1624 struct hifn_dma *dma = sc->sc_dma;
1625 u_int32_t cmdlen;
1626 int cmdi, resi, err = 0;
1627
1628 /*
1629 * need 1 cmd, and 1 res
1630 *
1631 * NB: check this first since it's easy.
1632 */
1633 HIFN_LOCK(sc);
1634 if ((dma->cmdu + 1) > HIFN_D_CMD_RSIZE ||
1635 (dma->resu + 1) > HIFN_D_RES_RSIZE) {
1636#ifdef HIFN_DEBUG
1637 if (hifn_debug) {
1638 device_printf(sc->sc_dev,
1639 "cmd/result exhaustion, cmdu %u resu %u\n",
1640 dma->cmdu, dma->resu);
1641 }
1642#endif
1643 hifnstats.hst_nomem_cr++;
1644 HIFN_UNLOCK(sc);
1645 return (ERESTART);
1646 }
1647
1648 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->src_map)) {
1649 hifnstats.hst_nomem_map++;
1650 HIFN_UNLOCK(sc);
1651 return (ENOMEM);
1652 }
1653
1654 if (crp->crp_flags & CRYPTO_F_IMBUF) {
1655 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map,
1656 cmd->src_m, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
1657 hifnstats.hst_nomem_load++;
1658 err = ENOMEM;
1659 goto err_srcmap1;
1660 }
1661 } else if (crp->crp_flags & CRYPTO_F_IOV) {
1662 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map,
1663 cmd->src_io, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
1664 hifnstats.hst_nomem_load++;
1665 err = ENOMEM;
1666 goto err_srcmap1;
1667 }
1668 } else {
1669 err = EINVAL;
1670 goto err_srcmap1;
1671 }
1672
1673 if (hifn_dmamap_aligned(&cmd->src)) {
1674 cmd->sloplen = cmd->src_mapsize & 3;
1675 cmd->dst = cmd->src;
1676 } else {
1677 if (crp->crp_flags & CRYPTO_F_IOV) {
1678 err = EINVAL;
1679 goto err_srcmap;
1680 } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
1681 int totlen, len;
1682 struct mbuf *m, *m0, *mlast;
1683
1684 KASSERT(cmd->dst_m == cmd->src_m,
1685 ("hifn_crypto: dst_m initialized improperly"));
1686 hifnstats.hst_unaligned++;
1687 /*
1688 * Source is not aligned on a longword boundary.
1689 * Copy the data to insure alignment. If we fail
1690 * to allocate mbufs or clusters while doing this
1691 * we return ERESTART so the operation is requeued
1692 * at the crypto later, but only if there are
1693 * ops already posted to the hardware; otherwise we
1694 * have no guarantee that we'll be re-entered.
1695 */
1696 totlen = cmd->src_mapsize;
1697 if (cmd->src_m->m_flags & M_PKTHDR) {
1698 len = MHLEN;
1699 MGETHDR(m0, M_DONTWAIT, MT_DATA);
1700 if (m0 && !m_dup_pkthdr(m0, cmd->src_m, M_DONTWAIT)) {
1701 m_free(m0);
1702 m0 = NULL;
1703 }
1704 } else {
1705 len = MLEN;
1706 MGET(m0, M_DONTWAIT, MT_DATA);
1707 }
1708 if (m0 == NULL) {
1709 hifnstats.hst_nomem_mbuf++;
1710 err = dma->cmdu ? ERESTART : ENOMEM;
1711 goto err_srcmap;
1712 }
1713 if (totlen >= MINCLSIZE) {
1714 MCLGET(m0, M_DONTWAIT);
1715 if ((m0->m_flags & M_EXT) == 0) {
1716 hifnstats.hst_nomem_mcl++;
1717 err = dma->cmdu ? ERESTART : ENOMEM;
1718 m_freem(m0);
1719 goto err_srcmap;
1720 }
1721 len = MCLBYTES;
1722 }
1723 totlen -= len;
1724 m0->m_pkthdr.len = m0->m_len = len;
1725 mlast = m0;
1726
1727 while (totlen > 0) {
1728 MGET(m, M_DONTWAIT, MT_DATA);
1729 if (m == NULL) {
1730 hifnstats.hst_nomem_mbuf++;
1731 err = dma->cmdu ? ERESTART : ENOMEM;
1732 m_freem(m0);
1733 goto err_srcmap;
1734 }
1735 len = MLEN;
1736 if (totlen >= MINCLSIZE) {
1737 MCLGET(m, M_DONTWAIT);
1738 if ((m->m_flags & M_EXT) == 0) {
1739 hifnstats.hst_nomem_mcl++;
1740 err = dma->cmdu ? ERESTART : ENOMEM;
1741 mlast->m_next = m;
1742 m_freem(m0);
1743 goto err_srcmap;
1744 }
1745 len = MCLBYTES;
1746 }
1747
1748 m->m_len = len;
1749 m0->m_pkthdr.len += len;
1750 totlen -= len;
1751
1752 mlast->m_next = m;
1753 mlast = m;
1754 }
1755 cmd->dst_m = m0;
1756 }
1757 }
1758
1759 if (cmd->dst_map == NULL) {
1760 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->dst_map)) {
1761 hifnstats.hst_nomem_map++;
1762 err = ENOMEM;
1763 goto err_srcmap;
1764 }
1765 if (crp->crp_flags & CRYPTO_F_IMBUF) {
1766 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map,
1767 cmd->dst_m, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
1768 hifnstats.hst_nomem_map++;
1769 err = ENOMEM;
1770 goto err_dstmap1;
1771 }
1772 } else if (crp->crp_flags & CRYPTO_F_IOV) {
1773 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map,
1774 cmd->dst_io, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
1775 hifnstats.hst_nomem_load++;
1776 err = ENOMEM;
1777 goto err_dstmap1;
1778 }
1779 }
1780 }
1781
1782#ifdef HIFN_DEBUG
1783 if (hifn_debug) {
1784 device_printf(sc->sc_dev,
1785 "Entering cmd: stat %8x ien %8x u %d/%d/%d/%d n %d/%d\n",
1786 READ_REG_1(sc, HIFN_1_DMA_CSR),
1787 READ_REG_1(sc, HIFN_1_DMA_IER),
1788 dma->cmdu, dma->srcu, dma->dstu, dma->resu,
1789 cmd->src_nsegs, cmd->dst_nsegs);
1790 }
1791#endif
1792
1793 if (cmd->src_map == cmd->dst_map) {
1794 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
1795 BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
1796 } else {
1797 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
1798 BUS_DMASYNC_PREWRITE);
1799 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
1800 BUS_DMASYNC_PREREAD);
1801 }
1802
1803 /*
1804 * need N src, and N dst
1805 */
1806 if ((dma->srcu + cmd->src_nsegs) > HIFN_D_SRC_RSIZE ||
1807 (dma->dstu + cmd->dst_nsegs + 1) > HIFN_D_DST_RSIZE) {
1808#ifdef HIFN_DEBUG
1809 if (hifn_debug) {
1810 device_printf(sc->sc_dev,
1811 "src/dst exhaustion, srcu %u+%u dstu %u+%u\n",
1812 dma->srcu, cmd->src_nsegs,
1813 dma->dstu, cmd->dst_nsegs);
1814 }
1815#endif
1816 hifnstats.hst_nomem_sd++;
1817 err = ERESTART;
1818 goto err_dstmap;
1819 }
1820
1821 if (dma->cmdi == HIFN_D_CMD_RSIZE) {
1822 dma->cmdi = 0;
1823 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
1824 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1825 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
1826 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1827 }
1828 cmdi = dma->cmdi++;
1829 cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]);
1830 HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE);
1831
1832 /* .p for command/result already set */
1833 dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST |
1834 HIFN_D_MASKDONEIRQ);
1835 HIFN_CMDR_SYNC(sc, cmdi,
1836 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1837 dma->cmdu++;
1838 if (sc->sc_c_busy == 0) {
1839 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA);
1840 sc->sc_c_busy = 1;
1841 }
1842
1843 /*
1844 * We don't worry about missing an interrupt (which a "command wait"
1845 * interrupt salvages us from), unless there is more than one command
1846 * in the queue.
1847 */
1848 if (dma->cmdu > 1) {
1849 sc->sc_dmaier |= HIFN_DMAIER_C_WAIT;
1850 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
1851 }
1852
1853 hifnstats.hst_ipackets++;
1854 hifnstats.hst_ibytes += cmd->src_mapsize;
1855
1856 hifn_dmamap_load_src(sc, cmd);
1857 if (sc->sc_s_busy == 0) {
1858 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
1859 sc->sc_s_busy = 1;
1860 }
1861
1862 /*
1863 * Unlike other descriptors, we don't mask done interrupt from
1864 * result descriptor.
1865 */
1866#ifdef HIFN_DEBUG
1867 if (hifn_debug)
1868 printf("load res\n");
1869#endif
1870 if (dma->resi == HIFN_D_RES_RSIZE) {
1871 dma->resi = 0;
1872 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
1873 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1874 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
1875 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1876 }
1877 resi = dma->resi++;
1878 KASSERT(dma->hifn_commands[resi] == NULL,
1879 ("hifn_crypto: command slot %u busy", resi));
1880 dma->hifn_commands[resi] = cmd;
1881 HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD);
1882 if ((hint & CRYPTO_HINT_MORE) && sc->sc_curbatch < hifn_maxbatch) {
1883 dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
1884 HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ);
1885 sc->sc_curbatch++;
1886 if (sc->sc_curbatch > hifnstats.hst_maxbatch)
1887 hifnstats.hst_maxbatch = sc->sc_curbatch;
1888 hifnstats.hst_totbatch++;
1889 } else {
1890 dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
1891 HIFN_D_VALID | HIFN_D_LAST);
1892 sc->sc_curbatch = 0;
1893 }
1894 HIFN_RESR_SYNC(sc, resi,
1895 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1896 dma->resu++;
1897 if (sc->sc_r_busy == 0) {
1898 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
1899 sc->sc_r_busy = 1;
1900 }
1901
1902 if (cmd->sloplen)
1903 cmd->slopidx = resi;
1904
1905 hifn_dmamap_load_dst(sc, cmd);
1906
1907 if (sc->sc_d_busy == 0) {
1908 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
1909 sc->sc_d_busy = 1;
1910 }
1911
1912#ifdef HIFN_DEBUG
1913 if (hifn_debug) {
1914 device_printf(sc->sc_dev, "command: stat %8x ier %8x\n",
1915 READ_REG_1(sc, HIFN_1_DMA_CSR),
1916 READ_REG_1(sc, HIFN_1_DMA_IER));
1917 }
1918#endif
1919
1920 sc->sc_active = 5;
1921 HIFN_UNLOCK(sc);
1922 KASSERT(err == 0, ("hifn_crypto: success with error %u", err));
1923 return (err); /* success */
1924
1925err_dstmap:
1926 if (cmd->src_map != cmd->dst_map)
1927 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
1928err_dstmap1:
1929 if (cmd->src_map != cmd->dst_map)
1930 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
1931err_srcmap:
1932 if (crp->crp_flags & CRYPTO_F_IMBUF) {
1933 if (cmd->src_m != cmd->dst_m)
1934 m_freem(cmd->dst_m);
1935 }
1936 bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
1937err_srcmap1:
1938 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
1939 HIFN_UNLOCK(sc);
1940 return (err);
1941}
1942
1943static void
1944hifn_tick(void* vsc)
1945{
1946 struct hifn_softc *sc = vsc;
1947
1948 HIFN_LOCK(sc);
1949 if (sc->sc_active == 0) {
1950 struct hifn_dma *dma = sc->sc_dma;
1951 u_int32_t r = 0;
1952
1953 if (dma->cmdu == 0 && sc->sc_c_busy) {
1954 sc->sc_c_busy = 0;
1955 r |= HIFN_DMACSR_C_CTRL_DIS;
1956 }
1957 if (dma->srcu == 0 && sc->sc_s_busy) {
1958 sc->sc_s_busy = 0;
1959 r |= HIFN_DMACSR_S_CTRL_DIS;
1960 }
1961 if (dma->dstu == 0 && sc->sc_d_busy) {
1962 sc->sc_d_busy = 0;
1963 r |= HIFN_DMACSR_D_CTRL_DIS;
1964 }
1965 if (dma->resu == 0 && sc->sc_r_busy) {
1966 sc->sc_r_busy = 0;
1967 r |= HIFN_DMACSR_R_CTRL_DIS;
1968 }
1969 if (r)
1970 WRITE_REG_1(sc, HIFN_1_DMA_CSR, r);
1971 } else
1972 sc->sc_active--;
1973 HIFN_UNLOCK(sc);
1974 callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
1975}
1976
1977static void
1978hifn_intr(void *arg)
1979{
1980 struct hifn_softc *sc = arg;
1981 struct hifn_dma *dma;
1982 u_int32_t dmacsr, restart;
1983 int i, u;
1984
1985 dmacsr = READ_REG_1(sc, HIFN_1_DMA_CSR);
1986
1987 /* Nothing in the DMA unit interrupted */
1988 if ((dmacsr & sc->sc_dmaier) == 0)
1989 return;
1990
1991 HIFN_LOCK(sc);
1992
1993 dma = sc->sc_dma;
1994
1995#ifdef HIFN_DEBUG
1996 if (hifn_debug) {
1997 device_printf(sc->sc_dev,
1998 "irq: stat %08x ien %08x damier %08x i %d/%d/%d/%d k %d/%d/%d/%d u %d/%d/%d/%d\n",
1999 dmacsr, READ_REG_1(sc, HIFN_1_DMA_IER), sc->sc_dmaier,
2000 dma->cmdi, dma->srci, dma->dsti, dma->resi,
2001 dma->cmdk, dma->srck, dma->dstk, dma->resk,
2002 dma->cmdu, dma->srcu, dma->dstu, dma->resu);
2003 }
2004#endif
2005
2006 WRITE_REG_1(sc, HIFN_1_DMA_CSR, dmacsr & sc->sc_dmaier);
2007
2008 if ((sc->sc_flags & HIFN_HAS_PUBLIC) &&
2009 (dmacsr & HIFN_DMACSR_PUBDONE))
2010 WRITE_REG_1(sc, HIFN_1_PUB_STATUS,
2011 READ_REG_1(sc, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);
2012
2013 restart = dmacsr & (HIFN_DMACSR_D_OVER | HIFN_DMACSR_R_OVER);
2014 if (restart)
2015 device_printf(sc->sc_dev, "overrun %x\n", dmacsr);
2016
2017 if (sc->sc_flags & HIFN_IS_7811) {
2018 if (dmacsr & HIFN_DMACSR_ILLR)
2019 device_printf(sc->sc_dev, "illegal read\n");
2020 if (dmacsr & HIFN_DMACSR_ILLW)
2021 device_printf(sc->sc_dev, "illegal write\n");
2022 }
2023
2024 restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |
2025 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);
2026 if (restart) {
2027 device_printf(sc->sc_dev, "abort, resetting.\n");
2028 hifnstats.hst_abort++;
2029 hifn_abort(sc);
2030 HIFN_UNLOCK(sc);
2031 return;
2032 }
2033
2034 if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) {
2035 /*
2036 * If no slots to process and we receive a "waiting on
2037 * command" interrupt, we disable the "waiting on command"
2038 * (by clearing it).
2039 */
2040 sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
2041 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
2042 }
2043
2044 /* clear the rings */
2045 i = dma->resk; u = dma->resu;
2046 while (u != 0) {
2047 HIFN_RESR_SYNC(sc, i,
2048 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2049 if (dma->resr[i].l & htole32(HIFN_D_VALID)) {
2050 HIFN_RESR_SYNC(sc, i,
2051 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2052 break;
2053 }
2054
2055 if (i != HIFN_D_RES_RSIZE) {
2056 struct hifn_command *cmd;
2057 u_int8_t *macbuf = NULL;
2058
2059 HIFN_RES_SYNC(sc, i, BUS_DMASYNC_POSTREAD);
2060 cmd = dma->hifn_commands[i];
2061 KASSERT(cmd != NULL,
2062 ("hifn_intr: null command slot %u", i));
2063 dma->hifn_commands[i] = NULL;
2064
2065 if (cmd->base_masks & HIFN_BASE_CMD_MAC) {
2066 macbuf = dma->result_bufs[i];
2067 macbuf += 12;
2068 }
2069
2070 hifn_callback(sc, cmd, macbuf);
2071 hifnstats.hst_opackets++;
2072 u--;
2073 }
2074
2075 if (++i == (HIFN_D_RES_RSIZE + 1))
2076 i = 0;
2077 }
2078 dma->resk = i; dma->resu = u;
2079
2080 i = dma->srck; u = dma->srcu;
2081 while (u != 0) {
2082 if (i == HIFN_D_SRC_RSIZE)
2083 i = 0;
2084 HIFN_SRCR_SYNC(sc, i,
2085 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2086 if (dma->srcr[i].l & htole32(HIFN_D_VALID)) {
2087 HIFN_SRCR_SYNC(sc, i,
2088 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2089 break;
2090 }
2091 i++, u--;
2092 }
2093 dma->srck = i; dma->srcu = u;
2094
2095 i = dma->cmdk; u = dma->cmdu;
2096 while (u != 0) {
2097 HIFN_CMDR_SYNC(sc, i,
2098 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2099 if (dma->cmdr[i].l & htole32(HIFN_D_VALID)) {
2100 HIFN_CMDR_SYNC(sc, i,
2101 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2102 break;
2103 }
2104 if (i != HIFN_D_CMD_RSIZE) {
2105 u--;
2106 HIFN_CMD_SYNC(sc, i, BUS_DMASYNC_POSTWRITE);
2107 }
2108 if (++i == (HIFN_D_CMD_RSIZE + 1))
2109 i = 0;
2110 }
2111 dma->cmdk = i; dma->cmdu = u;
2112
2113 HIFN_UNLOCK(sc);
2114
2115 if (sc->sc_needwakeup) { /* XXX check high watermark */
2116 int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
2117#ifdef HIFN_DEBUG
2118 if (hifn_debug)
2119 device_printf(sc->sc_dev,
2120 "wakeup crypto (%x) u %d/%d/%d/%d\n",
2121 sc->sc_needwakeup,
2122 dma->cmdu, dma->srcu, dma->dstu, dma->resu);
2123#endif
2124 sc->sc_needwakeup &= ~wakeup;
2125 crypto_unblock(sc->sc_cid, wakeup);
2126 }
2127}
2128
2129/*
2130 * Allocate a new 'session' and return an encoded session id. 'sidp'
2131 * contains our registration id, and should contain an encoded session
2132 * id on successful allocation.
2133 */
2134static int
2135hifn_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri)
2136{
2137 struct cryptoini *c;
2138 struct hifn_softc *sc = arg;
2139 int i, mac = 0, cry = 0;
2140
2141 KASSERT(sc != NULL, ("hifn_newsession: null softc"));
2142 if (sidp == NULL || cri == NULL || sc == NULL)
2143 return (EINVAL);
2144
2145 for (i = 0; i < sc->sc_maxses; i++)
2146 if (sc->sc_sessions[i].hs_state == HS_STATE_FREE)
2147 break;
2148 if (i == sc->sc_maxses)
2149 return (ENOMEM);
2150
2151 for (c = cri; c != NULL; c = c->cri_next) {
2152 switch (c->cri_alg) {
2153 case CRYPTO_MD5:
2154 case CRYPTO_SHA1:
2155 case CRYPTO_MD5_HMAC:
2156 case CRYPTO_SHA1_HMAC:
2157 if (mac)
2158 return (EINVAL);
2159 mac = 1;
2160 break;
2161 case CRYPTO_DES_CBC:
2162 case CRYPTO_3DES_CBC:
2163 /* XXX this may read fewer, does it matter? */
2164 read_random(sc->sc_sessions[i].hs_iv, HIFN_IV_LENGTH);
2165 /*FALLTHROUGH*/
2166 case CRYPTO_ARC4:
2167 if (cry)
2168 return (EINVAL);
2169 cry = 1;
2170 break;
2171 default:
2172 return (EINVAL);
2173 }
2174 }
2175 if (mac == 0 && cry == 0)
2176 return (EINVAL);
2177
2178 *sidp = HIFN_SID(device_get_unit(sc->sc_dev), i);
2179 sc->sc_sessions[i].hs_state = HS_STATE_USED;
2180
2181 return (0);
2182}
2183
2184/*
2185 * Deallocate a session.
2186 * XXX this routine should run a zero'd mac/encrypt key into context ram.
2187 * XXX to blow away any keys already stored there.
2188 */
2189static int
2190hifn_freesession(void *arg, u_int64_t tid)
2191{
2192 struct hifn_softc *sc = arg;
2193 int session;
2194 u_int32_t sid = CRYPTO_SESID2LID(tid);
2195
2196 KASSERT(sc != NULL, ("hifn_freesession: null softc"));
2197 if (sc == NULL)
2198 return (EINVAL);
2199
2200 session = HIFN_SESSION(sid);
2201 if (session >= sc->sc_maxses)
2202 return (EINVAL);
2203
2204 bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session]));
2205 return (0);
2206}
2207
2208static int
2209hifn_process(void *arg, struct cryptop *crp, int hint)
2210{
2211 struct hifn_softc *sc = arg;
2212 struct hifn_command *cmd = NULL;
2213 int session, err;
2214 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
2215
2216 if (crp == NULL || crp->crp_callback == NULL) {
2217 hifnstats.hst_invalid++;
2218 return (EINVAL);
2219 }
2220 session = HIFN_SESSION(crp->crp_sid);
2221
2222 if (sc == NULL || session >= sc->sc_maxses) {
2223 err = EINVAL;
2224 goto errout;
2225 }
2226
2227 cmd = malloc(sizeof(struct hifn_command), M_DEVBUF, M_NOWAIT | M_ZERO);
2228 if (cmd == NULL) {
2229 hifnstats.hst_nomem++;
2230 err = ENOMEM;
2231 goto errout;
2232 }
2233
2234 if (crp->crp_flags & CRYPTO_F_IMBUF) {
2235 cmd->src_m = (struct mbuf *)crp->crp_buf;
2236 cmd->dst_m = (struct mbuf *)crp->crp_buf;
2237 } else if (crp->crp_flags & CRYPTO_F_IOV) {
2238 cmd->src_io = (struct uio *)crp->crp_buf;
2239 cmd->dst_io = (struct uio *)crp->crp_buf;
2240 } else {
2241 err = EINVAL;
2242 goto errout; /* XXX we don't handle contiguous buffers! */
2243 }
2244
2245 crd1 = crp->crp_desc;
2246 if (crd1 == NULL) {
2247 err = EINVAL;
2248 goto errout;
2249 }
2250 crd2 = crd1->crd_next;
2251
2252 if (crd2 == NULL) {
2253 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
2254 crd1->crd_alg == CRYPTO_SHA1_HMAC ||
2255 crd1->crd_alg == CRYPTO_SHA1 ||
2256 crd1->crd_alg == CRYPTO_MD5) {
2257 maccrd = crd1;
2258 enccrd = NULL;
2259 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
2260 crd1->crd_alg == CRYPTO_3DES_CBC ||
2261 crd1->crd_alg == CRYPTO_ARC4) {
2262 if ((crd1->crd_flags & CRD_F_ENCRYPT) == 0)
2263 cmd->base_masks |= HIFN_BASE_CMD_DECODE;
2264 maccrd = NULL;
2265 enccrd = crd1;
2266 } else {
2267 err = EINVAL;
2268 goto errout;
2269 }
2270 } else {
2271 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
2272 crd1->crd_alg == CRYPTO_SHA1_HMAC ||
2273 crd1->crd_alg == CRYPTO_MD5 ||
2274 crd1->crd_alg == CRYPTO_SHA1) &&
2275 (crd2->crd_alg == CRYPTO_DES_CBC ||
2276 crd2->crd_alg == CRYPTO_3DES_CBC ||
2277 crd2->crd_alg == CRYPTO_ARC4) &&
2278 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
2279 cmd->base_masks = HIFN_BASE_CMD_DECODE;
2280 maccrd = crd1;
2281 enccrd = crd2;
2282 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
2283 crd1->crd_alg == CRYPTO_ARC4 ||
2284 crd1->crd_alg == CRYPTO_3DES_CBC) &&
2285 (crd2->crd_alg == CRYPTO_MD5_HMAC ||
2286 crd2->crd_alg == CRYPTO_SHA1_HMAC ||
2287 crd2->crd_alg == CRYPTO_MD5 ||
2288 crd2->crd_alg == CRYPTO_SHA1) &&
2289 (crd1->crd_flags & CRD_F_ENCRYPT)) {
2290 enccrd = crd1;
2291 maccrd = crd2;
2292 } else {
2293 /*
2294 * We cannot order the 7751 as requested
2295 */
2296 err = EINVAL;
2297 goto errout;
2298 }
2299 }
2300
2301 if (enccrd) {
2302 cmd->enccrd = enccrd;
2303 cmd->base_masks |= HIFN_BASE_CMD_CRYPT;
2304 switch (enccrd->crd_alg) {
2305 case CRYPTO_ARC4:
2306 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_RC4;
2307 if ((enccrd->crd_flags & CRD_F_ENCRYPT)
2308 != sc->sc_sessions[session].hs_prev_op)
2309 sc->sc_sessions[session].hs_state =
2310 HS_STATE_USED;
2311 break;
2312 case CRYPTO_DES_CBC:
2313 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_DES |
2314 HIFN_CRYPT_CMD_MODE_CBC |
2315 HIFN_CRYPT_CMD_NEW_IV;
2316 break;
2317 case CRYPTO_3DES_CBC:
2318 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_3DES |
2319 HIFN_CRYPT_CMD_MODE_CBC |
2320 HIFN_CRYPT_CMD_NEW_IV;
2321 break;
2322 default:
2323 err = EINVAL;
2324 goto errout;
2325 }
2326 if (enccrd->crd_alg != CRYPTO_ARC4) {
2327 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
2328 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
2329 bcopy(enccrd->crd_iv, cmd->iv,
2330 HIFN_IV_LENGTH);
2331 else
2332 bcopy(sc->sc_sessions[session].hs_iv,
2333 cmd->iv, HIFN_IV_LENGTH);
2334
2335 if ((enccrd->crd_flags & CRD_F_IV_PRESENT)
2336 == 0) {
2337 if (crp->crp_flags & CRYPTO_F_IMBUF)
2338 m_copyback(cmd->src_m,
2339 enccrd->crd_inject,
2340 HIFN_IV_LENGTH, cmd->iv);
2341 else if (crp->crp_flags & CRYPTO_F_IOV)
2342 cuio_copyback(cmd->src_io,
2343 enccrd->crd_inject,
2344 HIFN_IV_LENGTH, cmd->iv);
2345 }
2346 } else {
2347 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
2348 bcopy(enccrd->crd_iv, cmd->iv,
2349 HIFN_IV_LENGTH);
2350 else if (crp->crp_flags & CRYPTO_F_IMBUF)
2351 m_copydata(cmd->src_m,
2352 enccrd->crd_inject,
2353 HIFN_IV_LENGTH, cmd->iv);
2354 else if (crp->crp_flags & CRYPTO_F_IOV)
2355 cuio_copydata(cmd->src_io,
2356 enccrd->crd_inject,
2357 HIFN_IV_LENGTH, cmd->iv);
2358 }
2359 }
2360
2361 cmd->ck = enccrd->crd_key;
2362 cmd->cklen = enccrd->crd_klen >> 3;
2363
2364 if (sc->sc_sessions[session].hs_state == HS_STATE_USED)
2365 cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
2366 }
2367
2368 if (maccrd) {
2369 cmd->maccrd = maccrd;
2370 cmd->base_masks |= HIFN_BASE_CMD_MAC;
2371
2372 switch (maccrd->crd_alg) {
2373 case CRYPTO_MD5:
2374 cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
2375 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
2376 HIFN_MAC_CMD_POS_IPSEC;
2377 break;
2378 case CRYPTO_MD5_HMAC:
2379 cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
2380 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
2381 HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
2382 break;
2383 case CRYPTO_SHA1:
2384 cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
2385 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
2386 HIFN_MAC_CMD_POS_IPSEC;
2387 break;
2388 case CRYPTO_SHA1_HMAC:
2389 cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
2390 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
2391 HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
2392 break;
2393 }
2394
2395 if ((maccrd->crd_alg == CRYPTO_SHA1_HMAC ||
2396 maccrd->crd_alg == CRYPTO_MD5_HMAC) &&
2397 sc->sc_sessions[session].hs_state == HS_STATE_USED) {
2398 cmd->mac_masks |= HIFN_MAC_CMD_NEW_KEY;
2399 bcopy(maccrd->crd_key, cmd->mac, maccrd->crd_klen >> 3);
2400 bzero(cmd->mac + (maccrd->crd_klen >> 3),
2401 HIFN_MAC_KEY_LENGTH - (maccrd->crd_klen >> 3));
2402 }
2403 }
2404
2405 cmd->crp = crp;
2406 cmd->session_num = session;
2407 cmd->softc = sc;
2408
2409 err = hifn_crypto(sc, cmd, crp, hint);
2410 if (!err) {
2411 if (enccrd)
2412 sc->sc_sessions[session].hs_prev_op =
2413 enccrd->crd_flags & CRD_F_ENCRYPT;
2414 if (sc->sc_sessions[session].hs_state == HS_STATE_USED)
2415 sc->sc_sessions[session].hs_state = HS_STATE_KEY;
2416 return 0;
2417 } else if (err == ERESTART) {
2418 /*
2419 * There weren't enough resources to dispatch the request
2420 * to the part. Notify the caller so they'll requeue this
2421 * request and resubmit it again soon.
2422 */
2423#ifdef HIFN_DEBUG
2424 if (hifn_debug)
2425 device_printf(sc->sc_dev, "requeue request\n");
2426#endif
2427 free(cmd, M_DEVBUF);
2428 sc->sc_needwakeup |= CRYPTO_SYMQ;
2429 return (err);
2430 }
2431
2432errout:
2433 if (cmd != NULL)
2434 free(cmd, M_DEVBUF);
2435 if (err == EINVAL)
2436 hifnstats.hst_invalid++;
2437 else
2438 hifnstats.hst_nomem++;
2439 crp->crp_etype = err;
2440 crypto_done(crp);
2441 return (err);
2442}
2443
2444static void
2445hifn_abort(struct hifn_softc *sc)
2446{
2447 struct hifn_dma *dma = sc->sc_dma;
2448 struct hifn_command *cmd;
2449 struct cryptop *crp;
2450 int i, u;
2451
2452 i = dma->resk; u = dma->resu;
2453 while (u != 0) {
2454 cmd = dma->hifn_commands[i];
2455 KASSERT(cmd != NULL, ("hifn_abort: null command slot %u", i));
2456 dma->hifn_commands[i] = NULL;
2457 crp = cmd->crp;
2458
2459 if ((dma->resr[i].l & htole32(HIFN_D_VALID)) == 0) {
2460 /* Salvage what we can. */
2461 u_int8_t *macbuf;
2462
2463 if (cmd->base_masks & HIFN_BASE_CMD_MAC) {
2464 macbuf = dma->result_bufs[i];
2465 macbuf += 12;
2466 } else
2467 macbuf = NULL;
2468 hifnstats.hst_opackets++;
2469 hifn_callback(sc, cmd, macbuf);
2470 } else {
2471 if (cmd->src_map == cmd->dst_map) {
2472 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2473 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
2474 } else {
2475 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2476 BUS_DMASYNC_POSTWRITE);
2477 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2478 BUS_DMASYNC_POSTREAD);
2479 }
2480
2481 if (cmd->src_m != cmd->dst_m) {
2482 m_freem(cmd->src_m);
2483 crp->crp_buf = (caddr_t)cmd->dst_m;
2484 }
2485
2486 /* non-shared buffers cannot be restarted */
2487 if (cmd->src_map != cmd->dst_map) {
2488 /*
2489 * XXX should be EAGAIN, delayed until
2490 * after the reset.
2491 */
2492 crp->crp_etype = ENOMEM;
2493 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2494 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
2495 } else
2496 crp->crp_etype = ENOMEM;
2497
2498 bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2499 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
2500
2501 free(cmd, M_DEVBUF);
2502 if (crp->crp_etype != EAGAIN)
2503 crypto_done(crp);
2504 }
2505
2506 if (++i == HIFN_D_RES_RSIZE)
2507 i = 0;
2508 u--;
2509 }
2510 dma->resk = i; dma->resu = u;
2511
2512 /* Force upload of key next time */
2513 for (i = 0; i < sc->sc_maxses; i++)
2514 if (sc->sc_sessions[i].hs_state == HS_STATE_KEY)
2515 sc->sc_sessions[i].hs_state = HS_STATE_USED;
2516
2517 hifn_reset_board(sc, 1);
2518 hifn_init_dma(sc);
2519 hifn_init_pci_registers(sc);
2520}
2521
2522static void
2523hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *macbuf)
2524{
2525 struct hifn_dma *dma = sc->sc_dma;
2526 struct cryptop *crp = cmd->crp;
2527 struct cryptodesc *crd;
2528 struct mbuf *m;
2529 int totlen, i, u;
2530
2531 if (cmd->src_map == cmd->dst_map) {
2532 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2533 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
2534 } else {
2535 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2536 BUS_DMASYNC_POSTWRITE);
2537 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2538 BUS_DMASYNC_POSTREAD);
2539 }
2540
2541 if (crp->crp_flags & CRYPTO_F_IMBUF) {
2542 if (cmd->src_m != cmd->dst_m) {
2543 crp->crp_buf = (caddr_t)cmd->dst_m;
2544 totlen = cmd->src_mapsize;
2545 for (m = cmd->dst_m; m != NULL; m = m->m_next) {
2546 if (totlen < m->m_len) {
2547 m->m_len = totlen;
2548 totlen = 0;
2549 } else
2550 totlen -= m->m_len;
2551 }
2552 cmd->dst_m->m_pkthdr.len = cmd->src_m->m_pkthdr.len;
2553 m_freem(cmd->src_m);
2554 }
2555 }
2556
2557 if (cmd->sloplen != 0) {
2558 if (crp->crp_flags & CRYPTO_F_IMBUF)
2559 m_copyback((struct mbuf *)crp->crp_buf,
2560 cmd->src_mapsize - cmd->sloplen,
2561 cmd->sloplen, (caddr_t)&dma->slop[cmd->slopidx]);
2562 else if (crp->crp_flags & CRYPTO_F_IOV)
2563 cuio_copyback((struct uio *)crp->crp_buf,
2564 cmd->src_mapsize - cmd->sloplen,
2565 cmd->sloplen, (caddr_t)&dma->slop[cmd->slopidx]);
2566 }
2567
2568 i = dma->dstk; u = dma->dstu;
2569 while (u != 0) {
2570 if (i == HIFN_D_DST_RSIZE)
2571 i = 0;
2572 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
2573 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2574 if (dma->dstr[i].l & htole32(HIFN_D_VALID)) {
2575 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
2576 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2577 break;
2578 }
2579 i++, u--;
2580 }
2581 dma->dstk = i; dma->dstu = u;
2582
2583 hifnstats.hst_obytes += cmd->dst_mapsize;
2584
2585 if ((cmd->base_masks & (HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE)) ==
2586 HIFN_BASE_CMD_CRYPT) {
2587 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
2588 if (crd->crd_alg != CRYPTO_DES_CBC &&
2589 crd->crd_alg != CRYPTO_3DES_CBC)
2590 continue;
2591 if (crp->crp_flags & CRYPTO_F_IMBUF)
2592 m_copydata((struct mbuf *)crp->crp_buf,
2593 crd->crd_skip + crd->crd_len - HIFN_IV_LENGTH,
2594 HIFN_IV_LENGTH,
2595 cmd->softc->sc_sessions[cmd->session_num].hs_iv);
2596 else if (crp->crp_flags & CRYPTO_F_IOV) {
2597 cuio_copydata((struct uio *)crp->crp_buf,
2598 crd->crd_skip + crd->crd_len - HIFN_IV_LENGTH,
2599 HIFN_IV_LENGTH,
2600 cmd->softc->sc_sessions[cmd->session_num].hs_iv);
2601 }
2602 break;
2603 }
2604 }
2605
2606 if (macbuf != NULL) {
2607 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
2608 int len;
2609
2610 if (crd->crd_alg == CRYPTO_MD5)
2611 len = 16;
2612 else if (crd->crd_alg == CRYPTO_SHA1)
2613 len = 20;
2614 else if (crd->crd_alg == CRYPTO_MD5_HMAC ||
2615 crd->crd_alg == CRYPTO_SHA1_HMAC)
2616 len = 12;
2617 else
2618 continue;
2619
2620 if (crp->crp_flags & CRYPTO_F_IMBUF)
2621 m_copyback((struct mbuf *)crp->crp_buf,
2622 crd->crd_inject, len, macbuf);
2623 else if ((crp->crp_flags & CRYPTO_F_IOV) && crp->crp_mac)
2624 bcopy((caddr_t)macbuf, crp->crp_mac, len);
2625 break;
2626 }
2627 }
2628
2629 if (cmd->src_map != cmd->dst_map) {
2630 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2631 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
2632 }
2633 bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2634 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
2635 free(cmd, M_DEVBUF);
2636 crypto_done(crp);
2637}
2638
2639/*
2640 * 7811 PB3 rev/2 parts lock-up on burst writes to Group 0
2641 * and Group 1 registers; avoid conditions that could create
2642 * burst writes by doing a read in between the writes.
2643 *
2644 * NB: The read we interpose is always to the same register;
2645 * we do this because reading from an arbitrary (e.g. last)
2646 * register may not always work.
2647 */
2648static void
2649hifn_write_reg_0(struct hifn_softc *sc, bus_size_t reg, u_int32_t val)
2650{
2651 if (sc->sc_flags & HIFN_IS_7811) {
2652 if (sc->sc_bar0_lastreg == reg - 4)
2653 bus_space_read_4(sc->sc_st0, sc->sc_sh0, HIFN_0_PUCNFG);
2654 sc->sc_bar0_lastreg = reg;
2655 }
2656 bus_space_write_4(sc->sc_st0, sc->sc_sh0, reg, val);
2657}
2658
2659static void
2660hifn_write_reg_1(struct hifn_softc *sc, bus_size_t reg, u_int32_t val)
2661{
2662 if (sc->sc_flags & HIFN_IS_7811) {
2663 if (sc->sc_bar1_lastreg == reg - 4)
2664 bus_space_read_4(sc->sc_st1, sc->sc_sh1, HIFN_1_REVID);
2665 sc->sc_bar1_lastreg = reg;
2666 }
2667 bus_space_write_4(sc->sc_st1, sc->sc_sh1, reg, val);
2668}