sec.c revision 193579
1193579Sraj/*- 2193579Sraj * Copyright (C) 2008-2009 Semihalf, Piotr Ziecik 3193579Sraj * All rights reserved. 4193579Sraj * 5193579Sraj * Redistribution and use in source and binary forms, with or without 6193579Sraj * modification, are permitted provided that the following conditions 7193579Sraj * are met: 8193579Sraj * 1. Redistributions of source code must retain the above copyright 9193579Sraj * notice, this list of conditions and the following disclaimer. 10193579Sraj * 2. Redistributions in binary form must reproduce the above copyright 11193579Sraj * notice, this list of conditions and the following disclaimer in the 12193579Sraj * documentation and/or other materials provided with the distribution. 13193579Sraj * 14193579Sraj * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15193579Sraj * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16193579Sraj * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 17193579Sraj * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 18193579Sraj * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 19193579Sraj * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 20193579Sraj * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 21193579Sraj * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 22193579Sraj * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23193579Sraj * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24193579Sraj */ 25193579Sraj 26193579Sraj/* 27193579Sraj * Freescale integrated Security Engine (SEC) driver. Currently SEC 2.0 and 28193579Sraj * 3.0 are supported. 29193579Sraj */ 30193579Sraj 31193579Sraj#include <sys/cdefs.h> 32193579Sraj__FBSDID("$FreeBSD: head/sys/dev/sec/sec.c 193579 2009-06-06 09:37:55Z raj $"); 33193579Sraj 34193579Sraj#include <sys/param.h> 35193579Sraj#include <sys/systm.h> 36193579Sraj#include <sys/bus.h> 37193579Sraj#include <sys/endian.h> 38193579Sraj#include <sys/kernel.h> 39193579Sraj#include <sys/lock.h> 40193579Sraj#include <sys/malloc.h> 41193579Sraj#include <sys/mbuf.h> 42193579Sraj#include <sys/module.h> 43193579Sraj#include <sys/mutex.h> 44193579Sraj#include <sys/random.h> 45193579Sraj#include <sys/rman.h> 46193579Sraj 47193579Sraj#include <machine/bus.h> 48193579Sraj#include <machine/ocpbus.h> 49193579Sraj#include <machine/resource.h> 50193579Sraj 51193579Sraj#include <opencrypto/cryptodev.h> 52193579Sraj#include "cryptodev_if.h" 53193579Sraj 54193579Sraj#include <dev/sec/sec.h> 55193579Sraj 56193579Srajstatic int sec_probe(device_t dev); 57193579Srajstatic int sec_attach(device_t dev); 58193579Srajstatic int sec_detach(device_t dev); 59193579Srajstatic int sec_suspend(device_t dev); 60193579Srajstatic int sec_resume(device_t dev); 61193579Srajstatic void sec_shutdown(device_t dev); 62193579Srajstatic void sec_primary_intr(void *arg); 63193579Srajstatic void sec_secondary_intr(void *arg); 64193579Srajstatic int sec_setup_intr(struct sec_softc *sc, struct resource **ires, 65193579Sraj void **ihand, int *irid, driver_intr_t handler, const char *iname); 66193579Srajstatic void sec_release_intr(struct sec_softc *sc, struct resource *ires, 67193579Sraj void *ihand, int irid, const char *iname); 68193579Srajstatic int sec_controller_reset(struct sec_softc *sc); 69193579Srajstatic int sec_channel_reset(struct sec_softc *sc, int channel, int full); 70193579Srajstatic int sec_init(struct sec_softc *sc); 71193579Srajstatic int sec_alloc_dma_mem(struct sec_softc *sc, 72193579Sraj struct sec_dma_mem *dma_mem, bus_size_t size); 73193579Srajstatic int sec_desc_map_dma(struct sec_softc *sc, 74193579Sraj struct sec_dma_mem *dma_mem, void *mem, bus_size_t size, int type, 75193579Sraj struct sec_desc_map_info *sdmi); 76193579Srajstatic void sec_free_dma_mem(struct sec_dma_mem *dma_mem); 77193579Srajstatic void sec_enqueue(struct sec_softc *sc); 78193579Srajstatic int sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, 79193579Sraj int channel); 80193579Srajstatic int sec_eu_channel(struct sec_softc *sc, int eu); 81193579Srajstatic int sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, 82193579Sraj u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype); 83193579Srajstatic int sec_make_pointer_direct(struct sec_softc *sc, 84193579Sraj struct sec_desc *desc, u_int n, bus_addr_t data, bus_size_t dsize); 85193579Srajstatic int sec_alloc_session(struct sec_softc *sc); 86193579Srajstatic int sec_newsession(device_t dev, u_int32_t *sidp, 87193579Sraj struct cryptoini *cri); 88193579Srajstatic int sec_freesession(device_t dev, uint64_t tid); 89193579Srajstatic int sec_process(device_t dev, struct cryptop *crp, int hint); 90193579Srajstatic int sec_split_cri(struct cryptoini *cri, struct cryptoini **enc, 91193579Sraj struct cryptoini **mac); 92193579Srajstatic int sec_split_crp(struct cryptop *crp, struct cryptodesc **enc, 93193579Sraj struct cryptodesc **mac); 94193579Srajstatic int sec_build_common_ns_desc(struct sec_softc *sc, 95193579Sraj struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp, 96193579Sraj struct cryptodesc *enc, int buftype); 97193579Srajstatic int sec_build_common_s_desc(struct sec_softc *sc, 98193579Sraj struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp, 99193579Sraj struct cryptodesc *enc, struct cryptodesc *mac, int buftype); 100193579Sraj 101193579Srajstatic struct sec_session *sec_get_session(struct sec_softc *sc, u_int sid); 102193579Srajstatic struct sec_desc *sec_find_desc(struct sec_softc *sc, bus_addr_t paddr); 103193579Sraj 104193579Sraj/* AESU */ 105193579Srajstatic int sec_aesu_newsession(struct sec_softc *sc, 106193579Sraj struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac); 107193579Srajstatic int sec_aesu_make_desc(struct sec_softc *sc, 108193579Sraj struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp, 109193579Sraj int buftype); 110193579Sraj 111193579Sraj/* DEU */ 112193579Srajstatic int sec_deu_newsession(struct sec_softc *sc, 113193579Sraj struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac); 114193579Srajstatic int sec_deu_make_desc(struct sec_softc *sc, 115193579Sraj struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp, 116193579Sraj int buftype); 117193579Sraj 118193579Sraj/* MDEU */ 119193579Srajstatic int sec_mdeu_can_handle(u_int alg); 120193579Srajstatic int sec_mdeu_config(struct cryptodesc *crd, 121193579Sraj u_int *eu, u_int *mode, u_int *hashlen); 122193579Srajstatic int sec_mdeu_newsession(struct sec_softc *sc, 123193579Sraj struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac); 124193579Srajstatic int sec_mdeu_make_desc(struct sec_softc *sc, 125193579Sraj struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp, 126193579Sraj int buftype); 127193579Sraj 128193579Srajstatic device_method_t sec_methods[] = { 129193579Sraj /* Device interface */ 130193579Sraj DEVMETHOD(device_probe, sec_probe), 131193579Sraj DEVMETHOD(device_attach, sec_attach), 132193579Sraj DEVMETHOD(device_detach, sec_detach), 133193579Sraj 134193579Sraj DEVMETHOD(device_suspend, sec_suspend), 135193579Sraj DEVMETHOD(device_resume, sec_resume), 136193579Sraj DEVMETHOD(device_shutdown, sec_shutdown), 137193579Sraj 138193579Sraj /* Bus interface */ 139193579Sraj DEVMETHOD(bus_print_child, bus_generic_print_child), 140193579Sraj DEVMETHOD(bus_driver_added, bus_generic_driver_added), 141193579Sraj 142193579Sraj /* Crypto methods */ 143193579Sraj DEVMETHOD(cryptodev_newsession, sec_newsession), 144193579Sraj DEVMETHOD(cryptodev_freesession,sec_freesession), 145193579Sraj DEVMETHOD(cryptodev_process, sec_process), 146193579Sraj 147193579Sraj { 0, 0 } 148193579Sraj}; 149193579Srajstatic driver_t sec_driver = { 150193579Sraj "sec", 151193579Sraj sec_methods, 152193579Sraj sizeof(struct sec_softc), 153193579Sraj}; 154193579Sraj 155193579Srajstatic devclass_t sec_devclass; 156193579SrajDRIVER_MODULE(sec, ocpbus, sec_driver, sec_devclass, 0, 0); 157193579SrajMODULE_DEPEND(sec, crypto, 1, 1, 1); 158193579Sraj 159193579Srajstatic struct sec_eu_methods sec_eus[] = { 160193579Sraj { 161193579Sraj sec_aesu_newsession, 162193579Sraj sec_aesu_make_desc, 163193579Sraj }, 164193579Sraj { 165193579Sraj sec_deu_newsession, 166193579Sraj sec_deu_make_desc, 167193579Sraj }, 168193579Sraj { 169193579Sraj sec_mdeu_newsession, 170193579Sraj sec_mdeu_make_desc, 171193579Sraj }, 172193579Sraj { NULL, NULL } 173193579Sraj}; 174193579Sraj 175193579Srajstatic inline void 176193579Srajsec_sync_dma_mem(struct sec_dma_mem *dma_mem, bus_dmasync_op_t op) 177193579Sraj{ 178193579Sraj 179193579Sraj /* Sync only if dma memory is valid */ 180193579Sraj if (dma_mem->dma_vaddr != NULL) 181193579Sraj bus_dmamap_sync(dma_mem->dma_tag, dma_mem->dma_map, op); 182193579Sraj} 183193579Sraj 184193579Srajstatic inline void 185193579Srajsec_free_session(struct sec_softc *sc, struct sec_session *ses) 186193579Sraj{ 187193579Sraj 188193579Sraj SEC_LOCK(sc, sessions); 189193579Sraj ses->ss_used = 0; 190193579Sraj SEC_UNLOCK(sc, sessions); 191193579Sraj} 192193579Sraj 193193579Srajstatic inline void * 194193579Srajsec_get_pointer_data(struct sec_desc *desc, u_int n) 195193579Sraj{ 196193579Sraj 197193579Sraj return (desc->sd_ptr_dmem[n].dma_vaddr); 198193579Sraj} 199193579Sraj 200193579Srajstatic int 201193579Srajsec_probe(device_t dev) 202193579Sraj{ 203193579Sraj struct sec_softc *sc; 204193579Sraj device_t parent; 205193579Sraj uintptr_t devtype; 206193579Sraj uint64_t id; 207193579Sraj int error; 208193579Sraj 209193579Sraj parent = device_get_parent(dev); 210193579Sraj error = BUS_READ_IVAR(parent, dev, OCPBUS_IVAR_DEVTYPE, &devtype); 211193579Sraj if (error) 212193579Sraj return (error); 213193579Sraj 214193579Sraj if (devtype != OCPBUS_DEVTYPE_SEC) 215193579Sraj return (ENXIO); 216193579Sraj 217193579Sraj sc = device_get_softc(dev); 218193579Sraj 219193579Sraj sc->sc_rrid = 0; 220193579Sraj sc->sc_rres = bus_alloc_resource(dev, SYS_RES_MEMORY, &sc->sc_rrid, 221193579Sraj 0ul, ~0ul, SEC_IO_SIZE, RF_ACTIVE); 222193579Sraj 223193579Sraj if (sc->sc_rres == NULL) 224193579Sraj return (ENXIO); 225193579Sraj 226193579Sraj sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres); 227193579Sraj sc->sc_bas.bst = rman_get_bustag(sc->sc_rres); 228193579Sraj 229193579Sraj id = SEC_READ(sc, SEC_ID); 230193579Sraj 231193579Sraj bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); 232193579Sraj 233193579Sraj switch (id) { 234193579Sraj case SEC_20_ID: 235193579Sraj device_set_desc(dev, "Freescale Security Engine 2.0"); 236193579Sraj sc->sc_version = 2; 237193579Sraj break; 238193579Sraj case SEC_30_ID: 239193579Sraj device_set_desc(dev, "Freescale Security Engine 3.0"); 240193579Sraj sc->sc_version = 3; 241193579Sraj break; 242193579Sraj default: 243193579Sraj device_printf(dev, "unknown SEC ID 0x%016llx!\n", id); 244193579Sraj return (ENXIO); 245193579Sraj } 246193579Sraj 247193579Sraj return (0); 248193579Sraj} 249193579Sraj 250193579Srajstatic int 251193579Srajsec_attach(device_t dev) 252193579Sraj{ 253193579Sraj struct sec_softc *sc; 254193579Sraj struct sec_hw_lt *lt; 255193579Sraj int error = 0; 256193579Sraj int i; 257193579Sraj 258193579Sraj sc = device_get_softc(dev); 259193579Sraj sc->sc_dev = dev; 260193579Sraj sc->sc_blocked = 0; 261193579Sraj sc->sc_shutdown = 0; 262193579Sraj 263193579Sraj sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); 264193579Sraj if (sc->sc_cid < 0) { 265193579Sraj device_printf(dev, "could not get crypto driver ID!\n"); 266193579Sraj return (ENXIO); 267193579Sraj } 268193579Sraj 269193579Sraj /* Init locks */ 270193579Sraj mtx_init(&sc->sc_controller_lock, device_get_nameunit(dev), 271193579Sraj "SEC Controller lock", MTX_DEF); 272193579Sraj mtx_init(&sc->sc_descriptors_lock, device_get_nameunit(dev), 273193579Sraj "SEC Descriptors lock", MTX_DEF); 274193579Sraj mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev), 275193579Sraj "SEC Sessions lock", MTX_DEF); 276193579Sraj 277193579Sraj /* Allocate I/O memory for SEC registers */ 278193579Sraj sc->sc_rrid = 0; 279193579Sraj sc->sc_rres = bus_alloc_resource(dev, SYS_RES_MEMORY, &sc->sc_rrid, 280193579Sraj 0ul, ~0ul, SEC_IO_SIZE, RF_ACTIVE); 281193579Sraj 282193579Sraj if (sc->sc_rres == NULL) { 283193579Sraj device_printf(dev, "could not allocate I/O memory!\n"); 284193579Sraj goto fail1; 285193579Sraj } 286193579Sraj 287193579Sraj sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres); 288193579Sraj sc->sc_bas.bst = rman_get_bustag(sc->sc_rres); 289193579Sraj 290193579Sraj /* Setup interrupts */ 291193579Sraj sc->sc_pri_irid = 0; 292193579Sraj error = sec_setup_intr(sc, &sc->sc_pri_ires, &sc->sc_pri_ihand, 293193579Sraj &sc->sc_pri_irid, sec_primary_intr, "primary"); 294193579Sraj 295193579Sraj if (error) 296193579Sraj goto fail2; 297193579Sraj 298193579Sraj sc->sc_sec_irid = 1; 299193579Sraj error = sec_setup_intr(sc, &sc->sc_sec_ires, &sc->sc_sec_ihand, 300193579Sraj &sc->sc_sec_irid, sec_secondary_intr, "secondary"); 301193579Sraj 302193579Sraj if (error) 303193579Sraj goto fail3; 304193579Sraj 305193579Sraj /* Alloc DMA memory for descriptors and link tables */ 306193579Sraj error = sec_alloc_dma_mem(sc, &(sc->sc_desc_dmem), 307193579Sraj SEC_DESCRIPTORS * sizeof(struct sec_hw_desc)); 308193579Sraj 309193579Sraj if (error) 310193579Sraj goto fail4; 311193579Sraj 312193579Sraj error = sec_alloc_dma_mem(sc, &(sc->sc_lt_dmem), 313193579Sraj (SEC_LT_ENTRIES + 1) * sizeof(struct sec_hw_lt)); 314193579Sraj 315193579Sraj if (error) 316193579Sraj goto fail5; 317193579Sraj 318193579Sraj /* Fill in descriptors and link tables */ 319193579Sraj for (i = 0; i < SEC_DESCRIPTORS; i++) { 320193579Sraj sc->sc_desc[i].sd_desc = 321193579Sraj (struct sec_hw_desc*)(sc->sc_desc_dmem.dma_vaddr) + i; 322193579Sraj sc->sc_desc[i].sd_desc_paddr = sc->sc_desc_dmem.dma_paddr + 323193579Sraj (i * sizeof(struct sec_hw_desc)); 324193579Sraj } 325193579Sraj 326193579Sraj for (i = 0; i < SEC_LT_ENTRIES + 1; i++) { 327193579Sraj sc->sc_lt[i].sl_lt = 328193579Sraj (struct sec_hw_lt*)(sc->sc_lt_dmem.dma_vaddr) + i; 329193579Sraj sc->sc_lt[i].sl_lt_paddr = sc->sc_lt_dmem.dma_paddr + 330193579Sraj (i * sizeof(struct sec_hw_lt)); 331193579Sraj } 332193579Sraj 333193579Sraj /* Last entry in link table is used to create a circle */ 334193579Sraj lt = sc->sc_lt[SEC_LT_ENTRIES].sl_lt; 335193579Sraj lt->shl_length = 0; 336193579Sraj lt->shl_r = 0; 337193579Sraj lt->shl_n = 1; 338193579Sraj lt->shl_ptr = sc->sc_lt[0].sl_lt_paddr; 339193579Sraj 340193579Sraj /* Init descriptor and link table queues pointers */ 341193579Sraj SEC_CNT_INIT(sc, sc_free_desc_get_cnt, SEC_DESCRIPTORS); 342193579Sraj SEC_CNT_INIT(sc, sc_free_desc_put_cnt, SEC_DESCRIPTORS); 343193579Sraj SEC_CNT_INIT(sc, sc_ready_desc_get_cnt, SEC_DESCRIPTORS); 344193579Sraj SEC_CNT_INIT(sc, sc_ready_desc_put_cnt, SEC_DESCRIPTORS); 345193579Sraj SEC_CNT_INIT(sc, sc_queued_desc_get_cnt, SEC_DESCRIPTORS); 346193579Sraj SEC_CNT_INIT(sc, sc_queued_desc_put_cnt, SEC_DESCRIPTORS); 347193579Sraj SEC_CNT_INIT(sc, sc_lt_alloc_cnt, SEC_LT_ENTRIES); 348193579Sraj SEC_CNT_INIT(sc, sc_lt_free_cnt, SEC_LT_ENTRIES); 349193579Sraj 350193579Sraj /* Create masks for fast checks */ 351193579Sraj sc->sc_int_error_mask = 0; 352193579Sraj for (i = 0; i < SEC_CHANNELS; i++) 353193579Sraj sc->sc_int_error_mask |= (~0ULL & SEC_INT_CH_ERR(i)); 354193579Sraj 355193579Sraj switch (sc->sc_version) { 356193579Sraj case 2: 357193579Sraj sc->sc_channel_idle_mask = 358193579Sraj (SEC_CHAN_CSR2_FFLVL_M << SEC_CHAN_CSR2_FFLVL_S) | 359193579Sraj (SEC_CHAN_CSR2_MSTATE_M << SEC_CHAN_CSR2_MSTATE_S) | 360193579Sraj (SEC_CHAN_CSR2_PSTATE_M << SEC_CHAN_CSR2_PSTATE_S) | 361193579Sraj (SEC_CHAN_CSR2_GSTATE_M << SEC_CHAN_CSR2_GSTATE_S); 362193579Sraj break; 363193579Sraj case 3: 364193579Sraj sc->sc_channel_idle_mask = 365193579Sraj (SEC_CHAN_CSR3_FFLVL_M << SEC_CHAN_CSR3_FFLVL_S) | 366193579Sraj (SEC_CHAN_CSR3_MSTATE_M << SEC_CHAN_CSR3_MSTATE_S) | 367193579Sraj (SEC_CHAN_CSR3_PSTATE_M << SEC_CHAN_CSR3_PSTATE_S) | 368193579Sraj (SEC_CHAN_CSR3_GSTATE_M << SEC_CHAN_CSR3_GSTATE_S); 369193579Sraj break; 370193579Sraj } 371193579Sraj 372193579Sraj /* Init hardware */ 373193579Sraj error = sec_init(sc); 374193579Sraj 375193579Sraj if (error) 376193579Sraj goto fail6; 377193579Sraj 378193579Sraj /* Register in OCF (AESU) */ 379193579Sraj crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); 380193579Sraj 381193579Sraj /* Register in OCF (DEU) */ 382193579Sraj crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); 383193579Sraj crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); 384193579Sraj 385193579Sraj /* Register in OCF (MDEU) */ 386193579Sraj crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0); 387193579Sraj crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 388193579Sraj crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0); 389193579Sraj crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 390193579Sraj crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0); 391193579Sraj if (sc->sc_version >= 3) { 392193579Sraj crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0); 393193579Sraj crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0); 394193579Sraj } 395193579Sraj 396193579Sraj return (0); 397193579Sraj 398193579Srajfail6: 399193579Sraj sec_free_dma_mem(&(sc->sc_lt_dmem)); 400193579Srajfail5: 401193579Sraj sec_free_dma_mem(&(sc->sc_desc_dmem)); 402193579Srajfail4: 403193579Sraj sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 404193579Sraj sc->sc_sec_irid, "secondary"); 405193579Srajfail3: 406193579Sraj sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 407193579Sraj sc->sc_pri_irid, "primary"); 408193579Srajfail2: 409193579Sraj bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); 410193579Srajfail1: 411193579Sraj mtx_destroy(&sc->sc_controller_lock); 412193579Sraj mtx_destroy(&sc->sc_descriptors_lock); 413193579Sraj mtx_destroy(&sc->sc_sessions_lock); 414193579Sraj 415193579Sraj return (ENXIO); 416193579Sraj} 417193579Sraj 418193579Srajstatic int 419193579Srajsec_detach(device_t dev) 420193579Sraj{ 421193579Sraj struct sec_softc *sc = device_get_softc(dev); 422193579Sraj int i, error, timeout = SEC_TIMEOUT; 423193579Sraj 424193579Sraj /* Prepare driver to shutdown */ 425193579Sraj SEC_LOCK(sc, descriptors); 426193579Sraj sc->sc_shutdown = 1; 427193579Sraj SEC_UNLOCK(sc, descriptors); 428193579Sraj 429193579Sraj /* Wait until all queued processing finishes */ 430193579Sraj while (1) { 431193579Sraj SEC_LOCK(sc, descriptors); 432193579Sraj i = SEC_READY_DESC_CNT(sc) + SEC_QUEUED_DESC_CNT(sc); 433193579Sraj SEC_UNLOCK(sc, descriptors); 434193579Sraj 435193579Sraj if (i == 0) 436193579Sraj break; 437193579Sraj 438193579Sraj if (timeout < 0) { 439193579Sraj device_printf(dev, "queue flush timeout!\n"); 440193579Sraj 441193579Sraj /* DMA can be still active - stop it */ 442193579Sraj for (i = 0; i < SEC_CHANNELS; i++) 443193579Sraj sec_channel_reset(sc, i, 1); 444193579Sraj 445193579Sraj break; 446193579Sraj } 447193579Sraj 448193579Sraj timeout -= 1000; 449193579Sraj DELAY(1000); 450193579Sraj } 451193579Sraj 452193579Sraj /* Disable interrupts */ 453193579Sraj SEC_WRITE(sc, SEC_IER, 0); 454193579Sraj 455193579Sraj /* Unregister from OCF */ 456193579Sraj crypto_unregister_all(sc->sc_cid); 457193579Sraj 458193579Sraj /* Free DMA memory */ 459193579Sraj for (i = 0; i < SEC_DESCRIPTORS; i++) 460193579Sraj SEC_DESC_FREE_POINTERS(&(sc->sc_desc[i])); 461193579Sraj 462193579Sraj sec_free_dma_mem(&(sc->sc_lt_dmem)); 463193579Sraj sec_free_dma_mem(&(sc->sc_desc_dmem)); 464193579Sraj 465193579Sraj /* Release interrupts */ 466193579Sraj sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 467193579Sraj sc->sc_pri_irid, "primary"); 468193579Sraj sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 469193579Sraj sc->sc_sec_irid, "secondary"); 470193579Sraj 471193579Sraj /* Release memory */ 472193579Sraj if (sc->sc_rres) { 473193579Sraj error = bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, 474193579Sraj sc->sc_rres); 475193579Sraj if (error) 476193579Sraj device_printf(dev, "bus_release_resource() failed for" 477193579Sraj " I/O memory, error %d\n", error); 478193579Sraj 479193579Sraj sc->sc_rres = NULL; 480193579Sraj } 481193579Sraj 482193579Sraj mtx_destroy(&sc->sc_controller_lock); 483193579Sraj mtx_destroy(&sc->sc_descriptors_lock); 484193579Sraj mtx_destroy(&sc->sc_sessions_lock); 485193579Sraj 486193579Sraj return (0); 487193579Sraj} 488193579Sraj 489193579Srajstatic int 490193579Srajsec_suspend(device_t dev) 491193579Sraj{ 492193579Sraj 493193579Sraj return (0); 494193579Sraj} 495193579Sraj 496193579Srajstatic int 497193579Srajsec_resume(device_t dev) 498193579Sraj{ 499193579Sraj 500193579Sraj return (0); 501193579Sraj} 502193579Sraj 503193579Srajstatic void 504193579Srajsec_shutdown(device_t dev) 505193579Sraj{ 506193579Sraj} 507193579Sraj 508193579Srajstatic int 509193579Srajsec_setup_intr(struct sec_softc *sc, struct resource **ires, void **ihand, 510193579Sraj int *irid, driver_intr_t handler, const char *iname) 511193579Sraj{ 512193579Sraj int error; 513193579Sraj 514193579Sraj (*ires) = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ, irid, 515193579Sraj RF_ACTIVE); 516193579Sraj 517193579Sraj if ((*ires) == NULL) { 518193579Sraj device_printf(sc->sc_dev, "could not allocate %s IRQ\n", iname); 519193579Sraj return (ENXIO); 520193579Sraj } 521193579Sraj 522193579Sraj error = bus_setup_intr(sc->sc_dev, *ires, INTR_MPSAFE | INTR_TYPE_NET, 523193579Sraj NULL, handler, sc, ihand); 524193579Sraj 525193579Sraj if (error) { 526193579Sraj device_printf(sc->sc_dev, "failed to set up %s IRQ\n", iname); 527193579Sraj if (bus_release_resource(sc->sc_dev, SYS_RES_IRQ, *irid, *ires)) 528193579Sraj device_printf(sc->sc_dev, "could not release %s IRQ\n", 529193579Sraj iname); 530193579Sraj 531193579Sraj (*ires) = NULL; 532193579Sraj return (error); 533193579Sraj } 534193579Sraj 535193579Sraj return (0); 536193579Sraj} 537193579Sraj 538193579Srajstatic void 539193579Srajsec_release_intr(struct sec_softc *sc, struct resource *ires, void *ihand, 540193579Sraj int irid, const char *iname) 541193579Sraj{ 542193579Sraj int error; 543193579Sraj 544193579Sraj if (ires == NULL) 545193579Sraj return; 546193579Sraj 547193579Sraj error = bus_teardown_intr(sc->sc_dev, ires, ihand); 548193579Sraj if (error) 549193579Sraj device_printf(sc->sc_dev, "bus_teardown_intr() failed for %s" 550193579Sraj " IRQ, error %d\n", iname, error); 551193579Sraj 552193579Sraj error = bus_release_resource(sc->sc_dev, SYS_RES_IRQ, irid, ires); 553193579Sraj if (error) 554193579Sraj device_printf(sc->sc_dev, "bus_release_resource() failed for %s" 555193579Sraj " IRQ, error %d\n", iname, error); 556193579Sraj} 557193579Sraj 558193579Srajstatic void 559193579Srajsec_primary_intr(void *arg) 560193579Sraj{ 561193579Sraj struct sec_softc *sc = arg; 562193579Sraj struct sec_desc *desc; 563193579Sraj uint64_t isr; 564193579Sraj int i, wakeup = 0; 565193579Sraj 566193579Sraj SEC_LOCK(sc, controller); 567193579Sraj 568193579Sraj /* Check for errors */ 569193579Sraj isr = SEC_READ(sc, SEC_ISR); 570193579Sraj if (isr & sc->sc_int_error_mask) { 571193579Sraj /* Check each channel for error */ 572193579Sraj for (i = 0; i < SEC_CHANNELS; i++) { 573193579Sraj if ((isr & SEC_INT_CH_ERR(i)) == 0) 574193579Sraj continue; 575193579Sraj 576193579Sraj device_printf(sc->sc_dev, 577193579Sraj "I/O error on channel %i!\n", i); 578193579Sraj 579193579Sraj /* Find and mark problematic descriptor */ 580193579Sraj desc = sec_find_desc(sc, SEC_READ(sc, 581193579Sraj SEC_CHAN_CDPR(i))); 582193579Sraj 583193579Sraj if (desc != NULL) 584193579Sraj desc->sd_error = EIO; 585193579Sraj 586193579Sraj /* Do partial channel reset */ 587193579Sraj sec_channel_reset(sc, i, 0); 588193579Sraj } 589193579Sraj } 590193579Sraj 591193579Sraj /* ACK interrupt */ 592193579Sraj SEC_WRITE(sc, SEC_ICR, 0xFFFFFFFFFFFFFFFFULL); 593193579Sraj 594193579Sraj SEC_UNLOCK(sc, controller); 595193579Sraj SEC_LOCK(sc, descriptors); 596193579Sraj 597193579Sraj /* Handle processed descriptors */ 598193579Sraj SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 599193579Sraj 600193579Sraj while (SEC_QUEUED_DESC_CNT(sc) > 0) { 601193579Sraj desc = SEC_GET_QUEUED_DESC(sc); 602193579Sraj 603193579Sraj if (desc->sd_desc->shd_done != 0xFF && desc->sd_error == 0) { 604193579Sraj SEC_PUT_BACK_QUEUED_DESC(sc); 605193579Sraj break; 606193579Sraj } 607193579Sraj 608193579Sraj SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_PREREAD | 609193579Sraj BUS_DMASYNC_PREWRITE); 610193579Sraj 611193579Sraj desc->sd_crp->crp_etype = desc->sd_error; 612193579Sraj crypto_done(desc->sd_crp); 613193579Sraj 614193579Sraj SEC_DESC_FREE_POINTERS(desc); 615193579Sraj SEC_DESC_FREE_LT(sc, desc); 616193579Sraj SEC_DESC_QUEUED2FREE(sc); 617193579Sraj } 618193579Sraj 619193579Sraj SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 620193579Sraj 621193579Sraj if (!sc->sc_shutdown) { 622193579Sraj wakeup = sc->sc_blocked; 623193579Sraj sc->sc_blocked = 0; 624193579Sraj } 625193579Sraj 626193579Sraj SEC_UNLOCK(sc, descriptors); 627193579Sraj 628193579Sraj /* Enqueue ready descriptors in hardware */ 629193579Sraj sec_enqueue(sc); 630193579Sraj 631193579Sraj if (wakeup) 632193579Sraj crypto_unblock(sc->sc_cid, wakeup); 633193579Sraj} 634193579Sraj 635193579Srajstatic void 636193579Srajsec_secondary_intr(void *arg) 637193579Sraj{ 638193579Sraj struct sec_softc *sc = arg; 639193579Sraj 640193579Sraj device_printf(sc->sc_dev, "spurious secondary interrupt!\n"); 641193579Sraj sec_primary_intr(arg); 642193579Sraj} 643193579Sraj 644193579Srajstatic int 645193579Srajsec_controller_reset(struct sec_softc *sc) 646193579Sraj{ 647193579Sraj int timeout = SEC_TIMEOUT; 648193579Sraj 649193579Sraj /* Reset Controller */ 650193579Sraj SEC_WRITE(sc, SEC_MCR, SEC_MCR_SWR); 651193579Sraj 652193579Sraj while (SEC_READ(sc, SEC_MCR) & SEC_MCR_SWR) { 653193579Sraj DELAY(1000); 654193579Sraj timeout -= 1000; 655193579Sraj 656193579Sraj if (timeout < 0) { 657193579Sraj device_printf(sc->sc_dev, "timeout while waiting for " 658193579Sraj "device reset!\n"); 659193579Sraj return (ETIMEDOUT); 660193579Sraj } 661193579Sraj } 662193579Sraj 663193579Sraj return (0); 664193579Sraj} 665193579Sraj 666193579Srajstatic int 667193579Srajsec_channel_reset(struct sec_softc *sc, int channel, int full) 668193579Sraj{ 669193579Sraj int timeout = SEC_TIMEOUT; 670193579Sraj uint64_t bit = (full) ? SEC_CHAN_CCR_R : SEC_CHAN_CCR_CON; 671193579Sraj uint64_t reg; 672193579Sraj 673193579Sraj /* Reset Channel */ 674193579Sraj reg = SEC_READ(sc, SEC_CHAN_CCR(channel)); 675193579Sraj SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg | bit); 676193579Sraj 677193579Sraj while (SEC_READ(sc, SEC_CHAN_CCR(channel)) & bit) { 678193579Sraj DELAY(1000); 679193579Sraj timeout -= 1000; 680193579Sraj 681193579Sraj if (timeout < 0) { 682193579Sraj device_printf(sc->sc_dev, "timeout while waiting for " 683193579Sraj "channel reset!\n"); 684193579Sraj return (ETIMEDOUT); 685193579Sraj } 686193579Sraj } 687193579Sraj 688193579Sraj if (full) { 689193579Sraj reg = SEC_CHAN_CCR_CDIE | SEC_CHAN_CCR_NT | SEC_CHAN_CCR_BS; 690193579Sraj 691193579Sraj switch(sc->sc_version) { 692193579Sraj case 2: 693193579Sraj reg |= SEC_CHAN_CCR_CDWE; 694193579Sraj break; 695193579Sraj case 3: 696193579Sraj reg |= SEC_CHAN_CCR_AWSE | SEC_CHAN_CCR_WGN; 697193579Sraj break; 698193579Sraj } 699193579Sraj 700193579Sraj SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg); 701193579Sraj } 702193579Sraj 703193579Sraj return (0); 704193579Sraj} 705193579Sraj 706193579Srajstatic int 707193579Srajsec_init(struct sec_softc *sc) 708193579Sraj{ 709193579Sraj uint64_t reg; 710193579Sraj int error, i; 711193579Sraj 712193579Sraj /* Reset controller twice to clear all pending interrupts */ 713193579Sraj error = sec_controller_reset(sc); 714193579Sraj if (error) 715193579Sraj return (error); 716193579Sraj 717193579Sraj error = sec_controller_reset(sc); 718193579Sraj if (error) 719193579Sraj return (error); 720193579Sraj 721193579Sraj /* Reset channels */ 722193579Sraj for (i = 0; i < SEC_CHANNELS; i++) { 723193579Sraj error = sec_channel_reset(sc, i, 1); 724193579Sraj if (error) 725193579Sraj return (error); 726193579Sraj } 727193579Sraj 728193579Sraj /* Enable Interrupts */ 729193579Sraj reg = SEC_INT_ITO; 730193579Sraj for (i = 0; i < SEC_CHANNELS; i++) 731193579Sraj reg |= SEC_INT_CH_DN(i) | SEC_INT_CH_ERR(i); 732193579Sraj 733193579Sraj SEC_WRITE(sc, SEC_IER, reg); 734193579Sraj 735193579Sraj return (error); 736193579Sraj} 737193579Sraj 738193579Srajstatic void 739193579Srajsec_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 740193579Sraj{ 741193579Sraj struct sec_dma_mem *dma_mem = arg; 742193579Sraj 743193579Sraj if (error) 744193579Sraj return; 745193579Sraj 746193579Sraj KASSERT(nseg == 1, ("Wrong number of segments, should be 1")); 747193579Sraj dma_mem->dma_paddr = segs->ds_addr; 748193579Sraj} 749193579Sraj 750193579Srajstatic void 751193579Srajsec_dma_map_desc_cb(void *arg, bus_dma_segment_t *segs, int nseg, 752193579Sraj int error) 753193579Sraj{ 754193579Sraj struct sec_desc_map_info *sdmi = arg; 755193579Sraj struct sec_softc *sc = sdmi->sdmi_sc; 756193579Sraj struct sec_lt *lt = NULL; 757193579Sraj bus_addr_t addr; 758193579Sraj bus_size_t size; 759193579Sraj int i; 760193579Sraj 761193579Sraj SEC_LOCK_ASSERT(sc, descriptors); 762193579Sraj 763193579Sraj if (error) 764193579Sraj return; 765193579Sraj 766193579Sraj for (i = 0; i < nseg; i++) { 767193579Sraj addr = segs[i].ds_addr; 768193579Sraj size = segs[i].ds_len; 769193579Sraj 770193579Sraj /* Skip requested offset */ 771193579Sraj if (sdmi->sdmi_offset >= size) { 772193579Sraj sdmi->sdmi_offset -= size; 773193579Sraj continue; 774193579Sraj } 775193579Sraj 776193579Sraj addr += sdmi->sdmi_offset; 777193579Sraj size -= sdmi->sdmi_offset; 778193579Sraj sdmi->sdmi_offset = 0; 779193579Sraj 780193579Sraj /* Do not link more than requested */ 781193579Sraj if (sdmi->sdmi_size < size) 782193579Sraj size = sdmi->sdmi_size; 783193579Sraj 784193579Sraj lt = SEC_ALLOC_LT_ENTRY(sc); 785193579Sraj lt->sl_lt->shl_length = size; 786193579Sraj lt->sl_lt->shl_r = 0; 787193579Sraj lt->sl_lt->shl_n = 0; 788193579Sraj lt->sl_lt->shl_ptr = addr; 789193579Sraj 790193579Sraj if (sdmi->sdmi_lt_first == NULL) 791193579Sraj sdmi->sdmi_lt_first = lt; 792193579Sraj 793193579Sraj sdmi->sdmi_lt_used += 1; 794193579Sraj 795193579Sraj if ((sdmi->sdmi_size -= size) == 0) 796193579Sraj break; 797193579Sraj } 798193579Sraj 799193579Sraj sdmi->sdmi_lt_last = lt; 800193579Sraj} 801193579Sraj 802193579Srajstatic void 803193579Srajsec_dma_map_desc_cb2(void *arg, bus_dma_segment_t *segs, int nseg, 804193579Sraj bus_size_t size, int error) 805193579Sraj{ 806193579Sraj 807193579Sraj sec_dma_map_desc_cb(arg, segs, nseg, error); 808193579Sraj} 809193579Sraj 810193579Srajstatic int 811193579Srajsec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem, 812193579Sraj bus_size_t size) 813193579Sraj{ 814193579Sraj int error; 815193579Sraj 816193579Sraj if (dma_mem->dma_vaddr != NULL) 817193579Sraj return (EBUSY); 818193579Sraj 819193579Sraj error = bus_dma_tag_create(NULL, /* parent */ 820193579Sraj SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 821193579Sraj BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 822193579Sraj BUS_SPACE_MAXADDR, /* highaddr */ 823193579Sraj NULL, NULL, /* filtfunc, filtfuncarg */ 824193579Sraj size, 1, /* maxsize, nsegments */ 825193579Sraj size, 0, /* maxsegsz, flags */ 826193579Sraj NULL, NULL, /* lockfunc, lockfuncarg */ 827193579Sraj &(dma_mem->dma_tag)); /* dmat */ 828193579Sraj 829193579Sraj if (error) { 830193579Sraj device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 831193579Sraj " %i!\n", error); 832193579Sraj goto err1; 833193579Sraj } 834193579Sraj 835193579Sraj error = bus_dmamem_alloc(dma_mem->dma_tag, &(dma_mem->dma_vaddr), 836193579Sraj BUS_DMA_NOWAIT | BUS_DMA_ZERO, &(dma_mem->dma_map)); 837193579Sraj 838193579Sraj if (error) { 839193579Sraj device_printf(sc->sc_dev, "failed to allocate DMA safe" 840193579Sraj " memory, error %i!\n", error); 841193579Sraj goto err2; 842193579Sraj } 843193579Sraj 844193579Sraj error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map, 845193579Sraj dma_mem->dma_vaddr, size, sec_alloc_dma_mem_cb, dma_mem, 846193579Sraj BUS_DMA_NOWAIT); 847193579Sraj 848193579Sraj if (error) { 849193579Sraj device_printf(sc->sc_dev, "cannot get address of the DMA" 850193579Sraj " memory, error %i\n", error); 851193579Sraj goto err3; 852193579Sraj } 853193579Sraj 854193579Sraj dma_mem->dma_is_map = 0; 855193579Sraj return (0); 856193579Sraj 857193579Srajerr3: 858193579Sraj bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, dma_mem->dma_map); 859193579Srajerr2: 860193579Sraj bus_dma_tag_destroy(dma_mem->dma_tag); 861193579Srajerr1: 862193579Sraj dma_mem->dma_vaddr = NULL; 863193579Sraj return(error); 864193579Sraj} 865193579Sraj 866193579Srajstatic int 867193579Srajsec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, void *mem, 868193579Sraj bus_size_t size, int type, struct sec_desc_map_info *sdmi) 869193579Sraj{ 870193579Sraj int error; 871193579Sraj 872193579Sraj if (dma_mem->dma_vaddr != NULL) 873193579Sraj return (EBUSY); 874193579Sraj 875193579Sraj switch (type) { 876193579Sraj case SEC_MEMORY: 877193579Sraj break; 878193579Sraj case SEC_UIO: 879193579Sraj size = SEC_FREE_LT_CNT(sc) * SEC_MAX_DMA_BLOCK_SIZE; 880193579Sraj break; 881193579Sraj case SEC_MBUF: 882193579Sraj size = m_length((struct mbuf*)mem, NULL); 883193579Sraj break; 884193579Sraj default: 885193579Sraj return (EINVAL); 886193579Sraj } 887193579Sraj 888193579Sraj error = bus_dma_tag_create(NULL, /* parent */ 889193579Sraj SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 890193579Sraj BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 891193579Sraj BUS_SPACE_MAXADDR, /* highaddr */ 892193579Sraj NULL, NULL, /* filtfunc, filtfuncarg */ 893193579Sraj size, /* maxsize */ 894193579Sraj SEC_FREE_LT_CNT(sc), /* nsegments */ 895193579Sraj SEC_MAX_DMA_BLOCK_SIZE, 0, /* maxsegsz, flags */ 896193579Sraj NULL, NULL, /* lockfunc, lockfuncarg */ 897193579Sraj &(dma_mem->dma_tag)); /* dmat */ 898193579Sraj 899193579Sraj if (error) { 900193579Sraj device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 901193579Sraj " %i!\n", error); 902193579Sraj dma_mem->dma_vaddr = NULL; 903193579Sraj return (error); 904193579Sraj } 905193579Sraj 906193579Sraj error = bus_dmamap_create(dma_mem->dma_tag, 0, &(dma_mem->dma_map)); 907193579Sraj 908193579Sraj if (error) { 909193579Sraj device_printf(sc->sc_dev, "failed to create DMA map, error %i!" 910193579Sraj "\n", error); 911193579Sraj bus_dma_tag_destroy(dma_mem->dma_tag); 912193579Sraj return (error); 913193579Sraj } 914193579Sraj 915193579Sraj switch (type) { 916193579Sraj case SEC_MEMORY: 917193579Sraj error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map, 918193579Sraj mem, size, sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT); 919193579Sraj break; 920193579Sraj case SEC_UIO: 921193579Sraj error = bus_dmamap_load_uio(dma_mem->dma_tag, dma_mem->dma_map, 922193579Sraj mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT); 923193579Sraj break; 924193579Sraj case SEC_MBUF: 925193579Sraj error = bus_dmamap_load_mbuf(dma_mem->dma_tag, dma_mem->dma_map, 926193579Sraj mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT); 927193579Sraj break; 928193579Sraj } 929193579Sraj 930193579Sraj if (error) { 931193579Sraj device_printf(sc->sc_dev, "cannot get address of the DMA" 932193579Sraj " memory, error %i!\n", error); 933193579Sraj bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 934193579Sraj bus_dma_tag_destroy(dma_mem->dma_tag); 935193579Sraj return (error); 936193579Sraj } 937193579Sraj 938193579Sraj dma_mem->dma_is_map = 1; 939193579Sraj dma_mem->dma_vaddr = mem; 940193579Sraj 941193579Sraj return (0); 942193579Sraj} 943193579Sraj 944193579Srajstatic void 945193579Srajsec_free_dma_mem(struct sec_dma_mem *dma_mem) 946193579Sraj{ 947193579Sraj 948193579Sraj /* Check for double free */ 949193579Sraj if (dma_mem->dma_vaddr == NULL) 950193579Sraj return; 951193579Sraj 952193579Sraj bus_dmamap_unload(dma_mem->dma_tag, dma_mem->dma_map); 953193579Sraj 954193579Sraj if (dma_mem->dma_is_map) 955193579Sraj bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 956193579Sraj else 957193579Sraj bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, 958193579Sraj dma_mem->dma_map); 959193579Sraj 960193579Sraj bus_dma_tag_destroy(dma_mem->dma_tag); 961193579Sraj dma_mem->dma_vaddr = NULL; 962193579Sraj} 963193579Sraj 964193579Srajstatic int 965193579Srajsec_eu_channel(struct sec_softc *sc, int eu) 966193579Sraj{ 967193579Sraj uint64_t reg; 968193579Sraj int channel = 0; 969193579Sraj 970193579Sraj SEC_LOCK_ASSERT(sc, controller); 971193579Sraj 972193579Sraj reg = SEC_READ(sc, SEC_EUASR); 973193579Sraj 974193579Sraj switch (eu) { 975193579Sraj case SEC_EU_AFEU: 976193579Sraj channel = SEC_EUASR_AFEU(reg); 977193579Sraj break; 978193579Sraj case SEC_EU_DEU: 979193579Sraj channel = SEC_EUASR_DEU(reg); 980193579Sraj break; 981193579Sraj case SEC_EU_MDEU_A: 982193579Sraj case SEC_EU_MDEU_B: 983193579Sraj channel = SEC_EUASR_MDEU(reg); 984193579Sraj break; 985193579Sraj case SEC_EU_RNGU: 986193579Sraj channel = SEC_EUASR_RNGU(reg); 987193579Sraj break; 988193579Sraj case SEC_EU_PKEU: 989193579Sraj channel = SEC_EUASR_PKEU(reg); 990193579Sraj break; 991193579Sraj case SEC_EU_AESU: 992193579Sraj channel = SEC_EUASR_AESU(reg); 993193579Sraj break; 994193579Sraj case SEC_EU_KEU: 995193579Sraj channel = SEC_EUASR_KEU(reg); 996193579Sraj break; 997193579Sraj case SEC_EU_CRCU: 998193579Sraj channel = SEC_EUASR_CRCU(reg); 999193579Sraj break; 1000193579Sraj } 1001193579Sraj 1002193579Sraj return (channel - 1); 1003193579Sraj} 1004193579Sraj 1005193579Srajstatic int 1006193579Srajsec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, int channel) 1007193579Sraj{ 1008193579Sraj u_int fflvl = SEC_MAX_FIFO_LEVEL; 1009193579Sraj uint64_t reg; 1010193579Sraj int i; 1011193579Sraj 1012193579Sraj SEC_LOCK_ASSERT(sc, controller); 1013193579Sraj 1014193579Sraj /* Find free channel if have not got one */ 1015193579Sraj if (channel < 0) { 1016193579Sraj for (i = 0; i < SEC_CHANNELS; i++) { 1017193579Sraj reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 1018193579Sraj 1019193579Sraj if ((reg & sc->sc_channel_idle_mask) == 0) { 1020193579Sraj channel = i; 1021193579Sraj break; 1022193579Sraj } 1023193579Sraj } 1024193579Sraj } 1025193579Sraj 1026193579Sraj /* There is no free channel */ 1027193579Sraj if (channel < 0) 1028193579Sraj return (-1); 1029193579Sraj 1030193579Sraj /* Check FIFO level on selected channel */ 1031193579Sraj reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 1032193579Sraj 1033193579Sraj switch(sc->sc_version) { 1034193579Sraj case 2: 1035193579Sraj fflvl = (reg >> SEC_CHAN_CSR2_FFLVL_S) & SEC_CHAN_CSR2_FFLVL_M; 1036193579Sraj break; 1037193579Sraj case 3: 1038193579Sraj fflvl = (reg >> SEC_CHAN_CSR3_FFLVL_S) & SEC_CHAN_CSR3_FFLVL_M; 1039193579Sraj break; 1040193579Sraj } 1041193579Sraj 1042193579Sraj if (fflvl >= SEC_MAX_FIFO_LEVEL) 1043193579Sraj return (-1); 1044193579Sraj 1045193579Sraj /* Enqueue descriptor in channel */ 1046193579Sraj SEC_WRITE(sc, SEC_CHAN_FF(channel), desc->sd_desc_paddr); 1047193579Sraj 1048193579Sraj return (channel); 1049193579Sraj} 1050193579Sraj 1051193579Srajstatic void 1052193579Srajsec_enqueue(struct sec_softc *sc) 1053193579Sraj{ 1054193579Sraj struct sec_desc *desc; 1055193579Sraj int ch0, ch1; 1056193579Sraj 1057193579Sraj SEC_LOCK(sc, descriptors); 1058193579Sraj SEC_LOCK(sc, controller); 1059193579Sraj 1060193579Sraj while (SEC_READY_DESC_CNT(sc) > 0) { 1061193579Sraj desc = SEC_GET_READY_DESC(sc); 1062193579Sraj 1063193579Sraj ch0 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel0); 1064193579Sraj ch1 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel1); 1065193579Sraj 1066193579Sraj /* 1067193579Sraj * Both EU are used by the same channel. 1068193579Sraj * Enqueue descriptor in channel used by busy EUs. 1069193579Sraj */ 1070193579Sraj if (ch0 >= 0 && ch0 == ch1) { 1071193579Sraj if (sec_enqueue_desc(sc, desc, ch0) >= 0) { 1072193579Sraj SEC_DESC_READY2QUEUED(sc); 1073193579Sraj continue; 1074193579Sraj } 1075193579Sraj } 1076193579Sraj 1077193579Sraj /* 1078193579Sraj * Only one EU is free. 1079193579Sraj * Enqueue descriptor in channel used by busy EU. 1080193579Sraj */ 1081193579Sraj if ((ch0 >= 0 && ch1 < 0) || (ch1 >= 0 && ch0 < 0)) { 1082193579Sraj if (sec_enqueue_desc(sc, desc, (ch0 >= 0) ? ch0 : ch1) 1083193579Sraj >= 0) { 1084193579Sraj SEC_DESC_READY2QUEUED(sc); 1085193579Sraj continue; 1086193579Sraj } 1087193579Sraj } 1088193579Sraj 1089193579Sraj /* 1090193579Sraj * Both EU are free. 1091193579Sraj * Enqueue descriptor in first free channel. 1092193579Sraj */ 1093193579Sraj if (ch0 < 0 && ch1 < 0) { 1094193579Sraj if (sec_enqueue_desc(sc, desc, -1) >= 0) { 1095193579Sraj SEC_DESC_READY2QUEUED(sc); 1096193579Sraj continue; 1097193579Sraj } 1098193579Sraj } 1099193579Sraj 1100193579Sraj /* Current descriptor can not be queued at the moment */ 1101193579Sraj SEC_PUT_BACK_READY_DESC(sc); 1102193579Sraj break; 1103193579Sraj } 1104193579Sraj 1105193579Sraj SEC_UNLOCK(sc, controller); 1106193579Sraj SEC_UNLOCK(sc, descriptors); 1107193579Sraj} 1108193579Sraj 1109193579Srajstatic struct sec_desc * 1110193579Srajsec_find_desc(struct sec_softc *sc, bus_addr_t paddr) 1111193579Sraj{ 1112193579Sraj struct sec_desc *desc = NULL; 1113193579Sraj int i; 1114193579Sraj 1115193579Sraj SEC_LOCK_ASSERT(sc, descriptors); 1116193579Sraj 1117193579Sraj for (i = 0; i < SEC_CHANNELS; i++) { 1118193579Sraj if (sc->sc_desc[i].sd_desc_paddr == paddr) { 1119193579Sraj desc = &(sc->sc_desc[i]); 1120193579Sraj break; 1121193579Sraj } 1122193579Sraj } 1123193579Sraj 1124193579Sraj return (desc); 1125193579Sraj} 1126193579Sraj 1127193579Srajstatic int 1128193579Srajsec_make_pointer_direct(struct sec_softc *sc, struct sec_desc *desc, u_int n, 1129193579Sraj bus_addr_t data, bus_size_t dsize) 1130193579Sraj{ 1131193579Sraj struct sec_hw_desc_ptr *ptr; 1132193579Sraj 1133193579Sraj SEC_LOCK_ASSERT(sc, descriptors); 1134193579Sraj 1135193579Sraj ptr = &(desc->sd_desc->shd_pointer[n]); 1136193579Sraj ptr->shdp_length = dsize; 1137193579Sraj ptr->shdp_extent = 0; 1138193579Sraj ptr->shdp_j = 0; 1139193579Sraj ptr->shdp_ptr = data; 1140193579Sraj 1141193579Sraj return (0); 1142193579Sraj} 1143193579Sraj 1144193579Srajstatic int 1145193579Srajsec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, 1146193579Sraj u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype) 1147193579Sraj{ 1148193579Sraj struct sec_desc_map_info sdmi = { sc, dsize, doffset, NULL, NULL, 0 }; 1149193579Sraj struct sec_hw_desc_ptr *ptr; 1150193579Sraj int error; 1151193579Sraj 1152193579Sraj SEC_LOCK_ASSERT(sc, descriptors); 1153193579Sraj 1154193579Sraj /* For flat memory map only requested region */ 1155193579Sraj if (dtype == SEC_MEMORY) { 1156193579Sraj data = (uint8_t*)(data) + doffset; 1157193579Sraj sdmi.sdmi_offset = 0; 1158193579Sraj } 1159193579Sraj 1160193579Sraj error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), data, dsize, 1161193579Sraj dtype, &sdmi); 1162193579Sraj 1163193579Sraj if (error) 1164193579Sraj return (error); 1165193579Sraj 1166193579Sraj sdmi.sdmi_lt_last->sl_lt->shl_r = 1; 1167193579Sraj desc->sd_lt_used += sdmi.sdmi_lt_used; 1168193579Sraj 1169193579Sraj ptr = &(desc->sd_desc->shd_pointer[n]); 1170193579Sraj ptr->shdp_length = dsize; 1171193579Sraj ptr->shdp_extent = 0; 1172193579Sraj ptr->shdp_j = 1; 1173193579Sraj ptr->shdp_ptr = sdmi.sdmi_lt_first->sl_lt_paddr; 1174193579Sraj 1175193579Sraj return (0); 1176193579Sraj} 1177193579Sraj 1178193579Srajstatic int 1179193579Srajsec_split_cri(struct cryptoini *cri, struct cryptoini **enc, 1180193579Sraj struct cryptoini **mac) 1181193579Sraj{ 1182193579Sraj struct cryptoini *e, *m; 1183193579Sraj 1184193579Sraj e = cri; 1185193579Sraj m = cri->cri_next; 1186193579Sraj 1187193579Sraj /* We can haldle only two operations */ 1188193579Sraj if (m && m->cri_next) 1189193579Sraj return (EINVAL); 1190193579Sraj 1191193579Sraj if (sec_mdeu_can_handle(e->cri_alg)) { 1192193579Sraj cri = m; 1193193579Sraj m = e; 1194193579Sraj e = cri; 1195193579Sraj } 1196193579Sraj 1197193579Sraj if (m && !sec_mdeu_can_handle(m->cri_alg)) 1198193579Sraj return (EINVAL); 1199193579Sraj 1200193579Sraj *enc = e; 1201193579Sraj *mac = m; 1202193579Sraj 1203193579Sraj return (0); 1204193579Sraj} 1205193579Sraj 1206193579Srajstatic int 1207193579Srajsec_split_crp(struct cryptop *crp, struct cryptodesc **enc, 1208193579Sraj struct cryptodesc **mac) 1209193579Sraj{ 1210193579Sraj struct cryptodesc *e, *m, *t; 1211193579Sraj 1212193579Sraj e = crp->crp_desc; 1213193579Sraj m = e->crd_next; 1214193579Sraj 1215193579Sraj /* We can haldle only two operations */ 1216193579Sraj if (m && m->crd_next) 1217193579Sraj return (EINVAL); 1218193579Sraj 1219193579Sraj if (sec_mdeu_can_handle(e->crd_alg)) { 1220193579Sraj t = m; 1221193579Sraj m = e; 1222193579Sraj e = t; 1223193579Sraj } 1224193579Sraj 1225193579Sraj if (m && !sec_mdeu_can_handle(m->crd_alg)) 1226193579Sraj return (EINVAL); 1227193579Sraj 1228193579Sraj *enc = e; 1229193579Sraj *mac = m; 1230193579Sraj 1231193579Sraj return (0); 1232193579Sraj} 1233193579Sraj 1234193579Srajstatic int 1235193579Srajsec_alloc_session(struct sec_softc *sc) 1236193579Sraj{ 1237193579Sraj struct sec_session *ses = NULL; 1238193579Sraj int sid = -1; 1239193579Sraj u_int i; 1240193579Sraj 1241193579Sraj SEC_LOCK(sc, sessions); 1242193579Sraj 1243193579Sraj for (i = 0; i < SEC_MAX_SESSIONS; i++) { 1244193579Sraj if (sc->sc_sessions[i].ss_used == 0) { 1245193579Sraj ses = &(sc->sc_sessions[i]); 1246193579Sraj ses->ss_used = 1; 1247193579Sraj ses->ss_ivlen = 0; 1248193579Sraj ses->ss_klen = 0; 1249193579Sraj ses->ss_mklen = 0; 1250193579Sraj sid = i; 1251193579Sraj break; 1252193579Sraj } 1253193579Sraj } 1254193579Sraj 1255193579Sraj SEC_UNLOCK(sc, sessions); 1256193579Sraj 1257193579Sraj return (sid); 1258193579Sraj} 1259193579Sraj 1260193579Srajstatic struct sec_session * 1261193579Srajsec_get_session(struct sec_softc *sc, u_int sid) 1262193579Sraj{ 1263193579Sraj struct sec_session *ses; 1264193579Sraj 1265193579Sraj if (sid >= SEC_MAX_SESSIONS) 1266193579Sraj return (NULL); 1267193579Sraj 1268193579Sraj SEC_LOCK(sc, sessions); 1269193579Sraj 1270193579Sraj ses = &(sc->sc_sessions[sid]); 1271193579Sraj 1272193579Sraj if (ses->ss_used == 0) 1273193579Sraj ses = NULL; 1274193579Sraj 1275193579Sraj SEC_UNLOCK(sc, sessions); 1276193579Sraj 1277193579Sraj return (ses); 1278193579Sraj} 1279193579Sraj 1280193579Srajstatic int 1281193579Srajsec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri) 1282193579Sraj{ 1283193579Sraj struct sec_softc *sc = device_get_softc(dev); 1284193579Sraj struct sec_eu_methods *eu = sec_eus; 1285193579Sraj struct cryptoini *enc = NULL; 1286193579Sraj struct cryptoini *mac = NULL; 1287193579Sraj struct sec_session *ses; 1288193579Sraj int error = -1; 1289193579Sraj int sid; 1290193579Sraj 1291193579Sraj error = sec_split_cri(cri, &enc, &mac); 1292193579Sraj if (error) 1293193579Sraj return (error); 1294193579Sraj 1295193579Sraj /* Check key lengths */ 1296193579Sraj if (enc && enc->cri_key && (enc->cri_klen / 8) > SEC_MAX_KEY_LEN) 1297193579Sraj return (E2BIG); 1298193579Sraj 1299193579Sraj if (mac && mac->cri_key && (mac->cri_klen / 8) > SEC_MAX_KEY_LEN) 1300193579Sraj return (E2BIG); 1301193579Sraj 1302193579Sraj /* Only SEC 3.0 supports digests larger than 256 bits */ 1303193579Sraj if (sc->sc_version < 3 && mac && mac->cri_klen > 256) 1304193579Sraj return (E2BIG); 1305193579Sraj 1306193579Sraj sid = sec_alloc_session(sc); 1307193579Sraj if (sid < 0) 1308193579Sraj return (ENOMEM); 1309193579Sraj 1310193579Sraj ses = sec_get_session(sc, sid); 1311193579Sraj 1312193579Sraj /* Find EU for this session */ 1313193579Sraj while (eu->sem_make_desc != NULL) { 1314193579Sraj error = eu->sem_newsession(sc, ses, enc, mac); 1315193579Sraj if (error >= 0) 1316193579Sraj break; 1317193579Sraj 1318193579Sraj eu++; 1319193579Sraj } 1320193579Sraj 1321193579Sraj /* If not found, return EINVAL */ 1322193579Sraj if (error < 0) { 1323193579Sraj sec_free_session(sc, ses); 1324193579Sraj return (EINVAL); 1325193579Sraj } 1326193579Sraj 1327193579Sraj /* Save cipher key */ 1328193579Sraj if (enc && enc->cri_key) { 1329193579Sraj ses->ss_klen = enc->cri_klen / 8; 1330193579Sraj memcpy(ses->ss_key, enc->cri_key, ses->ss_klen); 1331193579Sraj } 1332193579Sraj 1333193579Sraj /* Save digest key */ 1334193579Sraj if (mac && mac->cri_key) { 1335193579Sraj ses->ss_mklen = mac->cri_klen / 8; 1336193579Sraj memcpy(ses->ss_mkey, mac->cri_key, ses->ss_mklen); 1337193579Sraj } 1338193579Sraj 1339193579Sraj ses->ss_eu = eu; 1340193579Sraj *sidp = sid; 1341193579Sraj 1342193579Sraj return (0); 1343193579Sraj} 1344193579Sraj 1345193579Srajstatic int 1346193579Srajsec_freesession(device_t dev, uint64_t tid) 1347193579Sraj{ 1348193579Sraj struct sec_softc *sc = device_get_softc(dev); 1349193579Sraj struct sec_session *ses; 1350193579Sraj int error = 0; 1351193579Sraj 1352193579Sraj ses = sec_get_session(sc, CRYPTO_SESID2LID(tid)); 1353193579Sraj if (ses == NULL) 1354193579Sraj return (EINVAL); 1355193579Sraj 1356193579Sraj sec_free_session(sc, ses); 1357193579Sraj 1358193579Sraj return (error); 1359193579Sraj} 1360193579Sraj 1361193579Srajstatic int 1362193579Srajsec_process(device_t dev, struct cryptop *crp, int hint) 1363193579Sraj{ 1364193579Sraj struct sec_softc *sc = device_get_softc(dev); 1365193579Sraj struct sec_desc *desc = NULL; 1366193579Sraj struct cryptodesc *mac, *enc; 1367193579Sraj struct sec_session *ses; 1368193579Sraj int buftype, error = 0; 1369193579Sraj 1370193579Sraj /* Check Session ID */ 1371193579Sraj ses = sec_get_session(sc, CRYPTO_SESID2LID(crp->crp_sid)); 1372193579Sraj if (ses == NULL) { 1373193579Sraj crp->crp_etype = EINVAL; 1374193579Sraj crypto_done(crp); 1375193579Sraj return (0); 1376193579Sraj } 1377193579Sraj 1378193579Sraj /* Check for input length */ 1379193579Sraj if (crp->crp_ilen > SEC_MAX_DMA_BLOCK_SIZE) { 1380193579Sraj crp->crp_etype = E2BIG; 1381193579Sraj crypto_done(crp); 1382193579Sraj return (0); 1383193579Sraj } 1384193579Sraj 1385193579Sraj /* Get descriptors */ 1386193579Sraj if (sec_split_crp(crp, &enc, &mac)) { 1387193579Sraj crp->crp_etype = EINVAL; 1388193579Sraj crypto_done(crp); 1389193579Sraj return (0); 1390193579Sraj } 1391193579Sraj 1392193579Sraj SEC_LOCK(sc, descriptors); 1393193579Sraj SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1394193579Sraj 1395193579Sraj /* Block driver if there is no free descriptors or we are going down */ 1396193579Sraj if (SEC_FREE_DESC_CNT(sc) == 0 || sc->sc_shutdown) { 1397193579Sraj sc->sc_blocked |= CRYPTO_SYMQ; 1398193579Sraj SEC_UNLOCK(sc, descriptors); 1399193579Sraj return (ERESTART); 1400193579Sraj } 1401193579Sraj 1402193579Sraj /* Prepare descriptor */ 1403193579Sraj desc = SEC_GET_FREE_DESC(sc); 1404193579Sraj desc->sd_lt_used = 0; 1405193579Sraj desc->sd_error = 0; 1406193579Sraj desc->sd_crp = crp; 1407193579Sraj 1408193579Sraj if (crp->crp_flags & CRYPTO_F_IOV) 1409193579Sraj buftype = SEC_UIO; 1410193579Sraj else if (crp->crp_flags & CRYPTO_F_IMBUF) 1411193579Sraj buftype = SEC_MBUF; 1412193579Sraj else 1413193579Sraj buftype = SEC_MEMORY; 1414193579Sraj 1415193579Sraj if (enc && enc->crd_flags & CRD_F_ENCRYPT) { 1416193579Sraj if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1417193579Sraj memcpy(desc->sd_desc->shd_iv, enc->crd_iv, 1418193579Sraj ses->ss_ivlen); 1419193579Sraj else 1420193579Sraj arc4rand(desc->sd_desc->shd_iv, ses->ss_ivlen, 0); 1421193579Sraj 1422193579Sraj if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0) 1423193579Sraj crypto_copyback(crp->crp_flags, crp->crp_buf, 1424193579Sraj enc->crd_inject, ses->ss_ivlen, 1425193579Sraj desc->sd_desc->shd_iv); 1426193579Sraj } else if (enc) { 1427193579Sraj if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1428193579Sraj memcpy(desc->sd_desc->shd_iv, enc->crd_iv, 1429193579Sraj ses->ss_ivlen); 1430193579Sraj else 1431193579Sraj crypto_copydata(crp->crp_flags, crp->crp_buf, 1432193579Sraj enc->crd_inject, ses->ss_ivlen, 1433193579Sraj desc->sd_desc->shd_iv); 1434193579Sraj } 1435193579Sraj 1436193579Sraj if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) { 1437193579Sraj if ((enc->crd_klen / 8) <= SEC_MAX_KEY_LEN) { 1438193579Sraj ses->ss_klen = enc->crd_klen / 8; 1439193579Sraj memcpy(ses->ss_key, enc->crd_key, ses->ss_klen); 1440193579Sraj } else 1441193579Sraj error = E2BIG; 1442193579Sraj } 1443193579Sraj 1444193579Sraj if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) { 1445193579Sraj if ((mac->crd_klen / 8) <= SEC_MAX_KEY_LEN) { 1446193579Sraj ses->ss_mklen = mac->crd_klen / 8; 1447193579Sraj memcpy(ses->ss_mkey, mac->crd_key, ses->ss_mklen); 1448193579Sraj } else 1449193579Sraj error = E2BIG; 1450193579Sraj } 1451193579Sraj 1452193579Sraj if (!error) { 1453193579Sraj memcpy(desc->sd_desc->shd_key, ses->ss_key, ses->ss_klen); 1454193579Sraj memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, ses->ss_mklen); 1455193579Sraj 1456193579Sraj error = ses->ss_eu->sem_make_desc(sc, ses, desc, crp, buftype); 1457193579Sraj } 1458193579Sraj 1459193579Sraj if (error) { 1460193579Sraj SEC_DESC_FREE_POINTERS(desc); 1461193579Sraj SEC_DESC_PUT_BACK_LT(sc, desc); 1462193579Sraj SEC_PUT_BACK_FREE_DESC(sc); 1463193579Sraj SEC_UNLOCK(sc, descriptors); 1464193579Sraj crp->crp_etype = error; 1465193579Sraj crypto_done(crp); 1466193579Sraj return (0); 1467193579Sraj } 1468193579Sraj 1469193579Sraj /* 1470193579Sraj * Skip DONE interrupt if this is not last request in burst, but only 1471193579Sraj * if we are running on SEC 3.X. On SEC 2.X we have to enable DONE 1472193579Sraj * signaling on each descriptor. 1473193579Sraj */ 1474193579Sraj if ((hint & CRYPTO_HINT_MORE) && sc->sc_version == 3) 1475193579Sraj desc->sd_desc->shd_dn = 0; 1476193579Sraj else 1477193579Sraj desc->sd_desc->shd_dn = 1; 1478193579Sraj 1479193579Sraj SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1480193579Sraj SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_POSTREAD | 1481193579Sraj BUS_DMASYNC_POSTWRITE); 1482193579Sraj SEC_DESC_FREE2READY(sc); 1483193579Sraj SEC_UNLOCK(sc, descriptors); 1484193579Sraj 1485193579Sraj /* Enqueue ready descriptors in hardware */ 1486193579Sraj sec_enqueue(sc); 1487193579Sraj 1488193579Sraj return (0); 1489193579Sraj} 1490193579Sraj 1491193579Srajstatic int 1492193579Srajsec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc, 1493193579Sraj struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc, 1494193579Sraj int buftype) 1495193579Sraj{ 1496193579Sraj struct sec_hw_desc *hd = desc->sd_desc; 1497193579Sraj int error; 1498193579Sraj 1499193579Sraj hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 1500193579Sraj hd->shd_eu_sel1 = SEC_EU_NONE; 1501193579Sraj hd->shd_mode1 = 0; 1502193579Sraj 1503193579Sraj /* Pointer 0: NULL */ 1504193579Sraj error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 1505193579Sraj if (error) 1506193579Sraj return (error); 1507193579Sraj 1508193579Sraj /* Pointer 1: IV IN */ 1509193579Sraj error = sec_make_pointer_direct(sc, desc, 1, desc->sd_desc_paddr + 1510193579Sraj offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen); 1511193579Sraj if (error) 1512193579Sraj return (error); 1513193579Sraj 1514193579Sraj /* Pointer 2: Cipher Key */ 1515193579Sraj error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 1516193579Sraj offsetof(struct sec_hw_desc, shd_key), ses->ss_klen); 1517193579Sraj if (error) 1518193579Sraj return (error); 1519193579Sraj 1520193579Sraj /* Pointer 3: Data IN */ 1521193579Sraj error = sec_make_pointer(sc, desc, 3, crp->crp_buf, enc->crd_skip, 1522193579Sraj enc->crd_len, buftype); 1523193579Sraj if (error) 1524193579Sraj return (error); 1525193579Sraj 1526193579Sraj /* Pointer 4: Data OUT */ 1527193579Sraj error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip, 1528193579Sraj enc->crd_len, buftype); 1529193579Sraj if (error) 1530193579Sraj return (error); 1531193579Sraj 1532193579Sraj /* Pointer 5: IV OUT (Not used: NULL) */ 1533193579Sraj error = sec_make_pointer_direct(sc, desc, 5, 0, 0); 1534193579Sraj if (error) 1535193579Sraj return (error); 1536193579Sraj 1537193579Sraj /* Pointer 6: NULL */ 1538193579Sraj error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 1539193579Sraj 1540193579Sraj return (error); 1541193579Sraj} 1542193579Sraj 1543193579Srajstatic int 1544193579Srajsec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc, 1545193579Sraj struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc, 1546193579Sraj struct cryptodesc *mac, int buftype) 1547193579Sraj{ 1548193579Sraj struct sec_hw_desc *hd = desc->sd_desc; 1549193579Sraj u_int eu, mode, hashlen; 1550193579Sraj int error; 1551193579Sraj 1552193579Sraj if (mac->crd_len < enc->crd_len) 1553193579Sraj return (EINVAL); 1554193579Sraj 1555193579Sraj if (mac->crd_skip + mac->crd_len != enc->crd_skip + enc->crd_len) 1556193579Sraj return (EINVAL); 1557193579Sraj 1558193579Sraj error = sec_mdeu_config(mac, &eu, &mode, &hashlen); 1559193579Sraj if (error) 1560193579Sraj return (error); 1561193579Sraj 1562193579Sraj hd->shd_desc_type = SEC_DT_HMAC_SNOOP; 1563193579Sraj hd->shd_eu_sel1 = eu; 1564193579Sraj hd->shd_mode1 = mode; 1565193579Sraj 1566193579Sraj /* Pointer 0: HMAC Key */ 1567193579Sraj error = sec_make_pointer_direct(sc, desc, 0, desc->sd_desc_paddr + 1568193579Sraj offsetof(struct sec_hw_desc, shd_mkey), ses->ss_mklen); 1569193579Sraj if (error) 1570193579Sraj return (error); 1571193579Sraj 1572193579Sraj /* Pointer 1: HMAC-Only Data IN */ 1573193579Sraj error = sec_make_pointer(sc, desc, 1, crp->crp_buf, mac->crd_skip, 1574193579Sraj mac->crd_len - enc->crd_len, buftype); 1575193579Sraj if (error) 1576193579Sraj return (error); 1577193579Sraj 1578193579Sraj /* Pointer 2: Cipher Key */ 1579193579Sraj error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 1580193579Sraj offsetof(struct sec_hw_desc, shd_key), ses->ss_klen); 1581193579Sraj if (error) 1582193579Sraj return (error); 1583193579Sraj 1584193579Sraj /* Pointer 3: IV IN */ 1585193579Sraj error = sec_make_pointer_direct(sc, desc, 3, desc->sd_desc_paddr + 1586193579Sraj offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen); 1587193579Sraj if (error) 1588193579Sraj return (error); 1589193579Sraj 1590193579Sraj /* Pointer 4: Data IN */ 1591193579Sraj error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip, 1592193579Sraj enc->crd_len, buftype); 1593193579Sraj if (error) 1594193579Sraj return (error); 1595193579Sraj 1596193579Sraj /* Pointer 5: Data OUT */ 1597193579Sraj error = sec_make_pointer(sc, desc, 5, crp->crp_buf, enc->crd_skip, 1598193579Sraj enc->crd_len, buftype); 1599193579Sraj if (error) 1600193579Sraj return (error); 1601193579Sraj 1602193579Sraj /* Pointer 6: HMAC OUT */ 1603193579Sraj error = sec_make_pointer(sc, desc, 6, crp->crp_buf, mac->crd_inject, 1604193579Sraj hashlen, buftype); 1605193579Sraj 1606193579Sraj return (error); 1607193579Sraj} 1608193579Sraj 1609193579Sraj/* AESU */ 1610193579Sraj 1611193579Srajstatic int 1612193579Srajsec_aesu_newsession(struct sec_softc *sc, struct sec_session *ses, 1613193579Sraj struct cryptoini *enc, struct cryptoini *mac) 1614193579Sraj{ 1615193579Sraj 1616193579Sraj if (enc == NULL) 1617193579Sraj return (-1); 1618193579Sraj 1619193579Sraj if (enc->cri_alg != CRYPTO_AES_CBC) 1620193579Sraj return (-1); 1621193579Sraj 1622193579Sraj ses->ss_ivlen = AES_BLOCK_LEN; 1623193579Sraj 1624193579Sraj return (0); 1625193579Sraj} 1626193579Sraj 1627193579Srajstatic int 1628193579Srajsec_aesu_make_desc(struct sec_softc *sc, struct sec_session *ses, 1629193579Sraj struct sec_desc *desc, struct cryptop *crp, int buftype) 1630193579Sraj{ 1631193579Sraj struct sec_hw_desc *hd = desc->sd_desc; 1632193579Sraj struct cryptodesc *enc, *mac; 1633193579Sraj int error; 1634193579Sraj 1635193579Sraj error = sec_split_crp(crp, &enc, &mac); 1636193579Sraj if (error) 1637193579Sraj return (error); 1638193579Sraj 1639193579Sraj if (!enc) 1640193579Sraj return (EINVAL); 1641193579Sraj 1642193579Sraj hd->shd_eu_sel0 = SEC_EU_AESU; 1643193579Sraj hd->shd_mode0 = SEC_AESU_MODE_CBC; 1644193579Sraj 1645193579Sraj if (enc->crd_alg != CRYPTO_AES_CBC) 1646193579Sraj return (EINVAL); 1647193579Sraj 1648193579Sraj if (enc->crd_flags & CRD_F_ENCRYPT) { 1649193579Sraj hd->shd_mode0 |= SEC_AESU_MODE_ED; 1650193579Sraj hd->shd_dir = 0; 1651193579Sraj } else 1652193579Sraj hd->shd_dir = 1; 1653193579Sraj 1654193579Sraj if (mac) 1655193579Sraj error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac, 1656193579Sraj buftype); 1657193579Sraj else 1658193579Sraj error = sec_build_common_ns_desc(sc, desc, ses, crp, enc, 1659193579Sraj buftype); 1660193579Sraj 1661193579Sraj return (error); 1662193579Sraj} 1663193579Sraj 1664193579Sraj/* DEU */ 1665193579Sraj 1666193579Srajstatic int 1667193579Srajsec_deu_newsession(struct sec_softc *sc, struct sec_session *ses, 1668193579Sraj struct cryptoini *enc, struct cryptoini *mac) 1669193579Sraj{ 1670193579Sraj 1671193579Sraj if (enc == NULL) 1672193579Sraj return (-1); 1673193579Sraj 1674193579Sraj switch (enc->cri_alg) { 1675193579Sraj case CRYPTO_DES_CBC: 1676193579Sraj case CRYPTO_3DES_CBC: 1677193579Sraj break; 1678193579Sraj default: 1679193579Sraj return (-1); 1680193579Sraj } 1681193579Sraj 1682193579Sraj ses->ss_ivlen = DES_BLOCK_LEN; 1683193579Sraj 1684193579Sraj return (0); 1685193579Sraj} 1686193579Sraj 1687193579Srajstatic int 1688193579Srajsec_deu_make_desc(struct sec_softc *sc, struct sec_session *ses, 1689193579Sraj struct sec_desc *desc, struct cryptop *crp, int buftype) 1690193579Sraj{ 1691193579Sraj struct sec_hw_desc *hd = desc->sd_desc; 1692193579Sraj struct cryptodesc *enc, *mac; 1693193579Sraj int error; 1694193579Sraj 1695193579Sraj error = sec_split_crp(crp, &enc, &mac); 1696193579Sraj if (error) 1697193579Sraj return (error); 1698193579Sraj 1699193579Sraj if (!enc) 1700193579Sraj return (EINVAL); 1701193579Sraj 1702193579Sraj hd->shd_eu_sel0 = SEC_EU_DEU; 1703193579Sraj hd->shd_mode0 = SEC_DEU_MODE_CBC; 1704193579Sraj 1705193579Sraj switch (enc->crd_alg) { 1706193579Sraj case CRYPTO_3DES_CBC: 1707193579Sraj hd->shd_mode0 |= SEC_DEU_MODE_TS; 1708193579Sraj break; 1709193579Sraj case CRYPTO_DES_CBC: 1710193579Sraj break; 1711193579Sraj default: 1712193579Sraj return (EINVAL); 1713193579Sraj } 1714193579Sraj 1715193579Sraj if (enc->crd_flags & CRD_F_ENCRYPT) { 1716193579Sraj hd->shd_mode0 |= SEC_DEU_MODE_ED; 1717193579Sraj hd->shd_dir = 0; 1718193579Sraj } else 1719193579Sraj hd->shd_dir = 1; 1720193579Sraj 1721193579Sraj if (mac) 1722193579Sraj error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac, 1723193579Sraj buftype); 1724193579Sraj else 1725193579Sraj error = sec_build_common_ns_desc(sc, desc, ses, crp, enc, 1726193579Sraj buftype); 1727193579Sraj 1728193579Sraj return (error); 1729193579Sraj} 1730193579Sraj 1731193579Sraj/* MDEU */ 1732193579Sraj 1733193579Srajstatic int 1734193579Srajsec_mdeu_can_handle(u_int alg) 1735193579Sraj{ 1736193579Sraj switch (alg) { 1737193579Sraj case CRYPTO_MD5: 1738193579Sraj case CRYPTO_SHA1: 1739193579Sraj case CRYPTO_MD5_HMAC: 1740193579Sraj case CRYPTO_SHA1_HMAC: 1741193579Sraj case CRYPTO_SHA2_256_HMAC: 1742193579Sraj case CRYPTO_SHA2_384_HMAC: 1743193579Sraj case CRYPTO_SHA2_512_HMAC: 1744193579Sraj return (1); 1745193579Sraj default: 1746193579Sraj return (0); 1747193579Sraj } 1748193579Sraj} 1749193579Sraj 1750193579Srajstatic int 1751193579Srajsec_mdeu_config(struct cryptodesc *crd, u_int *eu, u_int *mode, u_int *hashlen) 1752193579Sraj{ 1753193579Sraj 1754193579Sraj *mode = SEC_MDEU_MODE_PD | SEC_MDEU_MODE_INIT; 1755193579Sraj *eu = SEC_EU_NONE; 1756193579Sraj 1757193579Sraj switch (crd->crd_alg) { 1758193579Sraj case CRYPTO_MD5_HMAC: 1759193579Sraj *mode |= SEC_MDEU_MODE_HMAC; 1760193579Sraj /* FALLTHROUGH */ 1761193579Sraj case CRYPTO_MD5: 1762193579Sraj *eu = SEC_EU_MDEU_A; 1763193579Sraj *mode |= SEC_MDEU_MODE_MD5; 1764193579Sraj *hashlen = MD5_HASH_LEN; 1765193579Sraj break; 1766193579Sraj case CRYPTO_SHA1_HMAC: 1767193579Sraj *mode |= SEC_MDEU_MODE_HMAC; 1768193579Sraj /* FALLTHROUGH */ 1769193579Sraj case CRYPTO_SHA1: 1770193579Sraj *eu = SEC_EU_MDEU_A; 1771193579Sraj *mode |= SEC_MDEU_MODE_SHA1; 1772193579Sraj *hashlen = SHA1_HASH_LEN; 1773193579Sraj break; 1774193579Sraj case CRYPTO_SHA2_256_HMAC: 1775193579Sraj *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA256; 1776193579Sraj *eu = SEC_EU_MDEU_A; 1777193579Sraj break; 1778193579Sraj case CRYPTO_SHA2_384_HMAC: 1779193579Sraj *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA384; 1780193579Sraj *eu = SEC_EU_MDEU_B; 1781193579Sraj break; 1782193579Sraj case CRYPTO_SHA2_512_HMAC: 1783193579Sraj *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA512; 1784193579Sraj *eu = SEC_EU_MDEU_B; 1785193579Sraj break; 1786193579Sraj default: 1787193579Sraj return (EINVAL); 1788193579Sraj } 1789193579Sraj 1790193579Sraj if (*mode & SEC_MDEU_MODE_HMAC) 1791193579Sraj *hashlen = SEC_HMAC_HASH_LEN; 1792193579Sraj 1793193579Sraj return (0); 1794193579Sraj} 1795193579Sraj 1796193579Srajstatic int 1797193579Srajsec_mdeu_newsession(struct sec_softc *sc, struct sec_session *ses, 1798193579Sraj struct cryptoini *enc, struct cryptoini *mac) 1799193579Sraj{ 1800193579Sraj 1801193579Sraj if (mac && sec_mdeu_can_handle(mac->cri_alg)) 1802193579Sraj return (0); 1803193579Sraj 1804193579Sraj return (-1); 1805193579Sraj} 1806193579Sraj 1807193579Srajstatic int 1808193579Srajsec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses, 1809193579Sraj struct sec_desc *desc, struct cryptop *crp, int buftype) 1810193579Sraj{ 1811193579Sraj struct cryptodesc *enc, *mac; 1812193579Sraj struct sec_hw_desc *hd = desc->sd_desc; 1813193579Sraj u_int eu, mode, hashlen; 1814193579Sraj int error; 1815193579Sraj 1816193579Sraj error = sec_split_crp(crp, &enc, &mac); 1817193579Sraj if (error) 1818193579Sraj return (error); 1819193579Sraj 1820193579Sraj if (enc) 1821193579Sraj return (EINVAL); 1822193579Sraj 1823193579Sraj error = sec_mdeu_config(mac, &eu, &mode, &hashlen); 1824193579Sraj if (error) 1825193579Sraj return (error); 1826193579Sraj 1827193579Sraj hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 1828193579Sraj hd->shd_eu_sel0 = eu; 1829193579Sraj hd->shd_mode0 = mode; 1830193579Sraj hd->shd_eu_sel1 = SEC_EU_NONE; 1831193579Sraj hd->shd_mode1 = 0; 1832193579Sraj 1833193579Sraj /* Pointer 0: NULL */ 1834193579Sraj error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 1835193579Sraj if (error) 1836193579Sraj return (error); 1837193579Sraj 1838193579Sraj /* Pointer 1: Context In (Not used: NULL) */ 1839193579Sraj error = sec_make_pointer_direct(sc, desc, 1, 0, 0); 1840193579Sraj if (error) 1841193579Sraj return (error); 1842193579Sraj 1843193579Sraj /* Pointer 2: HMAC Key (or NULL, depending on digest type) */ 1844193579Sraj if (hd->shd_mode0 & SEC_MDEU_MODE_HMAC) 1845193579Sraj error = sec_make_pointer_direct(sc, desc, 2, 1846193579Sraj desc->sd_desc_paddr + offsetof(struct sec_hw_desc, 1847193579Sraj shd_mkey), ses->ss_mklen); 1848193579Sraj else 1849193579Sraj error = sec_make_pointer_direct(sc, desc, 2, 0, 0); 1850193579Sraj 1851193579Sraj if (error) 1852193579Sraj return (error); 1853193579Sraj 1854193579Sraj /* Pointer 3: Input Data */ 1855193579Sraj error = sec_make_pointer(sc, desc, 3, crp->crp_buf, mac->crd_skip, 1856193579Sraj mac->crd_len, buftype); 1857193579Sraj if (error) 1858193579Sraj return (error); 1859193579Sraj 1860193579Sraj /* Pointer 4: NULL */ 1861193579Sraj error = sec_make_pointer_direct(sc, desc, 4, 0, 0); 1862193579Sraj if (error) 1863193579Sraj return (error); 1864193579Sraj 1865193579Sraj /* Pointer 5: Hash out */ 1866193579Sraj error = sec_make_pointer(sc, desc, 5, crp->crp_buf, 1867193579Sraj mac->crd_inject, hashlen, buftype); 1868193579Sraj if (error) 1869193579Sraj return (error); 1870193579Sraj 1871193579Sraj /* Pointer 6: NULL */ 1872193579Sraj error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 1873193579Sraj 1874193579Sraj return (0); 1875193579Sraj} 1876