1193579Sraj/*- 2193579Sraj * Copyright (C) 2008-2009 Semihalf, Piotr Ziecik 3193579Sraj * All rights reserved. 4193579Sraj * 5193579Sraj * Redistribution and use in source and binary forms, with or without 6193579Sraj * modification, are permitted provided that the following conditions 7193579Sraj * are met: 8193579Sraj * 1. Redistributions of source code must retain the above copyright 9193579Sraj * notice, this list of conditions and the following disclaimer. 10193579Sraj * 2. Redistributions in binary form must reproduce the above copyright 11193579Sraj * notice, this list of conditions and the following disclaimer in the 12193579Sraj * documentation and/or other materials provided with the distribution. 13193579Sraj * 14193579Sraj * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15193579Sraj * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16193579Sraj * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 17193579Sraj * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 18193579Sraj * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 19193579Sraj * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 20193579Sraj * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 21193579Sraj * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 22193579Sraj * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23193579Sraj * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24193579Sraj */ 25193579Sraj 26193579Sraj/* 27193579Sraj * Freescale integrated Security Engine (SEC) driver. Currently SEC 2.0 and 28193579Sraj * 3.0 are supported. 29193579Sraj */ 30193579Sraj 31193579Sraj#include <sys/cdefs.h> 32193579Sraj__FBSDID("$FreeBSD$"); 33193579Sraj 34193579Sraj#include <sys/param.h> 35193579Sraj#include <sys/systm.h> 36193579Sraj#include <sys/bus.h> 37193579Sraj#include <sys/endian.h> 38193579Sraj#include <sys/kernel.h> 39193579Sraj#include <sys/lock.h> 40193579Sraj#include <sys/malloc.h> 41193579Sraj#include <sys/mbuf.h> 42193579Sraj#include <sys/module.h> 43193579Sraj#include <sys/mutex.h> 44193579Sraj#include <sys/random.h> 45193579Sraj#include <sys/rman.h> 46193579Sraj 47293039Sjhibbits#include <machine/_inttypes.h> 48193579Sraj#include <machine/bus.h> 49193579Sraj#include <machine/resource.h> 50193579Sraj 51193579Sraj#include <opencrypto/cryptodev.h> 52193579Sraj#include "cryptodev_if.h" 53193579Sraj 54209908Sraj#include <dev/ofw/ofw_bus_subr.h> 55193579Sraj#include <dev/sec/sec.h> 56193579Sraj 57193579Srajstatic int sec_probe(device_t dev); 58193579Srajstatic int sec_attach(device_t dev); 59193579Srajstatic int sec_detach(device_t dev); 60193579Srajstatic int sec_suspend(device_t dev); 61193579Srajstatic int sec_resume(device_t dev); 62194101Srajstatic int sec_shutdown(device_t dev); 63193579Srajstatic void sec_primary_intr(void *arg); 64193579Srajstatic void sec_secondary_intr(void *arg); 65193579Srajstatic int sec_setup_intr(struct sec_softc *sc, struct resource **ires, 66193579Sraj void **ihand, int *irid, driver_intr_t handler, const char *iname); 67193579Srajstatic void sec_release_intr(struct sec_softc *sc, struct resource *ires, 68193579Sraj void *ihand, int irid, const char *iname); 69193579Srajstatic int sec_controller_reset(struct sec_softc *sc); 70193579Srajstatic int sec_channel_reset(struct sec_softc *sc, int channel, int full); 71193579Srajstatic int sec_init(struct sec_softc *sc); 72193579Srajstatic int sec_alloc_dma_mem(struct sec_softc *sc, 73193579Sraj struct sec_dma_mem *dma_mem, bus_size_t size); 74193579Srajstatic int sec_desc_map_dma(struct sec_softc *sc, 75193579Sraj struct sec_dma_mem *dma_mem, void *mem, bus_size_t size, int type, 76193579Sraj struct sec_desc_map_info *sdmi); 77193579Srajstatic void sec_free_dma_mem(struct sec_dma_mem *dma_mem); 78193579Srajstatic void sec_enqueue(struct sec_softc *sc); 79193579Srajstatic int sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, 80193579Sraj int channel); 81193579Srajstatic int sec_eu_channel(struct sec_softc *sc, int eu); 82193579Srajstatic int sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, 83193579Sraj u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype); 84193579Srajstatic int sec_make_pointer_direct(struct sec_softc *sc, 85193579Sraj struct sec_desc *desc, u_int n, bus_addr_t data, bus_size_t dsize); 86193579Srajstatic int sec_alloc_session(struct sec_softc *sc); 87193579Srajstatic int sec_newsession(device_t dev, u_int32_t *sidp, 88193579Sraj struct cryptoini *cri); 89193579Srajstatic int sec_freesession(device_t dev, uint64_t tid); 90193579Srajstatic int sec_process(device_t dev, struct cryptop *crp, int hint); 91193579Srajstatic int sec_split_cri(struct cryptoini *cri, struct cryptoini **enc, 92193579Sraj struct cryptoini **mac); 93193579Srajstatic int sec_split_crp(struct cryptop *crp, struct cryptodesc **enc, 94193579Sraj struct cryptodesc **mac); 95193579Srajstatic int sec_build_common_ns_desc(struct sec_softc *sc, 96193579Sraj struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp, 97193579Sraj struct cryptodesc *enc, int buftype); 98193579Srajstatic int sec_build_common_s_desc(struct sec_softc *sc, 99193579Sraj struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp, 100193579Sraj struct cryptodesc *enc, struct cryptodesc *mac, int buftype); 101193579Sraj 102193579Srajstatic struct sec_session *sec_get_session(struct sec_softc *sc, u_int sid); 103193579Srajstatic struct sec_desc *sec_find_desc(struct sec_softc *sc, bus_addr_t paddr); 104193579Sraj 105193579Sraj/* AESU */ 106193579Srajstatic int sec_aesu_newsession(struct sec_softc *sc, 107193579Sraj struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac); 108193579Srajstatic int sec_aesu_make_desc(struct sec_softc *sc, 109193579Sraj struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp, 110193579Sraj int buftype); 111193579Sraj 112193579Sraj/* DEU */ 113193579Srajstatic int sec_deu_newsession(struct sec_softc *sc, 114193579Sraj struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac); 115193579Srajstatic int sec_deu_make_desc(struct sec_softc *sc, 116193579Sraj struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp, 117193579Sraj int buftype); 118193579Sraj 119193579Sraj/* MDEU */ 120193579Srajstatic int sec_mdeu_can_handle(u_int alg); 121193579Srajstatic int sec_mdeu_config(struct cryptodesc *crd, 122193579Sraj u_int *eu, u_int *mode, u_int *hashlen); 123193579Srajstatic int sec_mdeu_newsession(struct sec_softc *sc, 124193579Sraj struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac); 125193579Srajstatic int sec_mdeu_make_desc(struct sec_softc *sc, 126193579Sraj struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp, 127193579Sraj int buftype); 128193579Sraj 129193579Srajstatic device_method_t sec_methods[] = { 130193579Sraj /* Device interface */ 131193579Sraj DEVMETHOD(device_probe, sec_probe), 132193579Sraj DEVMETHOD(device_attach, sec_attach), 133193579Sraj DEVMETHOD(device_detach, sec_detach), 134193579Sraj 135193579Sraj DEVMETHOD(device_suspend, sec_suspend), 136193579Sraj DEVMETHOD(device_resume, sec_resume), 137193579Sraj DEVMETHOD(device_shutdown, sec_shutdown), 138193579Sraj 139193579Sraj /* Crypto methods */ 140193579Sraj DEVMETHOD(cryptodev_newsession, sec_newsession), 141193579Sraj DEVMETHOD(cryptodev_freesession,sec_freesession), 142193579Sraj DEVMETHOD(cryptodev_process, sec_process), 143193579Sraj 144227843Smarius DEVMETHOD_END 145193579Sraj}; 146193579Srajstatic driver_t sec_driver = { 147193579Sraj "sec", 148193579Sraj sec_methods, 149193579Sraj sizeof(struct sec_softc), 150193579Sraj}; 151193579Sraj 152193579Srajstatic devclass_t sec_devclass; 153209908SrajDRIVER_MODULE(sec, simplebus, sec_driver, sec_devclass, 0, 0); 154193579SrajMODULE_DEPEND(sec, crypto, 1, 1, 1); 155193579Sraj 156193579Srajstatic struct sec_eu_methods sec_eus[] = { 157193579Sraj { 158193579Sraj sec_aesu_newsession, 159193579Sraj sec_aesu_make_desc, 160193579Sraj }, 161193579Sraj { 162193579Sraj sec_deu_newsession, 163193579Sraj sec_deu_make_desc, 164193579Sraj }, 165193579Sraj { 166193579Sraj sec_mdeu_newsession, 167193579Sraj sec_mdeu_make_desc, 168193579Sraj }, 169193579Sraj { NULL, NULL } 170193579Sraj}; 171193579Sraj 172193579Srajstatic inline void 173193579Srajsec_sync_dma_mem(struct sec_dma_mem *dma_mem, bus_dmasync_op_t op) 174193579Sraj{ 175193579Sraj 176193579Sraj /* Sync only if dma memory is valid */ 177193579Sraj if (dma_mem->dma_vaddr != NULL) 178193579Sraj bus_dmamap_sync(dma_mem->dma_tag, dma_mem->dma_map, op); 179193579Sraj} 180193579Sraj 181193579Srajstatic inline void 182193579Srajsec_free_session(struct sec_softc *sc, struct sec_session *ses) 183193579Sraj{ 184193579Sraj 185193579Sraj SEC_LOCK(sc, sessions); 186193579Sraj ses->ss_used = 0; 187193579Sraj SEC_UNLOCK(sc, sessions); 188193579Sraj} 189193579Sraj 190193579Srajstatic inline void * 191193579Srajsec_get_pointer_data(struct sec_desc *desc, u_int n) 192193579Sraj{ 193193579Sraj 194193579Sraj return (desc->sd_ptr_dmem[n].dma_vaddr); 195193579Sraj} 196193579Sraj 197193579Srajstatic int 198193579Srajsec_probe(device_t dev) 199193579Sraj{ 200193579Sraj struct sec_softc *sc; 201193579Sraj uint64_t id; 202193579Sraj 203261410Sian if (!ofw_bus_status_okay(dev)) 204261410Sian return (ENXIO); 205261410Sian 206209908Sraj if (!ofw_bus_is_compatible(dev, "fsl,sec2.0")) 207193579Sraj return (ENXIO); 208193579Sraj 209193579Sraj sc = device_get_softc(dev); 210193579Sraj 211193579Sraj sc->sc_rrid = 0; 212209908Sraj sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid, 213209908Sraj RF_ACTIVE); 214193579Sraj 215193579Sraj if (sc->sc_rres == NULL) 216193579Sraj return (ENXIO); 217193579Sraj 218193579Sraj sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres); 219193579Sraj sc->sc_bas.bst = rman_get_bustag(sc->sc_rres); 220193579Sraj 221193579Sraj id = SEC_READ(sc, SEC_ID); 222193579Sraj 223193579Sraj bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); 224193579Sraj 225193579Sraj switch (id) { 226193579Sraj case SEC_20_ID: 227193579Sraj device_set_desc(dev, "Freescale Security Engine 2.0"); 228193579Sraj sc->sc_version = 2; 229193579Sraj break; 230193579Sraj case SEC_30_ID: 231193579Sraj device_set_desc(dev, "Freescale Security Engine 3.0"); 232193579Sraj sc->sc_version = 3; 233193579Sraj break; 234235938Sraj case SEC_31_ID: 235235938Sraj device_set_desc(dev, "Freescale Security Engine 3.1"); 236235938Sraj sc->sc_version = 3; 237235938Sraj break; 238193579Sraj default: 239293044Sjhibbits device_printf(dev, "unknown SEC ID 0x%016"PRIx64"!\n", id); 240193579Sraj return (ENXIO); 241193579Sraj } 242193579Sraj 243193579Sraj return (0); 244193579Sraj} 245193579Sraj 246193579Srajstatic int 247193579Srajsec_attach(device_t dev) 248193579Sraj{ 249193579Sraj struct sec_softc *sc; 250193579Sraj struct sec_hw_lt *lt; 251193579Sraj int error = 0; 252193579Sraj int i; 253193579Sraj 254193579Sraj sc = device_get_softc(dev); 255193579Sraj sc->sc_dev = dev; 256193579Sraj sc->sc_blocked = 0; 257193579Sraj sc->sc_shutdown = 0; 258193579Sraj 259193579Sraj sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); 260193579Sraj if (sc->sc_cid < 0) { 261193579Sraj device_printf(dev, "could not get crypto driver ID!\n"); 262193579Sraj return (ENXIO); 263193579Sraj } 264193579Sraj 265193579Sraj /* Init locks */ 266193579Sraj mtx_init(&sc->sc_controller_lock, device_get_nameunit(dev), 267193579Sraj "SEC Controller lock", MTX_DEF); 268193579Sraj mtx_init(&sc->sc_descriptors_lock, device_get_nameunit(dev), 269193579Sraj "SEC Descriptors lock", MTX_DEF); 270193579Sraj mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev), 271193579Sraj "SEC Sessions lock", MTX_DEF); 272193579Sraj 273193579Sraj /* Allocate I/O memory for SEC registers */ 274193579Sraj sc->sc_rrid = 0; 275209908Sraj sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid, 276209908Sraj RF_ACTIVE); 277193579Sraj 278193579Sraj if (sc->sc_rres == NULL) { 279193579Sraj device_printf(dev, "could not allocate I/O memory!\n"); 280193579Sraj goto fail1; 281193579Sraj } 282193579Sraj 283193579Sraj sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres); 284193579Sraj sc->sc_bas.bst = rman_get_bustag(sc->sc_rres); 285193579Sraj 286193579Sraj /* Setup interrupts */ 287193579Sraj sc->sc_pri_irid = 0; 288193579Sraj error = sec_setup_intr(sc, &sc->sc_pri_ires, &sc->sc_pri_ihand, 289193579Sraj &sc->sc_pri_irid, sec_primary_intr, "primary"); 290193579Sraj 291193579Sraj if (error) 292193579Sraj goto fail2; 293193579Sraj 294193579Sraj 295209908Sraj if (sc->sc_version == 3) { 296209908Sraj sc->sc_sec_irid = 1; 297209908Sraj error = sec_setup_intr(sc, &sc->sc_sec_ires, &sc->sc_sec_ihand, 298209908Sraj &sc->sc_sec_irid, sec_secondary_intr, "secondary"); 299193579Sraj 300209908Sraj if (error) 301209908Sraj goto fail3; 302209908Sraj } 303209908Sraj 304193579Sraj /* Alloc DMA memory for descriptors and link tables */ 305193579Sraj error = sec_alloc_dma_mem(sc, &(sc->sc_desc_dmem), 306193579Sraj SEC_DESCRIPTORS * sizeof(struct sec_hw_desc)); 307193579Sraj 308193579Sraj if (error) 309193579Sraj goto fail4; 310193579Sraj 311193579Sraj error = sec_alloc_dma_mem(sc, &(sc->sc_lt_dmem), 312193579Sraj (SEC_LT_ENTRIES + 1) * sizeof(struct sec_hw_lt)); 313193579Sraj 314193579Sraj if (error) 315193579Sraj goto fail5; 316193579Sraj 317193579Sraj /* Fill in descriptors and link tables */ 318193579Sraj for (i = 0; i < SEC_DESCRIPTORS; i++) { 319193579Sraj sc->sc_desc[i].sd_desc = 320193579Sraj (struct sec_hw_desc*)(sc->sc_desc_dmem.dma_vaddr) + i; 321193579Sraj sc->sc_desc[i].sd_desc_paddr = sc->sc_desc_dmem.dma_paddr + 322193579Sraj (i * sizeof(struct sec_hw_desc)); 323193579Sraj } 324193579Sraj 325193579Sraj for (i = 0; i < SEC_LT_ENTRIES + 1; i++) { 326193579Sraj sc->sc_lt[i].sl_lt = 327193579Sraj (struct sec_hw_lt*)(sc->sc_lt_dmem.dma_vaddr) + i; 328193579Sraj sc->sc_lt[i].sl_lt_paddr = sc->sc_lt_dmem.dma_paddr + 329193579Sraj (i * sizeof(struct sec_hw_lt)); 330193579Sraj } 331193579Sraj 332193579Sraj /* Last entry in link table is used to create a circle */ 333193579Sraj lt = sc->sc_lt[SEC_LT_ENTRIES].sl_lt; 334193579Sraj lt->shl_length = 0; 335193579Sraj lt->shl_r = 0; 336193579Sraj lt->shl_n = 1; 337193579Sraj lt->shl_ptr = sc->sc_lt[0].sl_lt_paddr; 338193579Sraj 339193579Sraj /* Init descriptor and link table queues pointers */ 340193579Sraj SEC_CNT_INIT(sc, sc_free_desc_get_cnt, SEC_DESCRIPTORS); 341193579Sraj SEC_CNT_INIT(sc, sc_free_desc_put_cnt, SEC_DESCRIPTORS); 342193579Sraj SEC_CNT_INIT(sc, sc_ready_desc_get_cnt, SEC_DESCRIPTORS); 343193579Sraj SEC_CNT_INIT(sc, sc_ready_desc_put_cnt, SEC_DESCRIPTORS); 344193579Sraj SEC_CNT_INIT(sc, sc_queued_desc_get_cnt, SEC_DESCRIPTORS); 345193579Sraj SEC_CNT_INIT(sc, sc_queued_desc_put_cnt, SEC_DESCRIPTORS); 346193579Sraj SEC_CNT_INIT(sc, sc_lt_alloc_cnt, SEC_LT_ENTRIES); 347193579Sraj SEC_CNT_INIT(sc, sc_lt_free_cnt, SEC_LT_ENTRIES); 348193579Sraj 349193579Sraj /* Create masks for fast checks */ 350193579Sraj sc->sc_int_error_mask = 0; 351193579Sraj for (i = 0; i < SEC_CHANNELS; i++) 352193579Sraj sc->sc_int_error_mask |= (~0ULL & SEC_INT_CH_ERR(i)); 353193579Sraj 354193579Sraj switch (sc->sc_version) { 355193579Sraj case 2: 356193579Sraj sc->sc_channel_idle_mask = 357193579Sraj (SEC_CHAN_CSR2_FFLVL_M << SEC_CHAN_CSR2_FFLVL_S) | 358193579Sraj (SEC_CHAN_CSR2_MSTATE_M << SEC_CHAN_CSR2_MSTATE_S) | 359193579Sraj (SEC_CHAN_CSR2_PSTATE_M << SEC_CHAN_CSR2_PSTATE_S) | 360193579Sraj (SEC_CHAN_CSR2_GSTATE_M << SEC_CHAN_CSR2_GSTATE_S); 361193579Sraj break; 362193579Sraj case 3: 363193579Sraj sc->sc_channel_idle_mask = 364193579Sraj (SEC_CHAN_CSR3_FFLVL_M << SEC_CHAN_CSR3_FFLVL_S) | 365193579Sraj (SEC_CHAN_CSR3_MSTATE_M << SEC_CHAN_CSR3_MSTATE_S) | 366193579Sraj (SEC_CHAN_CSR3_PSTATE_M << SEC_CHAN_CSR3_PSTATE_S) | 367193579Sraj (SEC_CHAN_CSR3_GSTATE_M << SEC_CHAN_CSR3_GSTATE_S); 368193579Sraj break; 369193579Sraj } 370193579Sraj 371193579Sraj /* Init hardware */ 372193579Sraj error = sec_init(sc); 373193579Sraj 374193579Sraj if (error) 375193579Sraj goto fail6; 376193579Sraj 377193579Sraj /* Register in OCF (AESU) */ 378193579Sraj crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); 379193579Sraj 380193579Sraj /* Register in OCF (DEU) */ 381193579Sraj crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); 382193579Sraj crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); 383193579Sraj 384193579Sraj /* Register in OCF (MDEU) */ 385193579Sraj crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0); 386193579Sraj crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 387193579Sraj crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0); 388193579Sraj crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 389193579Sraj crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0); 390193579Sraj if (sc->sc_version >= 3) { 391193579Sraj crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0); 392193579Sraj crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0); 393193579Sraj } 394193579Sraj 395193579Sraj return (0); 396193579Sraj 397193579Srajfail6: 398193579Sraj sec_free_dma_mem(&(sc->sc_lt_dmem)); 399193579Srajfail5: 400193579Sraj sec_free_dma_mem(&(sc->sc_desc_dmem)); 401193579Srajfail4: 402193579Sraj sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 403193579Sraj sc->sc_sec_irid, "secondary"); 404193579Srajfail3: 405193579Sraj sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 406193579Sraj sc->sc_pri_irid, "primary"); 407193579Srajfail2: 408193579Sraj bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); 409193579Srajfail1: 410193579Sraj mtx_destroy(&sc->sc_controller_lock); 411193579Sraj mtx_destroy(&sc->sc_descriptors_lock); 412193579Sraj mtx_destroy(&sc->sc_sessions_lock); 413193579Sraj 414193579Sraj return (ENXIO); 415193579Sraj} 416193579Sraj 417193579Srajstatic int 418193579Srajsec_detach(device_t dev) 419193579Sraj{ 420193579Sraj struct sec_softc *sc = device_get_softc(dev); 421193579Sraj int i, error, timeout = SEC_TIMEOUT; 422193579Sraj 423193579Sraj /* Prepare driver to shutdown */ 424193579Sraj SEC_LOCK(sc, descriptors); 425193579Sraj sc->sc_shutdown = 1; 426193579Sraj SEC_UNLOCK(sc, descriptors); 427193579Sraj 428193579Sraj /* Wait until all queued processing finishes */ 429193579Sraj while (1) { 430193579Sraj SEC_LOCK(sc, descriptors); 431193579Sraj i = SEC_READY_DESC_CNT(sc) + SEC_QUEUED_DESC_CNT(sc); 432193579Sraj SEC_UNLOCK(sc, descriptors); 433193579Sraj 434193579Sraj if (i == 0) 435193579Sraj break; 436193579Sraj 437193579Sraj if (timeout < 0) { 438193579Sraj device_printf(dev, "queue flush timeout!\n"); 439193579Sraj 440193579Sraj /* DMA can be still active - stop it */ 441193579Sraj for (i = 0; i < SEC_CHANNELS; i++) 442193579Sraj sec_channel_reset(sc, i, 1); 443193579Sraj 444193579Sraj break; 445193579Sraj } 446193579Sraj 447193579Sraj timeout -= 1000; 448193579Sraj DELAY(1000); 449193579Sraj } 450193579Sraj 451193579Sraj /* Disable interrupts */ 452193579Sraj SEC_WRITE(sc, SEC_IER, 0); 453193579Sraj 454193579Sraj /* Unregister from OCF */ 455193579Sraj crypto_unregister_all(sc->sc_cid); 456193579Sraj 457193579Sraj /* Free DMA memory */ 458193579Sraj for (i = 0; i < SEC_DESCRIPTORS; i++) 459193579Sraj SEC_DESC_FREE_POINTERS(&(sc->sc_desc[i])); 460193579Sraj 461193579Sraj sec_free_dma_mem(&(sc->sc_lt_dmem)); 462193579Sraj sec_free_dma_mem(&(sc->sc_desc_dmem)); 463193579Sraj 464193579Sraj /* Release interrupts */ 465193579Sraj sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 466193579Sraj sc->sc_pri_irid, "primary"); 467193579Sraj sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 468193579Sraj sc->sc_sec_irid, "secondary"); 469193579Sraj 470193579Sraj /* Release memory */ 471193579Sraj if (sc->sc_rres) { 472193579Sraj error = bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, 473193579Sraj sc->sc_rres); 474193579Sraj if (error) 475193579Sraj device_printf(dev, "bus_release_resource() failed for" 476193579Sraj " I/O memory, error %d\n", error); 477193579Sraj 478193579Sraj sc->sc_rres = NULL; 479193579Sraj } 480193579Sraj 481193579Sraj mtx_destroy(&sc->sc_controller_lock); 482193579Sraj mtx_destroy(&sc->sc_descriptors_lock); 483193579Sraj mtx_destroy(&sc->sc_sessions_lock); 484193579Sraj 485193579Sraj return (0); 486193579Sraj} 487193579Sraj 488193579Srajstatic int 489193579Srajsec_suspend(device_t dev) 490193579Sraj{ 491193579Sraj 492193579Sraj return (0); 493193579Sraj} 494193579Sraj 495193579Srajstatic int 496193579Srajsec_resume(device_t dev) 497193579Sraj{ 498193579Sraj 499193579Sraj return (0); 500193579Sraj} 501193579Sraj 502194101Srajstatic int 503193579Srajsec_shutdown(device_t dev) 504193579Sraj{ 505194101Sraj 506194101Sraj return (0); 507193579Sraj} 508193579Sraj 509193579Srajstatic int 510193579Srajsec_setup_intr(struct sec_softc *sc, struct resource **ires, void **ihand, 511193579Sraj int *irid, driver_intr_t handler, const char *iname) 512193579Sraj{ 513193579Sraj int error; 514193579Sraj 515193579Sraj (*ires) = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ, irid, 516193579Sraj RF_ACTIVE); 517193579Sraj 518193579Sraj if ((*ires) == NULL) { 519193579Sraj device_printf(sc->sc_dev, "could not allocate %s IRQ\n", iname); 520193579Sraj return (ENXIO); 521193579Sraj } 522193579Sraj 523193579Sraj error = bus_setup_intr(sc->sc_dev, *ires, INTR_MPSAFE | INTR_TYPE_NET, 524193579Sraj NULL, handler, sc, ihand); 525193579Sraj 526193579Sraj if (error) { 527193579Sraj device_printf(sc->sc_dev, "failed to set up %s IRQ\n", iname); 528193579Sraj if (bus_release_resource(sc->sc_dev, SYS_RES_IRQ, *irid, *ires)) 529193579Sraj device_printf(sc->sc_dev, "could not release %s IRQ\n", 530193579Sraj iname); 531193579Sraj 532193579Sraj (*ires) = NULL; 533193579Sraj return (error); 534193579Sraj } 535193579Sraj 536193579Sraj return (0); 537193579Sraj} 538193579Sraj 539193579Srajstatic void 540193579Srajsec_release_intr(struct sec_softc *sc, struct resource *ires, void *ihand, 541193579Sraj int irid, const char *iname) 542193579Sraj{ 543193579Sraj int error; 544193579Sraj 545193579Sraj if (ires == NULL) 546193579Sraj return; 547193579Sraj 548193579Sraj error = bus_teardown_intr(sc->sc_dev, ires, ihand); 549193579Sraj if (error) 550193579Sraj device_printf(sc->sc_dev, "bus_teardown_intr() failed for %s" 551193579Sraj " IRQ, error %d\n", iname, error); 552193579Sraj 553193579Sraj error = bus_release_resource(sc->sc_dev, SYS_RES_IRQ, irid, ires); 554193579Sraj if (error) 555193579Sraj device_printf(sc->sc_dev, "bus_release_resource() failed for %s" 556193579Sraj " IRQ, error %d\n", iname, error); 557193579Sraj} 558193579Sraj 559193579Srajstatic void 560193579Srajsec_primary_intr(void *arg) 561193579Sraj{ 562193579Sraj struct sec_softc *sc = arg; 563193579Sraj struct sec_desc *desc; 564193579Sraj uint64_t isr; 565193579Sraj int i, wakeup = 0; 566193579Sraj 567193579Sraj SEC_LOCK(sc, controller); 568193579Sraj 569193579Sraj /* Check for errors */ 570193579Sraj isr = SEC_READ(sc, SEC_ISR); 571193579Sraj if (isr & sc->sc_int_error_mask) { 572193579Sraj /* Check each channel for error */ 573193579Sraj for (i = 0; i < SEC_CHANNELS; i++) { 574193579Sraj if ((isr & SEC_INT_CH_ERR(i)) == 0) 575193579Sraj continue; 576193579Sraj 577193579Sraj device_printf(sc->sc_dev, 578193579Sraj "I/O error on channel %i!\n", i); 579193579Sraj 580193579Sraj /* Find and mark problematic descriptor */ 581193579Sraj desc = sec_find_desc(sc, SEC_READ(sc, 582193579Sraj SEC_CHAN_CDPR(i))); 583193579Sraj 584193579Sraj if (desc != NULL) 585193579Sraj desc->sd_error = EIO; 586193579Sraj 587193579Sraj /* Do partial channel reset */ 588193579Sraj sec_channel_reset(sc, i, 0); 589193579Sraj } 590193579Sraj } 591193579Sraj 592193579Sraj /* ACK interrupt */ 593193579Sraj SEC_WRITE(sc, SEC_ICR, 0xFFFFFFFFFFFFFFFFULL); 594193579Sraj 595193579Sraj SEC_UNLOCK(sc, controller); 596193579Sraj SEC_LOCK(sc, descriptors); 597193579Sraj 598193579Sraj /* Handle processed descriptors */ 599193579Sraj SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 600193579Sraj 601193579Sraj while (SEC_QUEUED_DESC_CNT(sc) > 0) { 602193579Sraj desc = SEC_GET_QUEUED_DESC(sc); 603193579Sraj 604193579Sraj if (desc->sd_desc->shd_done != 0xFF && desc->sd_error == 0) { 605193579Sraj SEC_PUT_BACK_QUEUED_DESC(sc); 606193579Sraj break; 607193579Sraj } 608193579Sraj 609193579Sraj SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_PREREAD | 610193579Sraj BUS_DMASYNC_PREWRITE); 611193579Sraj 612193579Sraj desc->sd_crp->crp_etype = desc->sd_error; 613193579Sraj crypto_done(desc->sd_crp); 614193579Sraj 615193579Sraj SEC_DESC_FREE_POINTERS(desc); 616193579Sraj SEC_DESC_FREE_LT(sc, desc); 617193579Sraj SEC_DESC_QUEUED2FREE(sc); 618193579Sraj } 619193579Sraj 620193579Sraj SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 621193579Sraj 622193579Sraj if (!sc->sc_shutdown) { 623193579Sraj wakeup = sc->sc_blocked; 624193579Sraj sc->sc_blocked = 0; 625193579Sraj } 626193579Sraj 627193579Sraj SEC_UNLOCK(sc, descriptors); 628193579Sraj 629193579Sraj /* Enqueue ready descriptors in hardware */ 630193579Sraj sec_enqueue(sc); 631193579Sraj 632193579Sraj if (wakeup) 633193579Sraj crypto_unblock(sc->sc_cid, wakeup); 634193579Sraj} 635193579Sraj 636193579Srajstatic void 637193579Srajsec_secondary_intr(void *arg) 638193579Sraj{ 639193579Sraj struct sec_softc *sc = arg; 640193579Sraj 641193579Sraj device_printf(sc->sc_dev, "spurious secondary interrupt!\n"); 642193579Sraj sec_primary_intr(arg); 643193579Sraj} 644193579Sraj 645193579Srajstatic int 646193579Srajsec_controller_reset(struct sec_softc *sc) 647193579Sraj{ 648193579Sraj int timeout = SEC_TIMEOUT; 649193579Sraj 650193579Sraj /* Reset Controller */ 651193579Sraj SEC_WRITE(sc, SEC_MCR, SEC_MCR_SWR); 652193579Sraj 653193579Sraj while (SEC_READ(sc, SEC_MCR) & SEC_MCR_SWR) { 654193579Sraj DELAY(1000); 655193579Sraj timeout -= 1000; 656193579Sraj 657193579Sraj if (timeout < 0) { 658193579Sraj device_printf(sc->sc_dev, "timeout while waiting for " 659193579Sraj "device reset!\n"); 660193579Sraj return (ETIMEDOUT); 661193579Sraj } 662193579Sraj } 663193579Sraj 664193579Sraj return (0); 665193579Sraj} 666193579Sraj 667193579Srajstatic int 668193579Srajsec_channel_reset(struct sec_softc *sc, int channel, int full) 669193579Sraj{ 670193579Sraj int timeout = SEC_TIMEOUT; 671193579Sraj uint64_t bit = (full) ? SEC_CHAN_CCR_R : SEC_CHAN_CCR_CON; 672193579Sraj uint64_t reg; 673193579Sraj 674193579Sraj /* Reset Channel */ 675193579Sraj reg = SEC_READ(sc, SEC_CHAN_CCR(channel)); 676193579Sraj SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg | bit); 677193579Sraj 678193579Sraj while (SEC_READ(sc, SEC_CHAN_CCR(channel)) & bit) { 679193579Sraj DELAY(1000); 680193579Sraj timeout -= 1000; 681193579Sraj 682193579Sraj if (timeout < 0) { 683193579Sraj device_printf(sc->sc_dev, "timeout while waiting for " 684193579Sraj "channel reset!\n"); 685193579Sraj return (ETIMEDOUT); 686193579Sraj } 687193579Sraj } 688193579Sraj 689193579Sraj if (full) { 690193579Sraj reg = SEC_CHAN_CCR_CDIE | SEC_CHAN_CCR_NT | SEC_CHAN_CCR_BS; 691193579Sraj 692193579Sraj switch(sc->sc_version) { 693193579Sraj case 2: 694193579Sraj reg |= SEC_CHAN_CCR_CDWE; 695193579Sraj break; 696193579Sraj case 3: 697193579Sraj reg |= SEC_CHAN_CCR_AWSE | SEC_CHAN_CCR_WGN; 698193579Sraj break; 699193579Sraj } 700193579Sraj 701193579Sraj SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg); 702193579Sraj } 703193579Sraj 704193579Sraj return (0); 705193579Sraj} 706193579Sraj 707193579Srajstatic int 708193579Srajsec_init(struct sec_softc *sc) 709193579Sraj{ 710193579Sraj uint64_t reg; 711193579Sraj int error, i; 712193579Sraj 713193579Sraj /* Reset controller twice to clear all pending interrupts */ 714193579Sraj error = sec_controller_reset(sc); 715193579Sraj if (error) 716193579Sraj return (error); 717193579Sraj 718193579Sraj error = sec_controller_reset(sc); 719193579Sraj if (error) 720193579Sraj return (error); 721193579Sraj 722193579Sraj /* Reset channels */ 723193579Sraj for (i = 0; i < SEC_CHANNELS; i++) { 724193579Sraj error = sec_channel_reset(sc, i, 1); 725193579Sraj if (error) 726193579Sraj return (error); 727193579Sraj } 728193579Sraj 729193579Sraj /* Enable Interrupts */ 730193579Sraj reg = SEC_INT_ITO; 731193579Sraj for (i = 0; i < SEC_CHANNELS; i++) 732193579Sraj reg |= SEC_INT_CH_DN(i) | SEC_INT_CH_ERR(i); 733193579Sraj 734193579Sraj SEC_WRITE(sc, SEC_IER, reg); 735193579Sraj 736193579Sraj return (error); 737193579Sraj} 738193579Sraj 739193579Srajstatic void 740193579Srajsec_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 741193579Sraj{ 742193579Sraj struct sec_dma_mem *dma_mem = arg; 743193579Sraj 744193579Sraj if (error) 745193579Sraj return; 746193579Sraj 747193579Sraj KASSERT(nseg == 1, ("Wrong number of segments, should be 1")); 748193579Sraj dma_mem->dma_paddr = segs->ds_addr; 749193579Sraj} 750193579Sraj 751193579Srajstatic void 752193579Srajsec_dma_map_desc_cb(void *arg, bus_dma_segment_t *segs, int nseg, 753193579Sraj int error) 754193579Sraj{ 755193579Sraj struct sec_desc_map_info *sdmi = arg; 756193579Sraj struct sec_softc *sc = sdmi->sdmi_sc; 757193579Sraj struct sec_lt *lt = NULL; 758193579Sraj bus_addr_t addr; 759193579Sraj bus_size_t size; 760193579Sraj int i; 761193579Sraj 762193579Sraj SEC_LOCK_ASSERT(sc, descriptors); 763193579Sraj 764193579Sraj if (error) 765193579Sraj return; 766193579Sraj 767193579Sraj for (i = 0; i < nseg; i++) { 768193579Sraj addr = segs[i].ds_addr; 769193579Sraj size = segs[i].ds_len; 770193579Sraj 771193579Sraj /* Skip requested offset */ 772193579Sraj if (sdmi->sdmi_offset >= size) { 773193579Sraj sdmi->sdmi_offset -= size; 774193579Sraj continue; 775193579Sraj } 776193579Sraj 777193579Sraj addr += sdmi->sdmi_offset; 778193579Sraj size -= sdmi->sdmi_offset; 779193579Sraj sdmi->sdmi_offset = 0; 780193579Sraj 781193579Sraj /* Do not link more than requested */ 782193579Sraj if (sdmi->sdmi_size < size) 783193579Sraj size = sdmi->sdmi_size; 784193579Sraj 785193579Sraj lt = SEC_ALLOC_LT_ENTRY(sc); 786193579Sraj lt->sl_lt->shl_length = size; 787193579Sraj lt->sl_lt->shl_r = 0; 788193579Sraj lt->sl_lt->shl_n = 0; 789193579Sraj lt->sl_lt->shl_ptr = addr; 790193579Sraj 791193579Sraj if (sdmi->sdmi_lt_first == NULL) 792193579Sraj sdmi->sdmi_lt_first = lt; 793193579Sraj 794193579Sraj sdmi->sdmi_lt_used += 1; 795193579Sraj 796193579Sraj if ((sdmi->sdmi_size -= size) == 0) 797193579Sraj break; 798193579Sraj } 799193579Sraj 800193579Sraj sdmi->sdmi_lt_last = lt; 801193579Sraj} 802193579Sraj 803193579Srajstatic void 804193579Srajsec_dma_map_desc_cb2(void *arg, bus_dma_segment_t *segs, int nseg, 805193579Sraj bus_size_t size, int error) 806193579Sraj{ 807193579Sraj 808193579Sraj sec_dma_map_desc_cb(arg, segs, nseg, error); 809193579Sraj} 810193579Sraj 811193579Srajstatic int 812193579Srajsec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem, 813193579Sraj bus_size_t size) 814193579Sraj{ 815193579Sraj int error; 816193579Sraj 817193579Sraj if (dma_mem->dma_vaddr != NULL) 818193579Sraj return (EBUSY); 819193579Sraj 820193579Sraj error = bus_dma_tag_create(NULL, /* parent */ 821193579Sraj SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 822193579Sraj BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 823193579Sraj BUS_SPACE_MAXADDR, /* highaddr */ 824193579Sraj NULL, NULL, /* filtfunc, filtfuncarg */ 825193579Sraj size, 1, /* maxsize, nsegments */ 826193579Sraj size, 0, /* maxsegsz, flags */ 827193579Sraj NULL, NULL, /* lockfunc, lockfuncarg */ 828193579Sraj &(dma_mem->dma_tag)); /* dmat */ 829193579Sraj 830193579Sraj if (error) { 831193579Sraj device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 832193579Sraj " %i!\n", error); 833193579Sraj goto err1; 834193579Sraj } 835193579Sraj 836193579Sraj error = bus_dmamem_alloc(dma_mem->dma_tag, &(dma_mem->dma_vaddr), 837193579Sraj BUS_DMA_NOWAIT | BUS_DMA_ZERO, &(dma_mem->dma_map)); 838193579Sraj 839193579Sraj if (error) { 840193579Sraj device_printf(sc->sc_dev, "failed to allocate DMA safe" 841193579Sraj " memory, error %i!\n", error); 842193579Sraj goto err2; 843193579Sraj } 844193579Sraj 845193579Sraj error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map, 846193579Sraj dma_mem->dma_vaddr, size, sec_alloc_dma_mem_cb, dma_mem, 847193579Sraj BUS_DMA_NOWAIT); 848193579Sraj 849193579Sraj if (error) { 850193579Sraj device_printf(sc->sc_dev, "cannot get address of the DMA" 851193579Sraj " memory, error %i\n", error); 852193579Sraj goto err3; 853193579Sraj } 854193579Sraj 855193579Sraj dma_mem->dma_is_map = 0; 856193579Sraj return (0); 857193579Sraj 858193579Srajerr3: 859193579Sraj bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, dma_mem->dma_map); 860193579Srajerr2: 861193579Sraj bus_dma_tag_destroy(dma_mem->dma_tag); 862193579Srajerr1: 863193579Sraj dma_mem->dma_vaddr = NULL; 864193579Sraj return(error); 865193579Sraj} 866193579Sraj 867193579Srajstatic int 868193579Srajsec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, void *mem, 869193579Sraj bus_size_t size, int type, struct sec_desc_map_info *sdmi) 870193579Sraj{ 871193579Sraj int error; 872193579Sraj 873193579Sraj if (dma_mem->dma_vaddr != NULL) 874193579Sraj return (EBUSY); 875193579Sraj 876193579Sraj switch (type) { 877193579Sraj case SEC_MEMORY: 878193579Sraj break; 879193579Sraj case SEC_UIO: 880193579Sraj size = SEC_FREE_LT_CNT(sc) * SEC_MAX_DMA_BLOCK_SIZE; 881193579Sraj break; 882193579Sraj case SEC_MBUF: 883193579Sraj size = m_length((struct mbuf*)mem, NULL); 884193579Sraj break; 885193579Sraj default: 886193579Sraj return (EINVAL); 887193579Sraj } 888193579Sraj 889193579Sraj error = bus_dma_tag_create(NULL, /* parent */ 890193579Sraj SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 891193579Sraj BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 892193579Sraj BUS_SPACE_MAXADDR, /* highaddr */ 893193579Sraj NULL, NULL, /* filtfunc, filtfuncarg */ 894193579Sraj size, /* maxsize */ 895193579Sraj SEC_FREE_LT_CNT(sc), /* nsegments */ 896193579Sraj SEC_MAX_DMA_BLOCK_SIZE, 0, /* maxsegsz, flags */ 897193579Sraj NULL, NULL, /* lockfunc, lockfuncarg */ 898193579Sraj &(dma_mem->dma_tag)); /* dmat */ 899193579Sraj 900193579Sraj if (error) { 901193579Sraj device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 902193579Sraj " %i!\n", error); 903193579Sraj dma_mem->dma_vaddr = NULL; 904193579Sraj return (error); 905193579Sraj } 906193579Sraj 907193579Sraj error = bus_dmamap_create(dma_mem->dma_tag, 0, &(dma_mem->dma_map)); 908193579Sraj 909193579Sraj if (error) { 910193579Sraj device_printf(sc->sc_dev, "failed to create DMA map, error %i!" 911193579Sraj "\n", error); 912193579Sraj bus_dma_tag_destroy(dma_mem->dma_tag); 913193579Sraj return (error); 914193579Sraj } 915193579Sraj 916193579Sraj switch (type) { 917193579Sraj case SEC_MEMORY: 918193579Sraj error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map, 919193579Sraj mem, size, sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT); 920193579Sraj break; 921193579Sraj case SEC_UIO: 922193579Sraj error = bus_dmamap_load_uio(dma_mem->dma_tag, dma_mem->dma_map, 923193579Sraj mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT); 924193579Sraj break; 925193579Sraj case SEC_MBUF: 926193579Sraj error = bus_dmamap_load_mbuf(dma_mem->dma_tag, dma_mem->dma_map, 927193579Sraj mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT); 928193579Sraj break; 929193579Sraj } 930193579Sraj 931193579Sraj if (error) { 932193579Sraj device_printf(sc->sc_dev, "cannot get address of the DMA" 933193579Sraj " memory, error %i!\n", error); 934193579Sraj bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 935193579Sraj bus_dma_tag_destroy(dma_mem->dma_tag); 936193579Sraj return (error); 937193579Sraj } 938193579Sraj 939193579Sraj dma_mem->dma_is_map = 1; 940193579Sraj dma_mem->dma_vaddr = mem; 941193579Sraj 942193579Sraj return (0); 943193579Sraj} 944193579Sraj 945193579Srajstatic void 946193579Srajsec_free_dma_mem(struct sec_dma_mem *dma_mem) 947193579Sraj{ 948193579Sraj 949193579Sraj /* Check for double free */ 950193579Sraj if (dma_mem->dma_vaddr == NULL) 951193579Sraj return; 952193579Sraj 953193579Sraj bus_dmamap_unload(dma_mem->dma_tag, dma_mem->dma_map); 954193579Sraj 955193579Sraj if (dma_mem->dma_is_map) 956193579Sraj bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 957193579Sraj else 958193579Sraj bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, 959193579Sraj dma_mem->dma_map); 960193579Sraj 961193579Sraj bus_dma_tag_destroy(dma_mem->dma_tag); 962193579Sraj dma_mem->dma_vaddr = NULL; 963193579Sraj} 964193579Sraj 965193579Srajstatic int 966193579Srajsec_eu_channel(struct sec_softc *sc, int eu) 967193579Sraj{ 968193579Sraj uint64_t reg; 969193579Sraj int channel = 0; 970193579Sraj 971193579Sraj SEC_LOCK_ASSERT(sc, controller); 972193579Sraj 973193579Sraj reg = SEC_READ(sc, SEC_EUASR); 974193579Sraj 975193579Sraj switch (eu) { 976193579Sraj case SEC_EU_AFEU: 977193579Sraj channel = SEC_EUASR_AFEU(reg); 978193579Sraj break; 979193579Sraj case SEC_EU_DEU: 980193579Sraj channel = SEC_EUASR_DEU(reg); 981193579Sraj break; 982193579Sraj case SEC_EU_MDEU_A: 983193579Sraj case SEC_EU_MDEU_B: 984193579Sraj channel = SEC_EUASR_MDEU(reg); 985193579Sraj break; 986193579Sraj case SEC_EU_RNGU: 987193579Sraj channel = SEC_EUASR_RNGU(reg); 988193579Sraj break; 989193579Sraj case SEC_EU_PKEU: 990193579Sraj channel = SEC_EUASR_PKEU(reg); 991193579Sraj break; 992193579Sraj case SEC_EU_AESU: 993193579Sraj channel = SEC_EUASR_AESU(reg); 994193579Sraj break; 995193579Sraj case SEC_EU_KEU: 996193579Sraj channel = SEC_EUASR_KEU(reg); 997193579Sraj break; 998193579Sraj case SEC_EU_CRCU: 999193579Sraj channel = SEC_EUASR_CRCU(reg); 1000193579Sraj break; 1001193579Sraj } 1002193579Sraj 1003193579Sraj return (channel - 1); 1004193579Sraj} 1005193579Sraj 1006193579Srajstatic int 1007193579Srajsec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, int channel) 1008193579Sraj{ 1009193579Sraj u_int fflvl = SEC_MAX_FIFO_LEVEL; 1010193579Sraj uint64_t reg; 1011193579Sraj int i; 1012193579Sraj 1013193579Sraj SEC_LOCK_ASSERT(sc, controller); 1014193579Sraj 1015193579Sraj /* Find free channel if have not got one */ 1016193579Sraj if (channel < 0) { 1017193579Sraj for (i = 0; i < SEC_CHANNELS; i++) { 1018193579Sraj reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 1019193579Sraj 1020193579Sraj if ((reg & sc->sc_channel_idle_mask) == 0) { 1021193579Sraj channel = i; 1022193579Sraj break; 1023193579Sraj } 1024193579Sraj } 1025193579Sraj } 1026193579Sraj 1027193579Sraj /* There is no free channel */ 1028193579Sraj if (channel < 0) 1029193579Sraj return (-1); 1030193579Sraj 1031193579Sraj /* Check FIFO level on selected channel */ 1032193579Sraj reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 1033193579Sraj 1034193579Sraj switch(sc->sc_version) { 1035193579Sraj case 2: 1036193579Sraj fflvl = (reg >> SEC_CHAN_CSR2_FFLVL_S) & SEC_CHAN_CSR2_FFLVL_M; 1037193579Sraj break; 1038193579Sraj case 3: 1039193579Sraj fflvl = (reg >> SEC_CHAN_CSR3_FFLVL_S) & SEC_CHAN_CSR3_FFLVL_M; 1040193579Sraj break; 1041193579Sraj } 1042193579Sraj 1043193579Sraj if (fflvl >= SEC_MAX_FIFO_LEVEL) 1044193579Sraj return (-1); 1045193579Sraj 1046193579Sraj /* Enqueue descriptor in channel */ 1047193579Sraj SEC_WRITE(sc, SEC_CHAN_FF(channel), desc->sd_desc_paddr); 1048193579Sraj 1049193579Sraj return (channel); 1050193579Sraj} 1051193579Sraj 1052193579Srajstatic void 1053193579Srajsec_enqueue(struct sec_softc *sc) 1054193579Sraj{ 1055193579Sraj struct sec_desc *desc; 1056193579Sraj int ch0, ch1; 1057193579Sraj 1058193579Sraj SEC_LOCK(sc, descriptors); 1059193579Sraj SEC_LOCK(sc, controller); 1060193579Sraj 1061193579Sraj while (SEC_READY_DESC_CNT(sc) > 0) { 1062193579Sraj desc = SEC_GET_READY_DESC(sc); 1063193579Sraj 1064193579Sraj ch0 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel0); 1065193579Sraj ch1 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel1); 1066193579Sraj 1067193579Sraj /* 1068193579Sraj * Both EU are used by the same channel. 1069193579Sraj * Enqueue descriptor in channel used by busy EUs. 1070193579Sraj */ 1071193579Sraj if (ch0 >= 0 && ch0 == ch1) { 1072193579Sraj if (sec_enqueue_desc(sc, desc, ch0) >= 0) { 1073193579Sraj SEC_DESC_READY2QUEUED(sc); 1074193579Sraj continue; 1075193579Sraj } 1076193579Sraj } 1077193579Sraj 1078193579Sraj /* 1079193579Sraj * Only one EU is free. 1080193579Sraj * Enqueue descriptor in channel used by busy EU. 1081193579Sraj */ 1082193579Sraj if ((ch0 >= 0 && ch1 < 0) || (ch1 >= 0 && ch0 < 0)) { 1083193579Sraj if (sec_enqueue_desc(sc, desc, (ch0 >= 0) ? ch0 : ch1) 1084193579Sraj >= 0) { 1085193579Sraj SEC_DESC_READY2QUEUED(sc); 1086193579Sraj continue; 1087193579Sraj } 1088193579Sraj } 1089193579Sraj 1090193579Sraj /* 1091193579Sraj * Both EU are free. 1092193579Sraj * Enqueue descriptor in first free channel. 1093193579Sraj */ 1094193579Sraj if (ch0 < 0 && ch1 < 0) { 1095193579Sraj if (sec_enqueue_desc(sc, desc, -1) >= 0) { 1096193579Sraj SEC_DESC_READY2QUEUED(sc); 1097193579Sraj continue; 1098193579Sraj } 1099193579Sraj } 1100193579Sraj 1101193579Sraj /* Current descriptor can not be queued at the moment */ 1102193579Sraj SEC_PUT_BACK_READY_DESC(sc); 1103193579Sraj break; 1104193579Sraj } 1105193579Sraj 1106193579Sraj SEC_UNLOCK(sc, controller); 1107193579Sraj SEC_UNLOCK(sc, descriptors); 1108193579Sraj} 1109193579Sraj 1110193579Srajstatic struct sec_desc * 1111193579Srajsec_find_desc(struct sec_softc *sc, bus_addr_t paddr) 1112193579Sraj{ 1113193579Sraj struct sec_desc *desc = NULL; 1114193579Sraj int i; 1115193579Sraj 1116193579Sraj SEC_LOCK_ASSERT(sc, descriptors); 1117193579Sraj 1118193579Sraj for (i = 0; i < SEC_CHANNELS; i++) { 1119193579Sraj if (sc->sc_desc[i].sd_desc_paddr == paddr) { 1120193579Sraj desc = &(sc->sc_desc[i]); 1121193579Sraj break; 1122193579Sraj } 1123193579Sraj } 1124193579Sraj 1125193579Sraj return (desc); 1126193579Sraj} 1127193579Sraj 1128193579Srajstatic int 1129193579Srajsec_make_pointer_direct(struct sec_softc *sc, struct sec_desc *desc, u_int n, 1130193579Sraj bus_addr_t data, bus_size_t dsize) 1131193579Sraj{ 1132193579Sraj struct sec_hw_desc_ptr *ptr; 1133193579Sraj 1134193579Sraj SEC_LOCK_ASSERT(sc, descriptors); 1135193579Sraj 1136193579Sraj ptr = &(desc->sd_desc->shd_pointer[n]); 1137193579Sraj ptr->shdp_length = dsize; 1138193579Sraj ptr->shdp_extent = 0; 1139193579Sraj ptr->shdp_j = 0; 1140193579Sraj ptr->shdp_ptr = data; 1141193579Sraj 1142193579Sraj return (0); 1143193579Sraj} 1144193579Sraj 1145193579Srajstatic int 1146193579Srajsec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, 1147193579Sraj u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype) 1148193579Sraj{ 1149193579Sraj struct sec_desc_map_info sdmi = { sc, dsize, doffset, NULL, NULL, 0 }; 1150193579Sraj struct sec_hw_desc_ptr *ptr; 1151193579Sraj int error; 1152193579Sraj 1153193579Sraj SEC_LOCK_ASSERT(sc, descriptors); 1154193579Sraj 1155193579Sraj /* For flat memory map only requested region */ 1156193579Sraj if (dtype == SEC_MEMORY) { 1157193579Sraj data = (uint8_t*)(data) + doffset; 1158193579Sraj sdmi.sdmi_offset = 0; 1159193579Sraj } 1160193579Sraj 1161193579Sraj error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), data, dsize, 1162193579Sraj dtype, &sdmi); 1163193579Sraj 1164193579Sraj if (error) 1165193579Sraj return (error); 1166193579Sraj 1167193579Sraj sdmi.sdmi_lt_last->sl_lt->shl_r = 1; 1168193579Sraj desc->sd_lt_used += sdmi.sdmi_lt_used; 1169193579Sraj 1170193579Sraj ptr = &(desc->sd_desc->shd_pointer[n]); 1171193579Sraj ptr->shdp_length = dsize; 1172193579Sraj ptr->shdp_extent = 0; 1173193579Sraj ptr->shdp_j = 1; 1174193579Sraj ptr->shdp_ptr = sdmi.sdmi_lt_first->sl_lt_paddr; 1175193579Sraj 1176193579Sraj return (0); 1177193579Sraj} 1178193579Sraj 1179193579Srajstatic int 1180193579Srajsec_split_cri(struct cryptoini *cri, struct cryptoini **enc, 1181193579Sraj struct cryptoini **mac) 1182193579Sraj{ 1183193579Sraj struct cryptoini *e, *m; 1184193579Sraj 1185193579Sraj e = cri; 1186193579Sraj m = cri->cri_next; 1187193579Sraj 1188193579Sraj /* We can haldle only two operations */ 1189193579Sraj if (m && m->cri_next) 1190193579Sraj return (EINVAL); 1191193579Sraj 1192193579Sraj if (sec_mdeu_can_handle(e->cri_alg)) { 1193193579Sraj cri = m; 1194193579Sraj m = e; 1195193579Sraj e = cri; 1196193579Sraj } 1197193579Sraj 1198193579Sraj if (m && !sec_mdeu_can_handle(m->cri_alg)) 1199193579Sraj return (EINVAL); 1200193579Sraj 1201193579Sraj *enc = e; 1202193579Sraj *mac = m; 1203193579Sraj 1204193579Sraj return (0); 1205193579Sraj} 1206193579Sraj 1207193579Srajstatic int 1208193579Srajsec_split_crp(struct cryptop *crp, struct cryptodesc **enc, 1209193579Sraj struct cryptodesc **mac) 1210193579Sraj{ 1211193579Sraj struct cryptodesc *e, *m, *t; 1212193579Sraj 1213193579Sraj e = crp->crp_desc; 1214193579Sraj m = e->crd_next; 1215193579Sraj 1216193579Sraj /* We can haldle only two operations */ 1217193579Sraj if (m && m->crd_next) 1218193579Sraj return (EINVAL); 1219193579Sraj 1220193579Sraj if (sec_mdeu_can_handle(e->crd_alg)) { 1221193579Sraj t = m; 1222193579Sraj m = e; 1223193579Sraj e = t; 1224193579Sraj } 1225193579Sraj 1226193579Sraj if (m && !sec_mdeu_can_handle(m->crd_alg)) 1227193579Sraj return (EINVAL); 1228193579Sraj 1229193579Sraj *enc = e; 1230193579Sraj *mac = m; 1231193579Sraj 1232193579Sraj return (0); 1233193579Sraj} 1234193579Sraj 1235193579Srajstatic int 1236193579Srajsec_alloc_session(struct sec_softc *sc) 1237193579Sraj{ 1238193579Sraj struct sec_session *ses = NULL; 1239193579Sraj int sid = -1; 1240193579Sraj u_int i; 1241193579Sraj 1242193579Sraj SEC_LOCK(sc, sessions); 1243193579Sraj 1244193579Sraj for (i = 0; i < SEC_MAX_SESSIONS; i++) { 1245193579Sraj if (sc->sc_sessions[i].ss_used == 0) { 1246193579Sraj ses = &(sc->sc_sessions[i]); 1247193579Sraj ses->ss_used = 1; 1248193579Sraj ses->ss_ivlen = 0; 1249193579Sraj ses->ss_klen = 0; 1250193579Sraj ses->ss_mklen = 0; 1251193579Sraj sid = i; 1252193579Sraj break; 1253193579Sraj } 1254193579Sraj } 1255193579Sraj 1256193579Sraj SEC_UNLOCK(sc, sessions); 1257193579Sraj 1258193579Sraj return (sid); 1259193579Sraj} 1260193579Sraj 1261193579Srajstatic struct sec_session * 1262193579Srajsec_get_session(struct sec_softc *sc, u_int sid) 1263193579Sraj{ 1264193579Sraj struct sec_session *ses; 1265193579Sraj 1266193579Sraj if (sid >= SEC_MAX_SESSIONS) 1267193579Sraj return (NULL); 1268193579Sraj 1269193579Sraj SEC_LOCK(sc, sessions); 1270193579Sraj 1271193579Sraj ses = &(sc->sc_sessions[sid]); 1272193579Sraj 1273193579Sraj if (ses->ss_used == 0) 1274193579Sraj ses = NULL; 1275193579Sraj 1276193579Sraj SEC_UNLOCK(sc, sessions); 1277193579Sraj 1278193579Sraj return (ses); 1279193579Sraj} 1280193579Sraj 1281193579Srajstatic int 1282193579Srajsec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri) 1283193579Sraj{ 1284193579Sraj struct sec_softc *sc = device_get_softc(dev); 1285193579Sraj struct sec_eu_methods *eu = sec_eus; 1286193579Sraj struct cryptoini *enc = NULL; 1287193579Sraj struct cryptoini *mac = NULL; 1288193579Sraj struct sec_session *ses; 1289193579Sraj int error = -1; 1290193579Sraj int sid; 1291193579Sraj 1292193579Sraj error = sec_split_cri(cri, &enc, &mac); 1293193579Sraj if (error) 1294193579Sraj return (error); 1295193579Sraj 1296193579Sraj /* Check key lengths */ 1297193579Sraj if (enc && enc->cri_key && (enc->cri_klen / 8) > SEC_MAX_KEY_LEN) 1298193579Sraj return (E2BIG); 1299193579Sraj 1300193579Sraj if (mac && mac->cri_key && (mac->cri_klen / 8) > SEC_MAX_KEY_LEN) 1301193579Sraj return (E2BIG); 1302193579Sraj 1303193579Sraj /* Only SEC 3.0 supports digests larger than 256 bits */ 1304193579Sraj if (sc->sc_version < 3 && mac && mac->cri_klen > 256) 1305193579Sraj return (E2BIG); 1306193579Sraj 1307193579Sraj sid = sec_alloc_session(sc); 1308193579Sraj if (sid < 0) 1309193579Sraj return (ENOMEM); 1310193579Sraj 1311193579Sraj ses = sec_get_session(sc, sid); 1312193579Sraj 1313193579Sraj /* Find EU for this session */ 1314193579Sraj while (eu->sem_make_desc != NULL) { 1315193579Sraj error = eu->sem_newsession(sc, ses, enc, mac); 1316193579Sraj if (error >= 0) 1317193579Sraj break; 1318193579Sraj 1319193579Sraj eu++; 1320193579Sraj } 1321193579Sraj 1322193579Sraj /* If not found, return EINVAL */ 1323193579Sraj if (error < 0) { 1324193579Sraj sec_free_session(sc, ses); 1325193579Sraj return (EINVAL); 1326193579Sraj } 1327193579Sraj 1328193579Sraj /* Save cipher key */ 1329193579Sraj if (enc && enc->cri_key) { 1330193579Sraj ses->ss_klen = enc->cri_klen / 8; 1331193579Sraj memcpy(ses->ss_key, enc->cri_key, ses->ss_klen); 1332193579Sraj } 1333193579Sraj 1334193579Sraj /* Save digest key */ 1335193579Sraj if (mac && mac->cri_key) { 1336193579Sraj ses->ss_mklen = mac->cri_klen / 8; 1337193579Sraj memcpy(ses->ss_mkey, mac->cri_key, ses->ss_mklen); 1338193579Sraj } 1339193579Sraj 1340193579Sraj ses->ss_eu = eu; 1341193579Sraj *sidp = sid; 1342193579Sraj 1343193579Sraj return (0); 1344193579Sraj} 1345193579Sraj 1346193579Srajstatic int 1347193579Srajsec_freesession(device_t dev, uint64_t tid) 1348193579Sraj{ 1349193579Sraj struct sec_softc *sc = device_get_softc(dev); 1350193579Sraj struct sec_session *ses; 1351193579Sraj int error = 0; 1352193579Sraj 1353193579Sraj ses = sec_get_session(sc, CRYPTO_SESID2LID(tid)); 1354193579Sraj if (ses == NULL) 1355193579Sraj return (EINVAL); 1356193579Sraj 1357193579Sraj sec_free_session(sc, ses); 1358193579Sraj 1359193579Sraj return (error); 1360193579Sraj} 1361193579Sraj 1362193579Srajstatic int 1363193579Srajsec_process(device_t dev, struct cryptop *crp, int hint) 1364193579Sraj{ 1365193579Sraj struct sec_softc *sc = device_get_softc(dev); 1366193579Sraj struct sec_desc *desc = NULL; 1367193579Sraj struct cryptodesc *mac, *enc; 1368193579Sraj struct sec_session *ses; 1369193579Sraj int buftype, error = 0; 1370193579Sraj 1371193579Sraj /* Check Session ID */ 1372193579Sraj ses = sec_get_session(sc, CRYPTO_SESID2LID(crp->crp_sid)); 1373193579Sraj if (ses == NULL) { 1374193579Sraj crp->crp_etype = EINVAL; 1375193579Sraj crypto_done(crp); 1376193579Sraj return (0); 1377193579Sraj } 1378193579Sraj 1379193579Sraj /* Check for input length */ 1380193579Sraj if (crp->crp_ilen > SEC_MAX_DMA_BLOCK_SIZE) { 1381193579Sraj crp->crp_etype = E2BIG; 1382193579Sraj crypto_done(crp); 1383193579Sraj return (0); 1384193579Sraj } 1385193579Sraj 1386193579Sraj /* Get descriptors */ 1387193579Sraj if (sec_split_crp(crp, &enc, &mac)) { 1388193579Sraj crp->crp_etype = EINVAL; 1389193579Sraj crypto_done(crp); 1390193579Sraj return (0); 1391193579Sraj } 1392193579Sraj 1393193579Sraj SEC_LOCK(sc, descriptors); 1394193579Sraj SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1395193579Sraj 1396193579Sraj /* Block driver if there is no free descriptors or we are going down */ 1397193579Sraj if (SEC_FREE_DESC_CNT(sc) == 0 || sc->sc_shutdown) { 1398193579Sraj sc->sc_blocked |= CRYPTO_SYMQ; 1399193579Sraj SEC_UNLOCK(sc, descriptors); 1400193579Sraj return (ERESTART); 1401193579Sraj } 1402193579Sraj 1403193579Sraj /* Prepare descriptor */ 1404193579Sraj desc = SEC_GET_FREE_DESC(sc); 1405193579Sraj desc->sd_lt_used = 0; 1406193579Sraj desc->sd_error = 0; 1407193579Sraj desc->sd_crp = crp; 1408193579Sraj 1409193579Sraj if (crp->crp_flags & CRYPTO_F_IOV) 1410193579Sraj buftype = SEC_UIO; 1411193579Sraj else if (crp->crp_flags & CRYPTO_F_IMBUF) 1412193579Sraj buftype = SEC_MBUF; 1413193579Sraj else 1414193579Sraj buftype = SEC_MEMORY; 1415193579Sraj 1416193579Sraj if (enc && enc->crd_flags & CRD_F_ENCRYPT) { 1417193579Sraj if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1418193579Sraj memcpy(desc->sd_desc->shd_iv, enc->crd_iv, 1419193579Sraj ses->ss_ivlen); 1420193579Sraj else 1421193579Sraj arc4rand(desc->sd_desc->shd_iv, ses->ss_ivlen, 0); 1422193579Sraj 1423193579Sraj if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0) 1424193579Sraj crypto_copyback(crp->crp_flags, crp->crp_buf, 1425193579Sraj enc->crd_inject, ses->ss_ivlen, 1426193579Sraj desc->sd_desc->shd_iv); 1427193579Sraj } else if (enc) { 1428193579Sraj if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1429193579Sraj memcpy(desc->sd_desc->shd_iv, enc->crd_iv, 1430193579Sraj ses->ss_ivlen); 1431193579Sraj else 1432193579Sraj crypto_copydata(crp->crp_flags, crp->crp_buf, 1433193579Sraj enc->crd_inject, ses->ss_ivlen, 1434193579Sraj desc->sd_desc->shd_iv); 1435193579Sraj } 1436193579Sraj 1437193579Sraj if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) { 1438193579Sraj if ((enc->crd_klen / 8) <= SEC_MAX_KEY_LEN) { 1439193579Sraj ses->ss_klen = enc->crd_klen / 8; 1440193579Sraj memcpy(ses->ss_key, enc->crd_key, ses->ss_klen); 1441193579Sraj } else 1442193579Sraj error = E2BIG; 1443193579Sraj } 1444193579Sraj 1445193579Sraj if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) { 1446193579Sraj if ((mac->crd_klen / 8) <= SEC_MAX_KEY_LEN) { 1447193579Sraj ses->ss_mklen = mac->crd_klen / 8; 1448193579Sraj memcpy(ses->ss_mkey, mac->crd_key, ses->ss_mklen); 1449193579Sraj } else 1450193579Sraj error = E2BIG; 1451193579Sraj } 1452193579Sraj 1453193579Sraj if (!error) { 1454193579Sraj memcpy(desc->sd_desc->shd_key, ses->ss_key, ses->ss_klen); 1455193579Sraj memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, ses->ss_mklen); 1456193579Sraj 1457193579Sraj error = ses->ss_eu->sem_make_desc(sc, ses, desc, crp, buftype); 1458193579Sraj } 1459193579Sraj 1460193579Sraj if (error) { 1461193579Sraj SEC_DESC_FREE_POINTERS(desc); 1462193579Sraj SEC_DESC_PUT_BACK_LT(sc, desc); 1463193579Sraj SEC_PUT_BACK_FREE_DESC(sc); 1464193579Sraj SEC_UNLOCK(sc, descriptors); 1465193579Sraj crp->crp_etype = error; 1466193579Sraj crypto_done(crp); 1467193579Sraj return (0); 1468193579Sraj } 1469193579Sraj 1470193579Sraj /* 1471193579Sraj * Skip DONE interrupt if this is not last request in burst, but only 1472193579Sraj * if we are running on SEC 3.X. On SEC 2.X we have to enable DONE 1473193579Sraj * signaling on each descriptor. 1474193579Sraj */ 1475193579Sraj if ((hint & CRYPTO_HINT_MORE) && sc->sc_version == 3) 1476193579Sraj desc->sd_desc->shd_dn = 0; 1477193579Sraj else 1478193579Sraj desc->sd_desc->shd_dn = 1; 1479193579Sraj 1480193579Sraj SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1481193579Sraj SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_POSTREAD | 1482193579Sraj BUS_DMASYNC_POSTWRITE); 1483193579Sraj SEC_DESC_FREE2READY(sc); 1484193579Sraj SEC_UNLOCK(sc, descriptors); 1485193579Sraj 1486193579Sraj /* Enqueue ready descriptors in hardware */ 1487193579Sraj sec_enqueue(sc); 1488193579Sraj 1489193579Sraj return (0); 1490193579Sraj} 1491193579Sraj 1492193579Srajstatic int 1493193579Srajsec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc, 1494193579Sraj struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc, 1495193579Sraj int buftype) 1496193579Sraj{ 1497193579Sraj struct sec_hw_desc *hd = desc->sd_desc; 1498193579Sraj int error; 1499193579Sraj 1500193579Sraj hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 1501193579Sraj hd->shd_eu_sel1 = SEC_EU_NONE; 1502193579Sraj hd->shd_mode1 = 0; 1503193579Sraj 1504193579Sraj /* Pointer 0: NULL */ 1505193579Sraj error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 1506193579Sraj if (error) 1507193579Sraj return (error); 1508193579Sraj 1509193579Sraj /* Pointer 1: IV IN */ 1510193579Sraj error = sec_make_pointer_direct(sc, desc, 1, desc->sd_desc_paddr + 1511193579Sraj offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen); 1512193579Sraj if (error) 1513193579Sraj return (error); 1514193579Sraj 1515193579Sraj /* Pointer 2: Cipher Key */ 1516193579Sraj error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 1517193579Sraj offsetof(struct sec_hw_desc, shd_key), ses->ss_klen); 1518193579Sraj if (error) 1519193579Sraj return (error); 1520193579Sraj 1521193579Sraj /* Pointer 3: Data IN */ 1522193579Sraj error = sec_make_pointer(sc, desc, 3, crp->crp_buf, enc->crd_skip, 1523193579Sraj enc->crd_len, buftype); 1524193579Sraj if (error) 1525193579Sraj return (error); 1526193579Sraj 1527193579Sraj /* Pointer 4: Data OUT */ 1528193579Sraj error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip, 1529193579Sraj enc->crd_len, buftype); 1530193579Sraj if (error) 1531193579Sraj return (error); 1532193579Sraj 1533193579Sraj /* Pointer 5: IV OUT (Not used: NULL) */ 1534193579Sraj error = sec_make_pointer_direct(sc, desc, 5, 0, 0); 1535193579Sraj if (error) 1536193579Sraj return (error); 1537193579Sraj 1538193579Sraj /* Pointer 6: NULL */ 1539193579Sraj error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 1540193579Sraj 1541193579Sraj return (error); 1542193579Sraj} 1543193579Sraj 1544193579Srajstatic int 1545193579Srajsec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc, 1546193579Sraj struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc, 1547193579Sraj struct cryptodesc *mac, int buftype) 1548193579Sraj{ 1549193579Sraj struct sec_hw_desc *hd = desc->sd_desc; 1550193579Sraj u_int eu, mode, hashlen; 1551193579Sraj int error; 1552193579Sraj 1553193579Sraj if (mac->crd_len < enc->crd_len) 1554193579Sraj return (EINVAL); 1555193579Sraj 1556193579Sraj if (mac->crd_skip + mac->crd_len != enc->crd_skip + enc->crd_len) 1557193579Sraj return (EINVAL); 1558193579Sraj 1559193579Sraj error = sec_mdeu_config(mac, &eu, &mode, &hashlen); 1560193579Sraj if (error) 1561193579Sraj return (error); 1562193579Sraj 1563193579Sraj hd->shd_desc_type = SEC_DT_HMAC_SNOOP; 1564193579Sraj hd->shd_eu_sel1 = eu; 1565193579Sraj hd->shd_mode1 = mode; 1566193579Sraj 1567193579Sraj /* Pointer 0: HMAC Key */ 1568193579Sraj error = sec_make_pointer_direct(sc, desc, 0, desc->sd_desc_paddr + 1569193579Sraj offsetof(struct sec_hw_desc, shd_mkey), ses->ss_mklen); 1570193579Sraj if (error) 1571193579Sraj return (error); 1572193579Sraj 1573193579Sraj /* Pointer 1: HMAC-Only Data IN */ 1574193579Sraj error = sec_make_pointer(sc, desc, 1, crp->crp_buf, mac->crd_skip, 1575193579Sraj mac->crd_len - enc->crd_len, buftype); 1576193579Sraj if (error) 1577193579Sraj return (error); 1578193579Sraj 1579193579Sraj /* Pointer 2: Cipher Key */ 1580193579Sraj error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 1581193579Sraj offsetof(struct sec_hw_desc, shd_key), ses->ss_klen); 1582193579Sraj if (error) 1583193579Sraj return (error); 1584193579Sraj 1585193579Sraj /* Pointer 3: IV IN */ 1586193579Sraj error = sec_make_pointer_direct(sc, desc, 3, desc->sd_desc_paddr + 1587193579Sraj offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen); 1588193579Sraj if (error) 1589193579Sraj return (error); 1590193579Sraj 1591193579Sraj /* Pointer 4: Data IN */ 1592193579Sraj error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip, 1593193579Sraj enc->crd_len, buftype); 1594193579Sraj if (error) 1595193579Sraj return (error); 1596193579Sraj 1597193579Sraj /* Pointer 5: Data OUT */ 1598193579Sraj error = sec_make_pointer(sc, desc, 5, crp->crp_buf, enc->crd_skip, 1599193579Sraj enc->crd_len, buftype); 1600193579Sraj if (error) 1601193579Sraj return (error); 1602193579Sraj 1603193579Sraj /* Pointer 6: HMAC OUT */ 1604193579Sraj error = sec_make_pointer(sc, desc, 6, crp->crp_buf, mac->crd_inject, 1605193579Sraj hashlen, buftype); 1606193579Sraj 1607193579Sraj return (error); 1608193579Sraj} 1609193579Sraj 1610193579Sraj/* AESU */ 1611193579Sraj 1612193579Srajstatic int 1613193579Srajsec_aesu_newsession(struct sec_softc *sc, struct sec_session *ses, 1614193579Sraj struct cryptoini *enc, struct cryptoini *mac) 1615193579Sraj{ 1616193579Sraj 1617193579Sraj if (enc == NULL) 1618193579Sraj return (-1); 1619193579Sraj 1620193579Sraj if (enc->cri_alg != CRYPTO_AES_CBC) 1621193579Sraj return (-1); 1622193579Sraj 1623193579Sraj ses->ss_ivlen = AES_BLOCK_LEN; 1624193579Sraj 1625193579Sraj return (0); 1626193579Sraj} 1627193579Sraj 1628193579Srajstatic int 1629193579Srajsec_aesu_make_desc(struct sec_softc *sc, struct sec_session *ses, 1630193579Sraj struct sec_desc *desc, struct cryptop *crp, int buftype) 1631193579Sraj{ 1632193579Sraj struct sec_hw_desc *hd = desc->sd_desc; 1633193579Sraj struct cryptodesc *enc, *mac; 1634193579Sraj int error; 1635193579Sraj 1636193579Sraj error = sec_split_crp(crp, &enc, &mac); 1637193579Sraj if (error) 1638193579Sraj return (error); 1639193579Sraj 1640193579Sraj if (!enc) 1641193579Sraj return (EINVAL); 1642193579Sraj 1643193579Sraj hd->shd_eu_sel0 = SEC_EU_AESU; 1644193579Sraj hd->shd_mode0 = SEC_AESU_MODE_CBC; 1645193579Sraj 1646193579Sraj if (enc->crd_alg != CRYPTO_AES_CBC) 1647193579Sraj return (EINVAL); 1648193579Sraj 1649193579Sraj if (enc->crd_flags & CRD_F_ENCRYPT) { 1650193579Sraj hd->shd_mode0 |= SEC_AESU_MODE_ED; 1651193579Sraj hd->shd_dir = 0; 1652193579Sraj } else 1653193579Sraj hd->shd_dir = 1; 1654193579Sraj 1655193579Sraj if (mac) 1656193579Sraj error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac, 1657193579Sraj buftype); 1658193579Sraj else 1659193579Sraj error = sec_build_common_ns_desc(sc, desc, ses, crp, enc, 1660193579Sraj buftype); 1661193579Sraj 1662193579Sraj return (error); 1663193579Sraj} 1664193579Sraj 1665193579Sraj/* DEU */ 1666193579Sraj 1667193579Srajstatic int 1668193579Srajsec_deu_newsession(struct sec_softc *sc, struct sec_session *ses, 1669193579Sraj struct cryptoini *enc, struct cryptoini *mac) 1670193579Sraj{ 1671193579Sraj 1672193579Sraj if (enc == NULL) 1673193579Sraj return (-1); 1674193579Sraj 1675193579Sraj switch (enc->cri_alg) { 1676193579Sraj case CRYPTO_DES_CBC: 1677193579Sraj case CRYPTO_3DES_CBC: 1678193579Sraj break; 1679193579Sraj default: 1680193579Sraj return (-1); 1681193579Sraj } 1682193579Sraj 1683193579Sraj ses->ss_ivlen = DES_BLOCK_LEN; 1684193579Sraj 1685193579Sraj return (0); 1686193579Sraj} 1687193579Sraj 1688193579Srajstatic int 1689193579Srajsec_deu_make_desc(struct sec_softc *sc, struct sec_session *ses, 1690193579Sraj struct sec_desc *desc, struct cryptop *crp, int buftype) 1691193579Sraj{ 1692193579Sraj struct sec_hw_desc *hd = desc->sd_desc; 1693193579Sraj struct cryptodesc *enc, *mac; 1694193579Sraj int error; 1695193579Sraj 1696193579Sraj error = sec_split_crp(crp, &enc, &mac); 1697193579Sraj if (error) 1698193579Sraj return (error); 1699193579Sraj 1700193579Sraj if (!enc) 1701193579Sraj return (EINVAL); 1702193579Sraj 1703193579Sraj hd->shd_eu_sel0 = SEC_EU_DEU; 1704193579Sraj hd->shd_mode0 = SEC_DEU_MODE_CBC; 1705193579Sraj 1706193579Sraj switch (enc->crd_alg) { 1707193579Sraj case CRYPTO_3DES_CBC: 1708193579Sraj hd->shd_mode0 |= SEC_DEU_MODE_TS; 1709193579Sraj break; 1710193579Sraj case CRYPTO_DES_CBC: 1711193579Sraj break; 1712193579Sraj default: 1713193579Sraj return (EINVAL); 1714193579Sraj } 1715193579Sraj 1716193579Sraj if (enc->crd_flags & CRD_F_ENCRYPT) { 1717193579Sraj hd->shd_mode0 |= SEC_DEU_MODE_ED; 1718193579Sraj hd->shd_dir = 0; 1719193579Sraj } else 1720193579Sraj hd->shd_dir = 1; 1721193579Sraj 1722193579Sraj if (mac) 1723193579Sraj error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac, 1724193579Sraj buftype); 1725193579Sraj else 1726193579Sraj error = sec_build_common_ns_desc(sc, desc, ses, crp, enc, 1727193579Sraj buftype); 1728193579Sraj 1729193579Sraj return (error); 1730193579Sraj} 1731193579Sraj 1732193579Sraj/* MDEU */ 1733193579Sraj 1734193579Srajstatic int 1735193579Srajsec_mdeu_can_handle(u_int alg) 1736193579Sraj{ 1737193579Sraj switch (alg) { 1738193579Sraj case CRYPTO_MD5: 1739193579Sraj case CRYPTO_SHA1: 1740193579Sraj case CRYPTO_MD5_HMAC: 1741193579Sraj case CRYPTO_SHA1_HMAC: 1742193579Sraj case CRYPTO_SHA2_256_HMAC: 1743193579Sraj case CRYPTO_SHA2_384_HMAC: 1744193579Sraj case CRYPTO_SHA2_512_HMAC: 1745193579Sraj return (1); 1746193579Sraj default: 1747193579Sraj return (0); 1748193579Sraj } 1749193579Sraj} 1750193579Sraj 1751193579Srajstatic int 1752193579Srajsec_mdeu_config(struct cryptodesc *crd, u_int *eu, u_int *mode, u_int *hashlen) 1753193579Sraj{ 1754193579Sraj 1755193579Sraj *mode = SEC_MDEU_MODE_PD | SEC_MDEU_MODE_INIT; 1756193579Sraj *eu = SEC_EU_NONE; 1757193579Sraj 1758193579Sraj switch (crd->crd_alg) { 1759193579Sraj case CRYPTO_MD5_HMAC: 1760193579Sraj *mode |= SEC_MDEU_MODE_HMAC; 1761193579Sraj /* FALLTHROUGH */ 1762193579Sraj case CRYPTO_MD5: 1763193579Sraj *eu = SEC_EU_MDEU_A; 1764193579Sraj *mode |= SEC_MDEU_MODE_MD5; 1765193579Sraj *hashlen = MD5_HASH_LEN; 1766193579Sraj break; 1767193579Sraj case CRYPTO_SHA1_HMAC: 1768193579Sraj *mode |= SEC_MDEU_MODE_HMAC; 1769193579Sraj /* FALLTHROUGH */ 1770193579Sraj case CRYPTO_SHA1: 1771193579Sraj *eu = SEC_EU_MDEU_A; 1772193579Sraj *mode |= SEC_MDEU_MODE_SHA1; 1773193579Sraj *hashlen = SHA1_HASH_LEN; 1774193579Sraj break; 1775193579Sraj case CRYPTO_SHA2_256_HMAC: 1776193579Sraj *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA256; 1777193579Sraj *eu = SEC_EU_MDEU_A; 1778193579Sraj break; 1779193579Sraj case CRYPTO_SHA2_384_HMAC: 1780193579Sraj *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA384; 1781193579Sraj *eu = SEC_EU_MDEU_B; 1782193579Sraj break; 1783193579Sraj case CRYPTO_SHA2_512_HMAC: 1784193579Sraj *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA512; 1785193579Sraj *eu = SEC_EU_MDEU_B; 1786193579Sraj break; 1787193579Sraj default: 1788193579Sraj return (EINVAL); 1789193579Sraj } 1790193579Sraj 1791193579Sraj if (*mode & SEC_MDEU_MODE_HMAC) 1792193579Sraj *hashlen = SEC_HMAC_HASH_LEN; 1793193579Sraj 1794193579Sraj return (0); 1795193579Sraj} 1796193579Sraj 1797193579Srajstatic int 1798193579Srajsec_mdeu_newsession(struct sec_softc *sc, struct sec_session *ses, 1799193579Sraj struct cryptoini *enc, struct cryptoini *mac) 1800193579Sraj{ 1801193579Sraj 1802193579Sraj if (mac && sec_mdeu_can_handle(mac->cri_alg)) 1803193579Sraj return (0); 1804193579Sraj 1805193579Sraj return (-1); 1806193579Sraj} 1807193579Sraj 1808193579Srajstatic int 1809193579Srajsec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses, 1810193579Sraj struct sec_desc *desc, struct cryptop *crp, int buftype) 1811193579Sraj{ 1812193579Sraj struct cryptodesc *enc, *mac; 1813193579Sraj struct sec_hw_desc *hd = desc->sd_desc; 1814193579Sraj u_int eu, mode, hashlen; 1815193579Sraj int error; 1816193579Sraj 1817193579Sraj error = sec_split_crp(crp, &enc, &mac); 1818193579Sraj if (error) 1819193579Sraj return (error); 1820193579Sraj 1821193579Sraj if (enc) 1822193579Sraj return (EINVAL); 1823193579Sraj 1824193579Sraj error = sec_mdeu_config(mac, &eu, &mode, &hashlen); 1825193579Sraj if (error) 1826193579Sraj return (error); 1827193579Sraj 1828193579Sraj hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 1829193579Sraj hd->shd_eu_sel0 = eu; 1830193579Sraj hd->shd_mode0 = mode; 1831193579Sraj hd->shd_eu_sel1 = SEC_EU_NONE; 1832193579Sraj hd->shd_mode1 = 0; 1833193579Sraj 1834193579Sraj /* Pointer 0: NULL */ 1835193579Sraj error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 1836193579Sraj if (error) 1837193579Sraj return (error); 1838193579Sraj 1839193579Sraj /* Pointer 1: Context In (Not used: NULL) */ 1840193579Sraj error = sec_make_pointer_direct(sc, desc, 1, 0, 0); 1841193579Sraj if (error) 1842193579Sraj return (error); 1843193579Sraj 1844193579Sraj /* Pointer 2: HMAC Key (or NULL, depending on digest type) */ 1845193579Sraj if (hd->shd_mode0 & SEC_MDEU_MODE_HMAC) 1846193579Sraj error = sec_make_pointer_direct(sc, desc, 2, 1847193579Sraj desc->sd_desc_paddr + offsetof(struct sec_hw_desc, 1848193579Sraj shd_mkey), ses->ss_mklen); 1849193579Sraj else 1850193579Sraj error = sec_make_pointer_direct(sc, desc, 2, 0, 0); 1851193579Sraj 1852193579Sraj if (error) 1853193579Sraj return (error); 1854193579Sraj 1855193579Sraj /* Pointer 3: Input Data */ 1856193579Sraj error = sec_make_pointer(sc, desc, 3, crp->crp_buf, mac->crd_skip, 1857193579Sraj mac->crd_len, buftype); 1858193579Sraj if (error) 1859193579Sraj return (error); 1860193579Sraj 1861193579Sraj /* Pointer 4: NULL */ 1862193579Sraj error = sec_make_pointer_direct(sc, desc, 4, 0, 0); 1863193579Sraj if (error) 1864193579Sraj return (error); 1865193579Sraj 1866193579Sraj /* Pointer 5: Hash out */ 1867193579Sraj error = sec_make_pointer(sc, desc, 5, crp->crp_buf, 1868193579Sraj mac->crd_inject, hashlen, buftype); 1869193579Sraj if (error) 1870193579Sraj return (error); 1871193579Sraj 1872193579Sraj /* Pointer 6: NULL */ 1873193579Sraj error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 1874193579Sraj 1875193579Sraj return (0); 1876193579Sraj} 1877