1227730Sraj/*- 2227730Sraj * Copyright (C) 2009-2011 Semihalf. 3227730Sraj * All rights reserved. 4227730Sraj * 5227730Sraj * Redistribution and use in source and binary forms, with or without 6227730Sraj * modification, are permitted provided that the following conditions 7227730Sraj * are met: 8227730Sraj * 1. Redistributions of source code must retain the above copyright 9227730Sraj * notice, this list of conditions and the following disclaimer. 10227730Sraj * 2. Redistributions in binary form must reproduce the above copyright 11227730Sraj * notice, this list of conditions and the following disclaimer in the 12227730Sraj * documentation and/or other materials provided with the distribution. 13227730Sraj * 14227730Sraj * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15227730Sraj * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16227730Sraj * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17227730Sraj * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18227730Sraj * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19227730Sraj * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20227730Sraj * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21227730Sraj * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22227730Sraj * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23227730Sraj * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24227730Sraj * SUCH DAMAGE. 25227730Sraj */ 26227730Sraj 27227730Sraj/* 28227730Sraj * CESA SRAM Memory Map: 29227730Sraj * 30301220Szbb * +------------------------+ <= sc->sc_sram_base_va + CESA_SRAM_SIZE 31227730Sraj * | | 32227730Sraj * | DATA | 33227730Sraj * | | 34301220Szbb * +------------------------+ <= sc->sc_sram_base_va + CESA_DATA(0) 35227730Sraj * | struct cesa_sa_data | 36227730Sraj * +------------------------+ 37227730Sraj * | struct cesa_sa_hdesc | 38301220Szbb * +------------------------+ <= sc->sc_sram_base_va 39227730Sraj */ 40227730Sraj 41227730Sraj#include <sys/cdefs.h> 42227730Sraj__FBSDID("$FreeBSD$"); 43227730Sraj 44227730Sraj#include <sys/param.h> 45227730Sraj#include <sys/systm.h> 46227730Sraj#include <sys/bus.h> 47227730Sraj#include <sys/endian.h> 48227730Sraj#include <sys/kernel.h> 49227730Sraj#include <sys/lock.h> 50227730Sraj#include <sys/mbuf.h> 51227730Sraj#include <sys/module.h> 52227730Sraj#include <sys/mutex.h> 53227730Sraj#include <sys/rman.h> 54227730Sraj 55227730Sraj#include <machine/bus.h> 56227730Sraj#include <machine/intr.h> 57227730Sraj#include <machine/resource.h> 58301220Szbb#include <machine/fdt.h> 59227730Sraj 60227730Sraj#include <dev/fdt/fdt_common.h> 61227730Sraj#include <dev/ofw/ofw_bus.h> 62227730Sraj#include <dev/ofw/ofw_bus_subr.h> 63227730Sraj 64227730Sraj#include <sys/md5.h> 65227730Sraj#include <crypto/sha1.h> 66301224Szbb#include <crypto/sha2/sha256.h> 67227730Sraj#include <crypto/rijndael/rijndael.h> 68227730Sraj#include <opencrypto/cryptodev.h> 69227730Sraj#include "cryptodev_if.h" 70227730Sraj 71227730Sraj#include <arm/mv/mvreg.h> 72227730Sraj#include <arm/mv/mvwin.h> 73227730Sraj#include <arm/mv/mvvar.h> 74227730Sraj#include "cesa.h" 75227730Sraj 76227730Srajstatic int cesa_probe(device_t); 77227730Srajstatic int cesa_attach(device_t); 78227730Srajstatic int cesa_detach(device_t); 79227730Srajstatic void cesa_intr(void *); 80227730Srajstatic int cesa_newsession(device_t, u_int32_t *, struct cryptoini *); 81227730Srajstatic int cesa_freesession(device_t, u_int64_t); 82227730Srajstatic int cesa_process(device_t, struct cryptop *, int); 83250291Sgberstatic int decode_win_cesa_setup(struct cesa_softc *sc); 84227730Sraj 85227730Srajstatic struct resource_spec cesa_res_spec[] = { 86227730Sraj { SYS_RES_MEMORY, 0, RF_ACTIVE }, 87301222Szbb { SYS_RES_MEMORY, 1, RF_ACTIVE }, 88227730Sraj { SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE }, 89227730Sraj { -1, 0 } 90227730Sraj}; 91227730Sraj 92227730Srajstatic device_method_t cesa_methods[] = { 93227730Sraj /* Device interface */ 94227730Sraj DEVMETHOD(device_probe, cesa_probe), 95227730Sraj DEVMETHOD(device_attach, cesa_attach), 96227730Sraj DEVMETHOD(device_detach, cesa_detach), 97227730Sraj 98227730Sraj /* Crypto device methods */ 99227730Sraj DEVMETHOD(cryptodev_newsession, cesa_newsession), 100227730Sraj DEVMETHOD(cryptodev_freesession,cesa_freesession), 101227730Sraj DEVMETHOD(cryptodev_process, cesa_process), 102227730Sraj 103227843Smarius DEVMETHOD_END 104227730Sraj}; 105227730Sraj 106227730Srajstatic driver_t cesa_driver = { 107227730Sraj "cesa", 108227730Sraj cesa_methods, 109227730Sraj sizeof (struct cesa_softc) 110227730Sraj}; 111227730Srajstatic devclass_t cesa_devclass; 112227730Sraj 113227730SrajDRIVER_MODULE(cesa, simplebus, cesa_driver, cesa_devclass, 0, 0); 114227730SrajMODULE_DEPEND(cesa, crypto, 1, 1, 1); 115227730Sraj 116227730Srajstatic void 117227730Srajcesa_dump_cshd(struct cesa_softc *sc, struct cesa_sa_hdesc *cshd) 118227730Sraj{ 119227730Sraj#ifdef DEBUG 120227730Sraj device_t dev; 121227730Sraj 122227730Sraj dev = sc->sc_dev; 123227730Sraj device_printf(dev, "CESA SA Hardware Descriptor:\n"); 124227730Sraj device_printf(dev, "\t\tconfig: 0x%08X\n", cshd->cshd_config); 125227730Sraj device_printf(dev, "\t\te_src: 0x%08X\n", cshd->cshd_enc_src); 126227730Sraj device_printf(dev, "\t\te_dst: 0x%08X\n", cshd->cshd_enc_dst); 127227730Sraj device_printf(dev, "\t\te_dlen: 0x%08X\n", cshd->cshd_enc_dlen); 128227730Sraj device_printf(dev, "\t\te_key: 0x%08X\n", cshd->cshd_enc_key); 129227730Sraj device_printf(dev, "\t\te_iv_1: 0x%08X\n", cshd->cshd_enc_iv); 130227730Sraj device_printf(dev, "\t\te_iv_2: 0x%08X\n", cshd->cshd_enc_iv_buf); 131227730Sraj device_printf(dev, "\t\tm_src: 0x%08X\n", cshd->cshd_mac_src); 132227730Sraj device_printf(dev, "\t\tm_dst: 0x%08X\n", cshd->cshd_mac_dst); 133227730Sraj device_printf(dev, "\t\tm_dlen: 0x%08X\n", cshd->cshd_mac_dlen); 134227730Sraj device_printf(dev, "\t\tm_tlen: 0x%08X\n", cshd->cshd_mac_total_dlen); 135227730Sraj device_printf(dev, "\t\tm_iv_i: 0x%08X\n", cshd->cshd_mac_iv_in); 136227730Sraj device_printf(dev, "\t\tm_iv_o: 0x%08X\n", cshd->cshd_mac_iv_out); 137227730Sraj#endif 138227730Sraj} 139227730Sraj 140227730Srajstatic void 141227730Srajcesa_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 142227730Sraj{ 143227730Sraj struct cesa_dma_mem *cdm; 144227730Sraj 145227730Sraj if (error) 146227730Sraj return; 147227730Sraj 148227730Sraj KASSERT(nseg == 1, ("Got wrong number of DMA segments, should be 1.")); 149227730Sraj cdm = arg; 150227730Sraj cdm->cdm_paddr = segs->ds_addr; 151227730Sraj} 152227730Sraj 153227730Srajstatic int 154227730Srajcesa_alloc_dma_mem(struct cesa_softc *sc, struct cesa_dma_mem *cdm, 155227730Sraj bus_size_t size) 156227730Sraj{ 157227730Sraj int error; 158227730Sraj 159227730Sraj KASSERT(cdm->cdm_vaddr == NULL, 160227730Sraj ("%s(): DMA memory descriptor in use.", __func__)); 161227730Sraj 162232883Sscottl error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ 163227730Sraj PAGE_SIZE, 0, /* alignment, boundary */ 164227730Sraj BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 165227730Sraj BUS_SPACE_MAXADDR, /* highaddr */ 166227730Sraj NULL, NULL, /* filtfunc, filtfuncarg */ 167227730Sraj size, 1, /* maxsize, nsegments */ 168227730Sraj size, 0, /* maxsegsz, flags */ 169227730Sraj NULL, NULL, /* lockfunc, lockfuncarg */ 170227730Sraj &cdm->cdm_tag); /* dmat */ 171227730Sraj if (error) { 172227730Sraj device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 173227730Sraj " %i!\n", error); 174227730Sraj 175227730Sraj goto err1; 176227730Sraj } 177227730Sraj 178227730Sraj error = bus_dmamem_alloc(cdm->cdm_tag, &cdm->cdm_vaddr, 179227730Sraj BUS_DMA_NOWAIT | BUS_DMA_ZERO, &cdm->cdm_map); 180227730Sraj if (error) { 181227730Sraj device_printf(sc->sc_dev, "failed to allocate DMA safe" 182227730Sraj " memory, error %i!\n", error); 183227730Sraj 184227730Sraj goto err2; 185227730Sraj } 186227730Sraj 187227730Sraj error = bus_dmamap_load(cdm->cdm_tag, cdm->cdm_map, cdm->cdm_vaddr, 188227730Sraj size, cesa_alloc_dma_mem_cb, cdm, BUS_DMA_NOWAIT); 189227730Sraj if (error) { 190227730Sraj device_printf(sc->sc_dev, "cannot get address of the DMA" 191227730Sraj " memory, error %i\n", error); 192227730Sraj 193227730Sraj goto err3; 194227730Sraj } 195227730Sraj 196227730Sraj return (0); 197227730Srajerr3: 198227730Sraj bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); 199227730Srajerr2: 200227730Sraj bus_dma_tag_destroy(cdm->cdm_tag); 201227730Srajerr1: 202227730Sraj cdm->cdm_vaddr = NULL; 203227730Sraj return (error); 204227730Sraj} 205227730Sraj 206227730Srajstatic void 207227730Srajcesa_free_dma_mem(struct cesa_dma_mem *cdm) 208227730Sraj{ 209227730Sraj 210227730Sraj bus_dmamap_unload(cdm->cdm_tag, cdm->cdm_map); 211227730Sraj bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); 212227730Sraj bus_dma_tag_destroy(cdm->cdm_tag); 213227730Sraj cdm->cdm_vaddr = NULL; 214227730Sraj} 215227730Sraj 216227730Srajstatic void 217227730Srajcesa_sync_dma_mem(struct cesa_dma_mem *cdm, bus_dmasync_op_t op) 218227730Sraj{ 219227730Sraj 220227730Sraj /* Sync only if dma memory is valid */ 221227730Sraj if (cdm->cdm_vaddr != NULL) 222227730Sraj bus_dmamap_sync(cdm->cdm_tag, cdm->cdm_map, op); 223227730Sraj} 224227730Sraj 225227730Srajstatic void 226227730Srajcesa_sync_desc(struct cesa_softc *sc, bus_dmasync_op_t op) 227227730Sraj{ 228227730Sraj 229227730Sraj cesa_sync_dma_mem(&sc->sc_tdesc_cdm, op); 230227730Sraj cesa_sync_dma_mem(&sc->sc_sdesc_cdm, op); 231227730Sraj cesa_sync_dma_mem(&sc->sc_requests_cdm, op); 232227730Sraj} 233227730Sraj 234227730Srajstatic struct cesa_session * 235227730Srajcesa_alloc_session(struct cesa_softc *sc) 236227730Sraj{ 237227730Sraj struct cesa_session *cs; 238227730Sraj 239227730Sraj CESA_GENERIC_ALLOC_LOCKED(sc, cs, sessions); 240227730Sraj 241227730Sraj return (cs); 242227730Sraj} 243227730Sraj 244227730Srajstatic struct cesa_session * 245227730Srajcesa_get_session(struct cesa_softc *sc, uint32_t sid) 246227730Sraj{ 247227730Sraj 248227730Sraj if (sid >= CESA_SESSIONS) 249227730Sraj return (NULL); 250227730Sraj 251227730Sraj return (&sc->sc_sessions[sid]); 252227730Sraj} 253227730Sraj 254227730Srajstatic void 255227730Srajcesa_free_session(struct cesa_softc *sc, struct cesa_session *cs) 256227730Sraj{ 257227730Sraj 258227730Sraj CESA_GENERIC_FREE_LOCKED(sc, cs, sessions); 259227730Sraj} 260227730Sraj 261227730Srajstatic struct cesa_request * 262227730Srajcesa_alloc_request(struct cesa_softc *sc) 263227730Sraj{ 264227730Sraj struct cesa_request *cr; 265227730Sraj 266227730Sraj CESA_GENERIC_ALLOC_LOCKED(sc, cr, requests); 267227730Sraj if (!cr) 268227730Sraj return (NULL); 269227730Sraj 270227730Sraj STAILQ_INIT(&cr->cr_tdesc); 271227730Sraj STAILQ_INIT(&cr->cr_sdesc); 272227730Sraj 273227730Sraj return (cr); 274227730Sraj} 275227730Sraj 276227730Srajstatic void 277227730Srajcesa_free_request(struct cesa_softc *sc, struct cesa_request *cr) 278227730Sraj{ 279227730Sraj 280227730Sraj /* Free TDMA descriptors assigned to this request */ 281227730Sraj CESA_LOCK(sc, tdesc); 282227730Sraj STAILQ_CONCAT(&sc->sc_free_tdesc, &cr->cr_tdesc); 283227730Sraj CESA_UNLOCK(sc, tdesc); 284227730Sraj 285227730Sraj /* Free SA descriptors assigned to this request */ 286227730Sraj CESA_LOCK(sc, sdesc); 287227730Sraj STAILQ_CONCAT(&sc->sc_free_sdesc, &cr->cr_sdesc); 288227730Sraj CESA_UNLOCK(sc, sdesc); 289227730Sraj 290298955Spfg /* Unload DMA memory associated with request */ 291227730Sraj if (cr->cr_dmap_loaded) { 292227730Sraj bus_dmamap_unload(sc->sc_data_dtag, cr->cr_dmap); 293227730Sraj cr->cr_dmap_loaded = 0; 294227730Sraj } 295227730Sraj 296227730Sraj CESA_GENERIC_FREE_LOCKED(sc, cr, requests); 297227730Sraj} 298227730Sraj 299227730Srajstatic void 300227730Srajcesa_enqueue_request(struct cesa_softc *sc, struct cesa_request *cr) 301227730Sraj{ 302227730Sraj 303227730Sraj CESA_LOCK(sc, requests); 304227730Sraj STAILQ_INSERT_TAIL(&sc->sc_ready_requests, cr, cr_stq); 305227730Sraj CESA_UNLOCK(sc, requests); 306227730Sraj} 307227730Sraj 308227730Srajstatic struct cesa_tdma_desc * 309227730Srajcesa_alloc_tdesc(struct cesa_softc *sc) 310227730Sraj{ 311227730Sraj struct cesa_tdma_desc *ctd; 312227730Sraj 313227730Sraj CESA_GENERIC_ALLOC_LOCKED(sc, ctd, tdesc); 314227730Sraj 315227730Sraj if (!ctd) 316227730Sraj device_printf(sc->sc_dev, "TDMA descriptors pool exhaused. " 317227730Sraj "Consider increasing CESA_TDMA_DESCRIPTORS.\n"); 318227730Sraj 319227730Sraj return (ctd); 320227730Sraj} 321227730Sraj 322227730Srajstatic struct cesa_sa_desc * 323227730Srajcesa_alloc_sdesc(struct cesa_softc *sc, struct cesa_request *cr) 324227730Sraj{ 325227730Sraj struct cesa_sa_desc *csd; 326227730Sraj 327227730Sraj CESA_GENERIC_ALLOC_LOCKED(sc, csd, sdesc); 328227730Sraj if (!csd) { 329227730Sraj device_printf(sc->sc_dev, "SA descriptors pool exhaused. " 330227730Sraj "Consider increasing CESA_SA_DESCRIPTORS.\n"); 331227730Sraj return (NULL); 332227730Sraj } 333227730Sraj 334227730Sraj STAILQ_INSERT_TAIL(&cr->cr_sdesc, csd, csd_stq); 335227730Sraj 336227730Sraj /* Fill-in SA descriptor with default values */ 337227730Sraj csd->csd_cshd->cshd_enc_key = CESA_SA_DATA(csd_key); 338227730Sraj csd->csd_cshd->cshd_enc_iv = CESA_SA_DATA(csd_iv); 339227730Sraj csd->csd_cshd->cshd_enc_iv_buf = CESA_SA_DATA(csd_iv); 340227730Sraj csd->csd_cshd->cshd_enc_src = 0; 341227730Sraj csd->csd_cshd->cshd_enc_dst = 0; 342227730Sraj csd->csd_cshd->cshd_enc_dlen = 0; 343227730Sraj csd->csd_cshd->cshd_mac_dst = CESA_SA_DATA(csd_hash); 344227730Sraj csd->csd_cshd->cshd_mac_iv_in = CESA_SA_DATA(csd_hiv_in); 345227730Sraj csd->csd_cshd->cshd_mac_iv_out = CESA_SA_DATA(csd_hiv_out); 346227730Sraj csd->csd_cshd->cshd_mac_src = 0; 347227730Sraj csd->csd_cshd->cshd_mac_dlen = 0; 348227730Sraj 349227730Sraj return (csd); 350227730Sraj} 351227730Sraj 352227730Srajstatic struct cesa_tdma_desc * 353227730Srajcesa_tdma_copy(struct cesa_softc *sc, bus_addr_t dst, bus_addr_t src, 354227730Sraj bus_size_t size) 355227730Sraj{ 356227730Sraj struct cesa_tdma_desc *ctd; 357227730Sraj 358227730Sraj ctd = cesa_alloc_tdesc(sc); 359227730Sraj if (!ctd) 360227730Sraj return (NULL); 361227730Sraj 362227730Sraj ctd->ctd_cthd->cthd_dst = dst; 363227730Sraj ctd->ctd_cthd->cthd_src = src; 364227730Sraj ctd->ctd_cthd->cthd_byte_count = size; 365227730Sraj 366227730Sraj /* Handle special control packet */ 367227730Sraj if (size != 0) 368227730Sraj ctd->ctd_cthd->cthd_flags = CESA_CTHD_OWNED; 369227730Sraj else 370227730Sraj ctd->ctd_cthd->cthd_flags = 0; 371227730Sraj 372227730Sraj return (ctd); 373227730Sraj} 374227730Sraj 375227730Srajstatic struct cesa_tdma_desc * 376227730Srajcesa_tdma_copyin_sa_data(struct cesa_softc *sc, struct cesa_request *cr) 377227730Sraj{ 378227730Sraj 379301220Szbb return (cesa_tdma_copy(sc, sc->sc_sram_base_pa + 380227730Sraj sizeof(struct cesa_sa_hdesc), cr->cr_csd_paddr, 381227730Sraj sizeof(struct cesa_sa_data))); 382227730Sraj} 383227730Sraj 384227730Srajstatic struct cesa_tdma_desc * 385227730Srajcesa_tdma_copyout_sa_data(struct cesa_softc *sc, struct cesa_request *cr) 386227730Sraj{ 387227730Sraj 388301220Szbb return (cesa_tdma_copy(sc, cr->cr_csd_paddr, sc->sc_sram_base_pa + 389227730Sraj sizeof(struct cesa_sa_hdesc), sizeof(struct cesa_sa_data))); 390227730Sraj} 391227730Sraj 392227730Srajstatic struct cesa_tdma_desc * 393227730Srajcesa_tdma_copy_sdesc(struct cesa_softc *sc, struct cesa_sa_desc *csd) 394227730Sraj{ 395227730Sraj 396301220Szbb return (cesa_tdma_copy(sc, sc->sc_sram_base_pa, csd->csd_cshd_paddr, 397227730Sraj sizeof(struct cesa_sa_hdesc))); 398227730Sraj} 399227730Sraj 400227730Srajstatic void 401227730Srajcesa_append_tdesc(struct cesa_request *cr, struct cesa_tdma_desc *ctd) 402227730Sraj{ 403227730Sraj struct cesa_tdma_desc *ctd_prev; 404227730Sraj 405227730Sraj if (!STAILQ_EMPTY(&cr->cr_tdesc)) { 406227730Sraj ctd_prev = STAILQ_LAST(&cr->cr_tdesc, cesa_tdma_desc, ctd_stq); 407227730Sraj ctd_prev->ctd_cthd->cthd_next = ctd->ctd_cthd_paddr; 408227730Sraj } 409227730Sraj 410227730Sraj ctd->ctd_cthd->cthd_next = 0; 411227730Sraj STAILQ_INSERT_TAIL(&cr->cr_tdesc, ctd, ctd_stq); 412227730Sraj} 413227730Sraj 414227730Srajstatic int 415227730Srajcesa_append_packet(struct cesa_softc *sc, struct cesa_request *cr, 416227730Sraj struct cesa_packet *cp, struct cesa_sa_desc *csd) 417227730Sraj{ 418227730Sraj struct cesa_tdma_desc *ctd, *tmp; 419227730Sraj 420227730Sraj /* Copy SA descriptor for this packet */ 421227730Sraj ctd = cesa_tdma_copy_sdesc(sc, csd); 422227730Sraj if (!ctd) 423227730Sraj return (ENOMEM); 424227730Sraj 425227730Sraj cesa_append_tdesc(cr, ctd); 426227730Sraj 427227730Sraj /* Copy data to be processed */ 428227730Sraj STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyin, ctd_stq, tmp) 429227730Sraj cesa_append_tdesc(cr, ctd); 430227730Sraj STAILQ_INIT(&cp->cp_copyin); 431227730Sraj 432227730Sraj /* Insert control descriptor */ 433227730Sraj ctd = cesa_tdma_copy(sc, 0, 0, 0); 434227730Sraj if (!ctd) 435227730Sraj return (ENOMEM); 436227730Sraj 437227730Sraj cesa_append_tdesc(cr, ctd); 438227730Sraj 439227730Sraj /* Copy back results */ 440227730Sraj STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyout, ctd_stq, tmp) 441227730Sraj cesa_append_tdesc(cr, ctd); 442227730Sraj STAILQ_INIT(&cp->cp_copyout); 443227730Sraj 444227730Sraj return (0); 445227730Sraj} 446227730Sraj 447227730Srajstatic int 448227730Srajcesa_set_mkey(struct cesa_session *cs, int alg, const uint8_t *mkey, int mklen) 449227730Sraj{ 450227730Sraj uint8_t ipad[CESA_MAX_HMAC_BLOCK_LEN]; 451227730Sraj uint8_t opad[CESA_MAX_HMAC_BLOCK_LEN]; 452227730Sraj SHA1_CTX sha1ctx; 453301224Szbb SHA256_CTX sha256ctx; 454227730Sraj MD5_CTX md5ctx; 455227730Sraj uint32_t *hout; 456227730Sraj uint32_t *hin; 457227730Sraj int i; 458227730Sraj 459227730Sraj memset(ipad, HMAC_IPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN); 460227730Sraj memset(opad, HMAC_OPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN); 461227730Sraj for (i = 0; i < mklen; i++) { 462227730Sraj ipad[i] ^= mkey[i]; 463227730Sraj opad[i] ^= mkey[i]; 464227730Sraj } 465227730Sraj 466227730Sraj hin = (uint32_t *)cs->cs_hiv_in; 467227730Sraj hout = (uint32_t *)cs->cs_hiv_out; 468227730Sraj 469227730Sraj switch (alg) { 470227730Sraj case CRYPTO_MD5_HMAC: 471227730Sraj MD5Init(&md5ctx); 472227730Sraj MD5Update(&md5ctx, ipad, MD5_HMAC_BLOCK_LEN); 473227730Sraj memcpy(hin, md5ctx.state, sizeof(md5ctx.state)); 474227730Sraj MD5Init(&md5ctx); 475227730Sraj MD5Update(&md5ctx, opad, MD5_HMAC_BLOCK_LEN); 476227730Sraj memcpy(hout, md5ctx.state, sizeof(md5ctx.state)); 477227730Sraj break; 478227730Sraj case CRYPTO_SHA1_HMAC: 479227730Sraj SHA1Init(&sha1ctx); 480227730Sraj SHA1Update(&sha1ctx, ipad, SHA1_HMAC_BLOCK_LEN); 481227730Sraj memcpy(hin, sha1ctx.h.b32, sizeof(sha1ctx.h.b32)); 482227730Sraj SHA1Init(&sha1ctx); 483227730Sraj SHA1Update(&sha1ctx, opad, SHA1_HMAC_BLOCK_LEN); 484227730Sraj memcpy(hout, sha1ctx.h.b32, sizeof(sha1ctx.h.b32)); 485227730Sraj break; 486301224Szbb case CRYPTO_SHA2_256_HMAC: 487301224Szbb SHA256_Init(&sha256ctx); 488301224Szbb SHA256_Update(&sha256ctx, ipad, SHA2_256_HMAC_BLOCK_LEN); 489301224Szbb memcpy(hin, sha256ctx.state, sizeof(sha256ctx.state)); 490301224Szbb SHA256_Init(&sha256ctx); 491301224Szbb SHA256_Update(&sha256ctx, opad, SHA2_256_HMAC_BLOCK_LEN); 492301224Szbb memcpy(hout, sha256ctx.state, sizeof(sha256ctx.state)); 493301224Szbb break; 494227730Sraj default: 495227730Sraj return (EINVAL); 496227730Sraj } 497227730Sraj 498227730Sraj for (i = 0; i < CESA_MAX_HASH_LEN / sizeof(uint32_t); i++) { 499227730Sraj hin[i] = htobe32(hin[i]); 500227730Sraj hout[i] = htobe32(hout[i]); 501227730Sraj } 502227730Sraj 503227730Sraj return (0); 504227730Sraj} 505227730Sraj 506227730Srajstatic int 507227730Srajcesa_prep_aes_key(struct cesa_session *cs) 508227730Sraj{ 509227730Sraj uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)]; 510227730Sraj uint32_t *dkey; 511227730Sraj int i; 512227730Sraj 513227730Sraj rijndaelKeySetupEnc(ek, cs->cs_key, cs->cs_klen * 8); 514227730Sraj 515227730Sraj cs->cs_config &= ~CESA_CSH_AES_KLEN_MASK; 516227730Sraj dkey = (uint32_t *)cs->cs_aes_dkey; 517227730Sraj 518227730Sraj switch (cs->cs_klen) { 519227730Sraj case 16: 520227730Sraj cs->cs_config |= CESA_CSH_AES_KLEN_128; 521227730Sraj for (i = 0; i < 4; i++) 522227730Sraj *dkey++ = htobe32(ek[4 * 10 + i]); 523227730Sraj break; 524227730Sraj case 24: 525227730Sraj cs->cs_config |= CESA_CSH_AES_KLEN_192; 526227730Sraj for (i = 0; i < 4; i++) 527227730Sraj *dkey++ = htobe32(ek[4 * 12 + i]); 528227730Sraj for (i = 0; i < 2; i++) 529227730Sraj *dkey++ = htobe32(ek[4 * 11 + 2 + i]); 530227730Sraj break; 531227730Sraj case 32: 532227730Sraj cs->cs_config |= CESA_CSH_AES_KLEN_256; 533227730Sraj for (i = 0; i < 4; i++) 534227730Sraj *dkey++ = htobe32(ek[4 * 14 + i]); 535227730Sraj for (i = 0; i < 4; i++) 536227730Sraj *dkey++ = htobe32(ek[4 * 13 + i]); 537227730Sraj break; 538227730Sraj default: 539227730Sraj return (EINVAL); 540227730Sraj } 541227730Sraj 542227730Sraj return (0); 543227730Sraj} 544227730Sraj 545227730Srajstatic int 546227730Srajcesa_is_hash(int alg) 547227730Sraj{ 548227730Sraj 549227730Sraj switch (alg) { 550227730Sraj case CRYPTO_MD5: 551227730Sraj case CRYPTO_MD5_HMAC: 552227730Sraj case CRYPTO_SHA1: 553227730Sraj case CRYPTO_SHA1_HMAC: 554301224Szbb case CRYPTO_SHA2_256_HMAC: 555227730Sraj return (1); 556227730Sraj default: 557227730Sraj return (0); 558227730Sraj } 559227730Sraj} 560227730Sraj 561227730Srajstatic void 562227730Srajcesa_start_packet(struct cesa_packet *cp, unsigned int size) 563227730Sraj{ 564227730Sraj 565227730Sraj cp->cp_size = size; 566227730Sraj cp->cp_offset = 0; 567227730Sraj STAILQ_INIT(&cp->cp_copyin); 568227730Sraj STAILQ_INIT(&cp->cp_copyout); 569227730Sraj} 570227730Sraj 571227730Srajstatic int 572227730Srajcesa_fill_packet(struct cesa_softc *sc, struct cesa_packet *cp, 573227730Sraj bus_dma_segment_t *seg) 574227730Sraj{ 575227730Sraj struct cesa_tdma_desc *ctd; 576227730Sraj unsigned int bsize; 577227730Sraj 578227730Sraj /* Calculate size of block copy */ 579227730Sraj bsize = MIN(seg->ds_len, cp->cp_size - cp->cp_offset); 580227730Sraj 581227730Sraj if (bsize > 0) { 582301220Szbb ctd = cesa_tdma_copy(sc, sc->sc_sram_base_pa + 583227730Sraj CESA_DATA(cp->cp_offset), seg->ds_addr, bsize); 584227730Sraj if (!ctd) 585227730Sraj return (-ENOMEM); 586227730Sraj 587227730Sraj STAILQ_INSERT_TAIL(&cp->cp_copyin, ctd, ctd_stq); 588227730Sraj 589301220Szbb ctd = cesa_tdma_copy(sc, seg->ds_addr, sc->sc_sram_base_pa + 590227730Sraj CESA_DATA(cp->cp_offset), bsize); 591227730Sraj if (!ctd) 592227730Sraj return (-ENOMEM); 593227730Sraj 594227730Sraj STAILQ_INSERT_TAIL(&cp->cp_copyout, ctd, ctd_stq); 595227730Sraj 596227730Sraj seg->ds_len -= bsize; 597227730Sraj seg->ds_addr += bsize; 598227730Sraj cp->cp_offset += bsize; 599227730Sraj } 600227730Sraj 601227730Sraj return (bsize); 602227730Sraj} 603227730Sraj 604227730Srajstatic void 605227730Srajcesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 606227730Sraj{ 607227730Sraj unsigned int mpsize, fragmented; 608227730Sraj unsigned int mlen, mskip, tmlen; 609227730Sraj struct cesa_chain_info *cci; 610227730Sraj unsigned int elen, eskip; 611227730Sraj unsigned int skip, len; 612227730Sraj struct cesa_sa_desc *csd; 613227730Sraj struct cesa_request *cr; 614227730Sraj struct cesa_softc *sc; 615227730Sraj struct cesa_packet cp; 616227730Sraj bus_dma_segment_t seg; 617227730Sraj uint32_t config; 618227730Sraj int size; 619227730Sraj 620227730Sraj cci = arg; 621227730Sraj sc = cci->cci_sc; 622227730Sraj cr = cci->cci_cr; 623227730Sraj 624227730Sraj if (error) { 625227730Sraj cci->cci_error = error; 626227730Sraj return; 627227730Sraj } 628227730Sraj 629227730Sraj elen = cci->cci_enc ? cci->cci_enc->crd_len : 0; 630227730Sraj eskip = cci->cci_enc ? cci->cci_enc->crd_skip : 0; 631227730Sraj mlen = cci->cci_mac ? cci->cci_mac->crd_len : 0; 632227730Sraj mskip = cci->cci_mac ? cci->cci_mac->crd_skip : 0; 633227730Sraj 634227730Sraj if (elen && mlen && 635227730Sraj ((eskip > mskip && ((eskip - mskip) & (cr->cr_cs->cs_ivlen - 1))) || 636227730Sraj (mskip > eskip && ((mskip - eskip) & (cr->cr_cs->cs_mblen - 1))) || 637227730Sraj (eskip > (mskip + mlen)) || (mskip > (eskip + elen)))) { 638227730Sraj /* 639227730Sraj * Data alignment in the request does not meet CESA requiremnts 640227730Sraj * for combined encryption/decryption and hashing. We have to 641227730Sraj * split the request to separate operations and process them 642227730Sraj * one by one. 643227730Sraj */ 644227730Sraj config = cci->cci_config; 645227730Sraj if ((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC) { 646227730Sraj config &= ~CESA_CSHD_OP_MASK; 647227730Sraj 648227730Sraj cci->cci_config = config | CESA_CSHD_MAC; 649227730Sraj cci->cci_enc = NULL; 650227730Sraj cci->cci_mac = cr->cr_mac; 651227730Sraj cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 652227730Sraj 653227730Sraj cci->cci_config = config | CESA_CSHD_ENC; 654227730Sraj cci->cci_enc = cr->cr_enc; 655227730Sraj cci->cci_mac = NULL; 656227730Sraj cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 657227730Sraj } else { 658227730Sraj config &= ~CESA_CSHD_OP_MASK; 659227730Sraj 660227730Sraj cci->cci_config = config | CESA_CSHD_ENC; 661227730Sraj cci->cci_enc = cr->cr_enc; 662227730Sraj cci->cci_mac = NULL; 663227730Sraj cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 664227730Sraj 665227730Sraj cci->cci_config = config | CESA_CSHD_MAC; 666227730Sraj cci->cci_enc = NULL; 667227730Sraj cci->cci_mac = cr->cr_mac; 668227730Sraj cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 669227730Sraj } 670227730Sraj 671227730Sraj return; 672227730Sraj } 673227730Sraj 674227730Sraj tmlen = mlen; 675227730Sraj fragmented = 0; 676227730Sraj mpsize = CESA_MAX_PACKET_SIZE; 677227730Sraj mpsize &= ~((cr->cr_cs->cs_ivlen - 1) | (cr->cr_cs->cs_mblen - 1)); 678227730Sraj 679227730Sraj if (elen && mlen) { 680227730Sraj skip = MIN(eskip, mskip); 681227730Sraj len = MAX(elen + eskip, mlen + mskip) - skip; 682227730Sraj } else if (elen) { 683227730Sraj skip = eskip; 684227730Sraj len = elen; 685227730Sraj } else { 686227730Sraj skip = mskip; 687227730Sraj len = mlen; 688227730Sraj } 689227730Sraj 690227730Sraj /* Start first packet in chain */ 691227730Sraj cesa_start_packet(&cp, MIN(mpsize, len)); 692227730Sraj 693227730Sraj while (nseg-- && len > 0) { 694227730Sraj seg = *(segs++); 695227730Sraj 696227730Sraj /* 697227730Sraj * Skip data in buffer on which neither ENC nor MAC operation 698227730Sraj * is requested. 699227730Sraj */ 700227730Sraj if (skip > 0) { 701227730Sraj size = MIN(skip, seg.ds_len); 702227730Sraj skip -= size; 703227730Sraj 704227730Sraj seg.ds_addr += size; 705227730Sraj seg.ds_len -= size; 706227730Sraj 707227730Sraj if (eskip > 0) 708227730Sraj eskip -= size; 709227730Sraj 710227730Sraj if (mskip > 0) 711227730Sraj mskip -= size; 712227730Sraj 713227730Sraj if (seg.ds_len == 0) 714227730Sraj continue; 715227730Sraj } 716227730Sraj 717227730Sraj while (1) { 718227730Sraj /* 719227730Sraj * Fill in current packet with data. Break if there is 720227730Sraj * no more data in current DMA segment or an error 721298955Spfg * occurred. 722227730Sraj */ 723227730Sraj size = cesa_fill_packet(sc, &cp, &seg); 724227730Sraj if (size <= 0) { 725227730Sraj error = -size; 726227730Sraj break; 727227730Sraj } 728227730Sraj 729227730Sraj len -= size; 730227730Sraj 731227730Sraj /* If packet is full, append it to the chain */ 732227730Sraj if (cp.cp_size == cp.cp_offset) { 733227730Sraj csd = cesa_alloc_sdesc(sc, cr); 734227730Sraj if (!csd) { 735227730Sraj error = ENOMEM; 736227730Sraj break; 737227730Sraj } 738227730Sraj 739227730Sraj /* Create SA descriptor for this packet */ 740227730Sraj csd->csd_cshd->cshd_config = cci->cci_config; 741227730Sraj csd->csd_cshd->cshd_mac_total_dlen = tmlen; 742227730Sraj 743227730Sraj /* 744227730Sraj * Enable fragmentation if request will not fit 745227730Sraj * into one packet. 746227730Sraj */ 747227730Sraj if (len > 0) { 748227730Sraj if (!fragmented) { 749227730Sraj fragmented = 1; 750227730Sraj csd->csd_cshd->cshd_config |= 751227730Sraj CESA_CSHD_FRAG_FIRST; 752227730Sraj } else 753227730Sraj csd->csd_cshd->cshd_config |= 754227730Sraj CESA_CSHD_FRAG_MIDDLE; 755227730Sraj } else if (fragmented) 756227730Sraj csd->csd_cshd->cshd_config |= 757227730Sraj CESA_CSHD_FRAG_LAST; 758227730Sraj 759227730Sraj if (eskip < cp.cp_size && elen > 0) { 760227730Sraj csd->csd_cshd->cshd_enc_src = 761227730Sraj CESA_DATA(eskip); 762227730Sraj csd->csd_cshd->cshd_enc_dst = 763227730Sraj CESA_DATA(eskip); 764227730Sraj csd->csd_cshd->cshd_enc_dlen = 765227730Sraj MIN(elen, cp.cp_size - eskip); 766227730Sraj } 767227730Sraj 768227730Sraj if (mskip < cp.cp_size && mlen > 0) { 769227730Sraj csd->csd_cshd->cshd_mac_src = 770227730Sraj CESA_DATA(mskip); 771227730Sraj csd->csd_cshd->cshd_mac_dlen = 772227730Sraj MIN(mlen, cp.cp_size - mskip); 773227730Sraj } 774227730Sraj 775227730Sraj elen -= csd->csd_cshd->cshd_enc_dlen; 776227730Sraj eskip -= MIN(eskip, cp.cp_size); 777227730Sraj mlen -= csd->csd_cshd->cshd_mac_dlen; 778227730Sraj mskip -= MIN(mskip, cp.cp_size); 779227730Sraj 780227730Sraj cesa_dump_cshd(sc, csd->csd_cshd); 781227730Sraj 782227730Sraj /* Append packet to the request */ 783227730Sraj error = cesa_append_packet(sc, cr, &cp, csd); 784227730Sraj if (error) 785227730Sraj break; 786227730Sraj 787227730Sraj /* Start a new packet, as current is full */ 788227730Sraj cesa_start_packet(&cp, MIN(mpsize, len)); 789227730Sraj } 790227730Sraj } 791227730Sraj 792227730Sraj if (error) 793227730Sraj break; 794227730Sraj } 795227730Sraj 796227730Sraj if (error) { 797227730Sraj /* 798227730Sraj * Move all allocated resources to the request. They will be 799227730Sraj * freed later. 800227730Sraj */ 801227730Sraj STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyin); 802227730Sraj STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyout); 803227730Sraj cci->cci_error = error; 804227730Sraj } 805227730Sraj} 806227730Sraj 807227730Srajstatic void 808227730Srajcesa_create_chain_cb2(void *arg, bus_dma_segment_t *segs, int nseg, 809227730Sraj bus_size_t size, int error) 810227730Sraj{ 811227730Sraj 812227730Sraj cesa_create_chain_cb(arg, segs, nseg, error); 813227730Sraj} 814227730Sraj 815227730Srajstatic int 816227730Srajcesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr) 817227730Sraj{ 818227730Sraj struct cesa_chain_info cci; 819227730Sraj struct cesa_tdma_desc *ctd; 820227730Sraj uint32_t config; 821227730Sraj int error; 822227730Sraj 823227730Sraj error = 0; 824227730Sraj CESA_LOCK_ASSERT(sc, sessions); 825227730Sraj 826227730Sraj /* Create request metadata */ 827227730Sraj if (cr->cr_enc) { 828227730Sraj if (cr->cr_enc->crd_alg == CRYPTO_AES_CBC && 829227730Sraj (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0) 830227730Sraj memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_aes_dkey, 831227730Sraj cr->cr_cs->cs_klen); 832227730Sraj else 833227730Sraj memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_key, 834227730Sraj cr->cr_cs->cs_klen); 835227730Sraj } 836227730Sraj 837227730Sraj if (cr->cr_mac) { 838227730Sraj memcpy(cr->cr_csd->csd_hiv_in, cr->cr_cs->cs_hiv_in, 839227730Sraj CESA_MAX_HASH_LEN); 840227730Sraj memcpy(cr->cr_csd->csd_hiv_out, cr->cr_cs->cs_hiv_out, 841227730Sraj CESA_MAX_HASH_LEN); 842227730Sraj } 843227730Sraj 844227730Sraj ctd = cesa_tdma_copyin_sa_data(sc, cr); 845227730Sraj if (!ctd) 846227730Sraj return (ENOMEM); 847227730Sraj 848227730Sraj cesa_append_tdesc(cr, ctd); 849227730Sraj 850227730Sraj /* Prepare SA configuration */ 851227730Sraj config = cr->cr_cs->cs_config; 852227730Sraj 853227730Sraj if (cr->cr_enc && (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0) 854227730Sraj config |= CESA_CSHD_DECRYPT; 855227730Sraj if (cr->cr_enc && !cr->cr_mac) 856227730Sraj config |= CESA_CSHD_ENC; 857227730Sraj if (!cr->cr_enc && cr->cr_mac) 858227730Sraj config |= CESA_CSHD_MAC; 859227730Sraj if (cr->cr_enc && cr->cr_mac) 860227730Sraj config |= (config & CESA_CSHD_DECRYPT) ? CESA_CSHD_MAC_AND_ENC : 861227730Sraj CESA_CSHD_ENC_AND_MAC; 862227730Sraj 863227730Sraj /* Create data packets */ 864227730Sraj cci.cci_sc = sc; 865227730Sraj cci.cci_cr = cr; 866227730Sraj cci.cci_enc = cr->cr_enc; 867227730Sraj cci.cci_mac = cr->cr_mac; 868227730Sraj cci.cci_config = config; 869227730Sraj cci.cci_error = 0; 870227730Sraj 871227730Sraj if (cr->cr_crp->crp_flags & CRYPTO_F_IOV) 872227730Sraj error = bus_dmamap_load_uio(sc->sc_data_dtag, 873227730Sraj cr->cr_dmap, (struct uio *)cr->cr_crp->crp_buf, 874227730Sraj cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT); 875227730Sraj else if (cr->cr_crp->crp_flags & CRYPTO_F_IMBUF) 876227730Sraj error = bus_dmamap_load_mbuf(sc->sc_data_dtag, 877227730Sraj cr->cr_dmap, (struct mbuf *)cr->cr_crp->crp_buf, 878227730Sraj cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT); 879227730Sraj else 880227730Sraj error = bus_dmamap_load(sc->sc_data_dtag, 881227730Sraj cr->cr_dmap, cr->cr_crp->crp_buf, 882227730Sraj cr->cr_crp->crp_ilen, cesa_create_chain_cb, &cci, 883227730Sraj BUS_DMA_NOWAIT); 884227730Sraj 885227730Sraj if (!error) 886227730Sraj cr->cr_dmap_loaded = 1; 887227730Sraj 888227730Sraj if (cci.cci_error) 889227730Sraj error = cci.cci_error; 890227730Sraj 891227730Sraj if (error) 892227730Sraj return (error); 893227730Sraj 894227730Sraj /* Read back request metadata */ 895227730Sraj ctd = cesa_tdma_copyout_sa_data(sc, cr); 896227730Sraj if (!ctd) 897227730Sraj return (ENOMEM); 898227730Sraj 899227730Sraj cesa_append_tdesc(cr, ctd); 900227730Sraj 901227730Sraj return (0); 902227730Sraj} 903227730Sraj 904227730Srajstatic void 905227730Srajcesa_execute(struct cesa_softc *sc) 906227730Sraj{ 907227730Sraj struct cesa_tdma_desc *prev_ctd, *ctd; 908227730Sraj struct cesa_request *prev_cr, *cr; 909227730Sraj 910227730Sraj CESA_LOCK(sc, requests); 911227730Sraj 912227730Sraj /* 913227730Sraj * If ready list is empty, there is nothing to execute. If queued list 914227730Sraj * is not empty, the hardware is busy and we cannot start another 915227730Sraj * execution. 916227730Sraj */ 917227730Sraj if (STAILQ_EMPTY(&sc->sc_ready_requests) || 918227730Sraj !STAILQ_EMPTY(&sc->sc_queued_requests)) { 919227730Sraj CESA_UNLOCK(sc, requests); 920227730Sraj return; 921227730Sraj } 922227730Sraj 923227730Sraj /* Move all ready requests to queued list */ 924227730Sraj STAILQ_CONCAT(&sc->sc_queued_requests, &sc->sc_ready_requests); 925227730Sraj STAILQ_INIT(&sc->sc_ready_requests); 926227730Sraj 927227730Sraj /* Create one execution chain from all requests on the list */ 928227730Sraj if (STAILQ_FIRST(&sc->sc_queued_requests) != 929227730Sraj STAILQ_LAST(&sc->sc_queued_requests, cesa_request, cr_stq)) { 930227730Sraj prev_cr = NULL; 931227730Sraj cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_POSTREAD | 932227730Sraj BUS_DMASYNC_POSTWRITE); 933227730Sraj 934227730Sraj STAILQ_FOREACH(cr, &sc->sc_queued_requests, cr_stq) { 935227730Sraj if (prev_cr) { 936227730Sraj ctd = STAILQ_FIRST(&cr->cr_tdesc); 937227730Sraj prev_ctd = STAILQ_LAST(&prev_cr->cr_tdesc, 938227730Sraj cesa_tdma_desc, ctd_stq); 939227730Sraj 940227730Sraj prev_ctd->ctd_cthd->cthd_next = 941227730Sraj ctd->ctd_cthd_paddr; 942227730Sraj } 943227730Sraj 944227730Sraj prev_cr = cr; 945227730Sraj } 946227730Sraj 947227730Sraj cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_PREREAD | 948227730Sraj BUS_DMASYNC_PREWRITE); 949227730Sraj } 950227730Sraj 951227730Sraj /* Start chain execution in hardware */ 952227730Sraj cr = STAILQ_FIRST(&sc->sc_queued_requests); 953227730Sraj ctd = STAILQ_FIRST(&cr->cr_tdesc); 954227730Sraj 955301222Szbb CESA_TDMA_WRITE(sc, CESA_TDMA_ND, ctd->ctd_cthd_paddr); 956301224Szbb#if defined (SOC_MV_ARMADA38X) 957301224Szbb CESA_REG_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE | CESA_SA_CMD_SHA2); 958301224Szbb#else 959301222Szbb CESA_REG_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE); 960301224Szbb#endif 961227730Sraj 962227730Sraj CESA_UNLOCK(sc, requests); 963227730Sraj} 964227730Sraj 965227730Srajstatic int 966227730Srajcesa_setup_sram(struct cesa_softc *sc) 967227730Sraj{ 968227730Sraj phandle_t sram_node; 969227730Sraj ihandle_t sram_ihandle; 970301220Szbb pcell_t sram_handle, sram_reg[2]; 971301220Szbb int rv; 972227730Sraj 973301282Szbb rv = OF_getencprop(ofw_bus_get_node(sc->sc_dev), "sram-handle", 974301220Szbb (void *)&sram_handle, sizeof(sram_handle)); 975301220Szbb if (rv <= 0) 976301220Szbb return (rv); 977227730Sraj 978227730Sraj sram_ihandle = (ihandle_t)sram_handle; 979227730Sraj sram_node = OF_instance_to_package(sram_ihandle); 980227730Sraj 981301282Szbb rv = OF_getencprop(sram_node, "reg", (void *)sram_reg, sizeof(sram_reg)); 982301220Szbb if (rv <= 0) 983301220Szbb return (rv); 984227730Sraj 985301282Szbb sc->sc_sram_base_pa = sram_reg[0]; 986301220Szbb /* Store SRAM size to be able to unmap in detach() */ 987301282Szbb sc->sc_sram_size = sram_reg[1]; 988227730Sraj 989301220Szbb#if defined(SOC_MV_ARMADA38X) 990301282Szbb void *sram_va; 991301282Szbb 992301220Szbb /* SRAM memory was not mapped in platform_sram_devmap(), map it now */ 993301282Szbb sram_va = pmap_mapdev(sc->sc_sram_base_pa, sc->sc_sram_size); 994301282Szbb if (sram_va == NULL) 995301282Szbb return (ENOMEM); 996301282Szbb sc->sc_sram_base_va = (vm_offset_t)sram_va; 997301220Szbb#endif 998227730Sraj return (0); 999227730Sraj} 1000227730Sraj 1001227730Srajstatic int 1002227730Srajcesa_probe(device_t dev) 1003227730Sraj{ 1004261410Sian 1005261410Sian if (!ofw_bus_status_okay(dev)) 1006261410Sian return (ENXIO); 1007261410Sian 1008227730Sraj if (!ofw_bus_is_compatible(dev, "mrvl,cesa")) 1009227730Sraj return (ENXIO); 1010227730Sraj 1011227730Sraj device_set_desc(dev, "Marvell Cryptographic Engine and Security " 1012227730Sraj "Accelerator"); 1013227730Sraj 1014227730Sraj return (BUS_PROBE_DEFAULT); 1015227730Sraj} 1016227730Sraj 1017227730Srajstatic int 1018227730Srajcesa_attach(device_t dev) 1019227730Sraj{ 1020227730Sraj struct cesa_softc *sc; 1021227730Sraj uint32_t d, r; 1022227730Sraj int error; 1023227730Sraj int i; 1024227730Sraj 1025227730Sraj sc = device_get_softc(dev); 1026227730Sraj sc->sc_blocked = 0; 1027227730Sraj sc->sc_error = 0; 1028227730Sraj sc->sc_dev = dev; 1029227730Sraj 1030250291Sgber /* Check if CESA peripheral device has power turned on */ 1031257326Srrs#if defined(SOC_MV_KIRKWOOD) 1032257326Srrs if (soc_power_ctrl_get(CPU_PM_CTRL_CRYPTO) == CPU_PM_CTRL_CRYPTO) { 1033257326Srrs device_printf(dev, "not powered on\n"); 1034257326Srrs return (ENXIO); 1035257326Srrs } 1036257326Srrs#else 1037250291Sgber if (soc_power_ctrl_get(CPU_PM_CTRL_CRYPTO) != CPU_PM_CTRL_CRYPTO) { 1038250291Sgber device_printf(dev, "not powered on\n"); 1039250291Sgber return (ENXIO); 1040227730Sraj } 1041257326Srrs#endif 1042227730Sraj soc_id(&d, &r); 1043227730Sraj 1044227730Sraj switch (d) { 1045227730Sraj case MV_DEV_88F6281: 1046238873Shrs case MV_DEV_88F6282: 1047301225Szbb case MV_DEV_88F6828: 1048227730Sraj sc->sc_tperr = 0; 1049227730Sraj break; 1050227730Sraj case MV_DEV_MV78100: 1051227730Sraj case MV_DEV_MV78100_Z0: 1052227730Sraj sc->sc_tperr = CESA_ICR_TPERR; 1053227730Sraj break; 1054227730Sraj default: 1055227730Sraj return (ENXIO); 1056227730Sraj } 1057227730Sraj 1058227730Sraj /* Initialize mutexes */ 1059227730Sraj mtx_init(&sc->sc_sc_lock, device_get_nameunit(dev), 1060227730Sraj "CESA Shared Data", MTX_DEF); 1061227730Sraj mtx_init(&sc->sc_tdesc_lock, device_get_nameunit(dev), 1062227730Sraj "CESA TDMA Descriptors Pool", MTX_DEF); 1063227730Sraj mtx_init(&sc->sc_sdesc_lock, device_get_nameunit(dev), 1064227730Sraj "CESA SA Descriptors Pool", MTX_DEF); 1065227730Sraj mtx_init(&sc->sc_requests_lock, device_get_nameunit(dev), 1066227730Sraj "CESA Requests Pool", MTX_DEF); 1067227730Sraj mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev), 1068227730Sraj "CESA Sessions Pool", MTX_DEF); 1069227730Sraj 1070227730Sraj /* Allocate I/O and IRQ resources */ 1071227730Sraj error = bus_alloc_resources(dev, cesa_res_spec, sc->sc_res); 1072227730Sraj if (error) { 1073227730Sraj device_printf(dev, "could not allocate resources\n"); 1074227730Sraj goto err0; 1075227730Sraj } 1076227730Sraj 1077250291Sgber /* Setup CESA decoding windows */ 1078250291Sgber error = decode_win_cesa_setup(sc); 1079250291Sgber if (error) { 1080250291Sgber device_printf(dev, "could not setup decoding windows\n"); 1081250291Sgber goto err1; 1082250291Sgber } 1083250291Sgber 1084250291Sgber /* Acquire SRAM base address */ 1085250291Sgber error = cesa_setup_sram(sc); 1086250291Sgber if (error) { 1087250291Sgber device_printf(dev, "could not setup SRAM\n"); 1088250291Sgber goto err1; 1089250291Sgber } 1090250291Sgber 1091227730Sraj /* Setup interrupt handler */ 1092301222Szbb error = bus_setup_intr(dev, sc->sc_res[RES_CESA_IRQ], INTR_TYPE_NET | 1093301222Szbb INTR_MPSAFE, NULL, cesa_intr, sc, &(sc->sc_icookie)); 1094227730Sraj if (error) { 1095227730Sraj device_printf(dev, "could not setup engine completion irq\n"); 1096301220Szbb goto err2; 1097227730Sraj } 1098227730Sraj 1099227730Sraj /* Create DMA tag for processed data */ 1100232883Sscottl error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 1101227730Sraj 1, 0, /* alignment, boundary */ 1102227730Sraj BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 1103227730Sraj BUS_SPACE_MAXADDR, /* highaddr */ 1104227730Sraj NULL, NULL, /* filtfunc, filtfuncarg */ 1105227730Sraj CESA_MAX_REQUEST_SIZE, /* maxsize */ 1106227730Sraj CESA_MAX_FRAGMENTS, /* nsegments */ 1107227730Sraj CESA_MAX_REQUEST_SIZE, 0, /* maxsegsz, flags */ 1108227730Sraj NULL, NULL, /* lockfunc, lockfuncarg */ 1109227730Sraj &sc->sc_data_dtag); /* dmat */ 1110227730Sraj if (error) 1111301220Szbb goto err3; 1112227730Sraj 1113227730Sraj /* Initialize data structures: TDMA Descriptors Pool */ 1114227730Sraj error = cesa_alloc_dma_mem(sc, &sc->sc_tdesc_cdm, 1115227730Sraj CESA_TDMA_DESCRIPTORS * sizeof(struct cesa_tdma_hdesc)); 1116227730Sraj if (error) 1117301220Szbb goto err4; 1118227730Sraj 1119227730Sraj STAILQ_INIT(&sc->sc_free_tdesc); 1120227730Sraj for (i = 0; i < CESA_TDMA_DESCRIPTORS; i++) { 1121227730Sraj sc->sc_tdesc[i].ctd_cthd = 1122227730Sraj (struct cesa_tdma_hdesc *)(sc->sc_tdesc_cdm.cdm_vaddr) + i; 1123227730Sraj sc->sc_tdesc[i].ctd_cthd_paddr = sc->sc_tdesc_cdm.cdm_paddr + 1124227730Sraj (i * sizeof(struct cesa_tdma_hdesc)); 1125227730Sraj STAILQ_INSERT_TAIL(&sc->sc_free_tdesc, &sc->sc_tdesc[i], 1126227730Sraj ctd_stq); 1127227730Sraj } 1128227730Sraj 1129227730Sraj /* Initialize data structures: SA Descriptors Pool */ 1130227730Sraj error = cesa_alloc_dma_mem(sc, &sc->sc_sdesc_cdm, 1131227730Sraj CESA_SA_DESCRIPTORS * sizeof(struct cesa_sa_hdesc)); 1132227730Sraj if (error) 1133301220Szbb goto err5; 1134227730Sraj 1135227730Sraj STAILQ_INIT(&sc->sc_free_sdesc); 1136227730Sraj for (i = 0; i < CESA_SA_DESCRIPTORS; i++) { 1137227730Sraj sc->sc_sdesc[i].csd_cshd = 1138227730Sraj (struct cesa_sa_hdesc *)(sc->sc_sdesc_cdm.cdm_vaddr) + i; 1139227730Sraj sc->sc_sdesc[i].csd_cshd_paddr = sc->sc_sdesc_cdm.cdm_paddr + 1140227730Sraj (i * sizeof(struct cesa_sa_hdesc)); 1141227730Sraj STAILQ_INSERT_TAIL(&sc->sc_free_sdesc, &sc->sc_sdesc[i], 1142227730Sraj csd_stq); 1143227730Sraj } 1144227730Sraj 1145227730Sraj /* Initialize data structures: Requests Pool */ 1146227730Sraj error = cesa_alloc_dma_mem(sc, &sc->sc_requests_cdm, 1147227730Sraj CESA_REQUESTS * sizeof(struct cesa_sa_data)); 1148227730Sraj if (error) 1149301220Szbb goto err6; 1150227730Sraj 1151227730Sraj STAILQ_INIT(&sc->sc_free_requests); 1152227730Sraj STAILQ_INIT(&sc->sc_ready_requests); 1153227730Sraj STAILQ_INIT(&sc->sc_queued_requests); 1154227730Sraj for (i = 0; i < CESA_REQUESTS; i++) { 1155227730Sraj sc->sc_requests[i].cr_csd = 1156227730Sraj (struct cesa_sa_data *)(sc->sc_requests_cdm.cdm_vaddr) + i; 1157227730Sraj sc->sc_requests[i].cr_csd_paddr = 1158227730Sraj sc->sc_requests_cdm.cdm_paddr + 1159227730Sraj (i * sizeof(struct cesa_sa_data)); 1160227730Sraj 1161227730Sraj /* Preallocate DMA maps */ 1162227730Sraj error = bus_dmamap_create(sc->sc_data_dtag, 0, 1163227730Sraj &sc->sc_requests[i].cr_dmap); 1164227730Sraj if (error && i > 0) { 1165227730Sraj i--; 1166227730Sraj do { 1167227730Sraj bus_dmamap_destroy(sc->sc_data_dtag, 1168227730Sraj sc->sc_requests[i].cr_dmap); 1169227730Sraj } while (i--); 1170227730Sraj 1171301220Szbb goto err7; 1172227730Sraj } 1173227730Sraj 1174227730Sraj STAILQ_INSERT_TAIL(&sc->sc_free_requests, &sc->sc_requests[i], 1175227730Sraj cr_stq); 1176227730Sraj } 1177227730Sraj 1178227730Sraj /* Initialize data structures: Sessions Pool */ 1179227730Sraj STAILQ_INIT(&sc->sc_free_sessions); 1180227730Sraj for (i = 0; i < CESA_SESSIONS; i++) { 1181227730Sraj sc->sc_sessions[i].cs_sid = i; 1182227730Sraj STAILQ_INSERT_TAIL(&sc->sc_free_sessions, &sc->sc_sessions[i], 1183227730Sraj cs_stq); 1184227730Sraj } 1185227730Sraj 1186227730Sraj /* 1187227730Sraj * Initialize TDMA: 1188227730Sraj * - Burst limit: 128 bytes, 1189227730Sraj * - Outstanding reads enabled, 1190227730Sraj * - No byte-swap. 1191227730Sraj */ 1192301222Szbb CESA_TDMA_WRITE(sc, CESA_TDMA_CR, CESA_TDMA_CR_DBL128 | 1193301222Szbb CESA_TDMA_CR_SBL128 | CESA_TDMA_CR_ORDEN | CESA_TDMA_CR_NBS | 1194301224Szbb#if defined (SOC_MV_ARMADA38X) 1195301224Szbb CESA_TDMA_NUM_OUTSTAND | 1196301224Szbb#endif 1197301222Szbb CESA_TDMA_CR_ENABLE); 1198227730Sraj 1199227730Sraj /* 1200227730Sraj * Initialize SA: 1201227730Sraj * - SA descriptor is present at beginning of CESA SRAM, 1202227730Sraj * - Multi-packet chain mode, 1203227730Sraj * - Cooperation with TDMA enabled. 1204227730Sraj */ 1205301222Szbb CESA_REG_WRITE(sc, CESA_SA_DPR, 0); 1206301222Szbb CESA_REG_WRITE(sc, CESA_SA_CR, CESA_SA_CR_ACTIVATE_TDMA | 1207227730Sraj CESA_SA_CR_WAIT_FOR_TDMA | CESA_SA_CR_MULTI_MODE); 1208227730Sraj 1209227730Sraj /* Unmask interrupts */ 1210301222Szbb CESA_REG_WRITE(sc, CESA_ICR, 0); 1211301222Szbb CESA_REG_WRITE(sc, CESA_ICM, CESA_ICM_ACCTDMA | sc->sc_tperr); 1212301222Szbb CESA_TDMA_WRITE(sc, CESA_TDMA_ECR, 0); 1213301222Szbb CESA_TDMA_WRITE(sc, CESA_TDMA_EMR, CESA_TDMA_EMR_MISS | 1214227730Sraj CESA_TDMA_EMR_DOUBLE_HIT | CESA_TDMA_EMR_BOTH_HIT | 1215227730Sraj CESA_TDMA_EMR_DATA_ERROR); 1216227730Sraj 1217227730Sraj /* Register in OCF */ 1218227730Sraj sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); 1219301225Szbb if (sc->sc_cid < 0) { 1220227730Sraj device_printf(dev, "could not get crypto driver id\n"); 1221301220Szbb goto err8; 1222227730Sraj } 1223227730Sraj 1224227730Sraj crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); 1225227730Sraj crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); 1226227730Sraj crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); 1227227730Sraj crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0); 1228227730Sraj crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 1229227730Sraj crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0); 1230227730Sraj crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 1231301224Szbb crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0); 1232227730Sraj 1233227730Sraj return (0); 1234301220Szbberr8: 1235227730Sraj for (i = 0; i < CESA_REQUESTS; i++) 1236227730Sraj bus_dmamap_destroy(sc->sc_data_dtag, 1237227730Sraj sc->sc_requests[i].cr_dmap); 1238301220Szbberr7: 1239301220Szbb cesa_free_dma_mem(&sc->sc_requests_cdm); 1240227730Srajerr6: 1241301220Szbb cesa_free_dma_mem(&sc->sc_sdesc_cdm); 1242227730Srajerr5: 1243301220Szbb cesa_free_dma_mem(&sc->sc_tdesc_cdm); 1244227730Srajerr4: 1245301220Szbb bus_dma_tag_destroy(sc->sc_data_dtag); 1246227730Srajerr3: 1247301222Szbb bus_teardown_intr(dev, sc->sc_res[RES_CESA_IRQ], sc->sc_icookie); 1248227730Srajerr2: 1249301220Szbb#if defined(SOC_MV_ARMADA38X) 1250301282Szbb pmap_unmapdev(sc->sc_sram_base_va, sc->sc_sram_size); 1251301220Szbb#endif 1252227730Srajerr1: 1253227730Sraj bus_release_resources(dev, cesa_res_spec, sc->sc_res); 1254227730Srajerr0: 1255227730Sraj mtx_destroy(&sc->sc_sessions_lock); 1256227730Sraj mtx_destroy(&sc->sc_requests_lock); 1257227730Sraj mtx_destroy(&sc->sc_sdesc_lock); 1258227730Sraj mtx_destroy(&sc->sc_tdesc_lock); 1259227730Sraj mtx_destroy(&sc->sc_sc_lock); 1260227730Sraj return (ENXIO); 1261227730Sraj} 1262227730Sraj 1263227730Srajstatic int 1264227730Srajcesa_detach(device_t dev) 1265227730Sraj{ 1266227730Sraj struct cesa_softc *sc; 1267227730Sraj int i; 1268227730Sraj 1269227730Sraj sc = device_get_softc(dev); 1270227730Sraj 1271227730Sraj /* TODO: Wait for queued requests completion before shutdown. */ 1272227730Sraj 1273227730Sraj /* Mask interrupts */ 1274301222Szbb CESA_REG_WRITE(sc, CESA_ICM, 0); 1275301222Szbb CESA_TDMA_WRITE(sc, CESA_TDMA_EMR, 0); 1276227730Sraj 1277227730Sraj /* Unregister from OCF */ 1278227730Sraj crypto_unregister_all(sc->sc_cid); 1279227730Sraj 1280227730Sraj /* Free DMA Maps */ 1281227730Sraj for (i = 0; i < CESA_REQUESTS; i++) 1282227730Sraj bus_dmamap_destroy(sc->sc_data_dtag, 1283227730Sraj sc->sc_requests[i].cr_dmap); 1284227730Sraj 1285227730Sraj /* Free DMA Memory */ 1286227730Sraj cesa_free_dma_mem(&sc->sc_requests_cdm); 1287227730Sraj cesa_free_dma_mem(&sc->sc_sdesc_cdm); 1288227730Sraj cesa_free_dma_mem(&sc->sc_tdesc_cdm); 1289227730Sraj 1290227730Sraj /* Free DMA Tag */ 1291227730Sraj bus_dma_tag_destroy(sc->sc_data_dtag); 1292227730Sraj 1293227730Sraj /* Stop interrupt */ 1294301222Szbb bus_teardown_intr(dev, sc->sc_res[RES_CESA_IRQ], sc->sc_icookie); 1295227730Sraj 1296227730Sraj /* Relase I/O and IRQ resources */ 1297227730Sraj bus_release_resources(dev, cesa_res_spec, sc->sc_res); 1298227730Sraj 1299301220Szbb#if defined(SOC_MV_ARMADA38X) 1300301220Szbb /* Unmap SRAM memory */ 1301301282Szbb pmap_unmapdev(sc->sc_sram_base_va, sc->sc_sram_size); 1302301220Szbb#endif 1303298955Spfg /* Destroy mutexes */ 1304227730Sraj mtx_destroy(&sc->sc_sessions_lock); 1305227730Sraj mtx_destroy(&sc->sc_requests_lock); 1306227730Sraj mtx_destroy(&sc->sc_sdesc_lock); 1307227730Sraj mtx_destroy(&sc->sc_tdesc_lock); 1308227730Sraj mtx_destroy(&sc->sc_sc_lock); 1309227730Sraj 1310227730Sraj return (0); 1311227730Sraj} 1312227730Sraj 1313227730Srajstatic void 1314227730Srajcesa_intr(void *arg) 1315227730Sraj{ 1316227730Sraj STAILQ_HEAD(, cesa_request) requests; 1317227730Sraj struct cesa_request *cr, *tmp; 1318227730Sraj struct cesa_softc *sc; 1319227730Sraj uint32_t ecr, icr; 1320227730Sraj int blocked; 1321227730Sraj 1322227730Sraj sc = arg; 1323227730Sraj 1324227730Sraj /* Ack interrupt */ 1325301222Szbb ecr = CESA_TDMA_READ(sc, CESA_TDMA_ECR); 1326301222Szbb CESA_TDMA_WRITE(sc, CESA_TDMA_ECR, 0); 1327301222Szbb icr = CESA_REG_READ(sc, CESA_ICR); 1328301222Szbb CESA_REG_WRITE(sc, CESA_ICR, 0); 1329227730Sraj 1330227730Sraj /* Check for TDMA errors */ 1331227730Sraj if (ecr & CESA_TDMA_ECR_MISS) { 1332227730Sraj device_printf(sc->sc_dev, "TDMA Miss error detected!\n"); 1333227730Sraj sc->sc_error = EIO; 1334227730Sraj } 1335227730Sraj 1336227730Sraj if (ecr & CESA_TDMA_ECR_DOUBLE_HIT) { 1337227730Sraj device_printf(sc->sc_dev, "TDMA Double Hit error detected!\n"); 1338227730Sraj sc->sc_error = EIO; 1339227730Sraj } 1340227730Sraj 1341227730Sraj if (ecr & CESA_TDMA_ECR_BOTH_HIT) { 1342227730Sraj device_printf(sc->sc_dev, "TDMA Both Hit error detected!\n"); 1343227730Sraj sc->sc_error = EIO; 1344227730Sraj } 1345227730Sraj 1346227730Sraj if (ecr & CESA_TDMA_ECR_DATA_ERROR) { 1347227730Sraj device_printf(sc->sc_dev, "TDMA Data error detected!\n"); 1348227730Sraj sc->sc_error = EIO; 1349227730Sraj } 1350227730Sraj 1351227730Sraj /* Check for CESA errors */ 1352227730Sraj if (icr & sc->sc_tperr) { 1353227730Sraj device_printf(sc->sc_dev, "CESA SRAM Parity error detected!\n"); 1354227730Sraj sc->sc_error = EIO; 1355227730Sraj } 1356227730Sraj 1357227730Sraj /* If there is nothing more to do, return */ 1358227730Sraj if ((icr & CESA_ICR_ACCTDMA) == 0) 1359227730Sraj return; 1360227730Sraj 1361227730Sraj /* Get all finished requests */ 1362227730Sraj CESA_LOCK(sc, requests); 1363227730Sraj STAILQ_INIT(&requests); 1364227730Sraj STAILQ_CONCAT(&requests, &sc->sc_queued_requests); 1365227730Sraj STAILQ_INIT(&sc->sc_queued_requests); 1366227730Sraj CESA_UNLOCK(sc, requests); 1367227730Sraj 1368227730Sraj /* Execute all ready requests */ 1369227730Sraj cesa_execute(sc); 1370227730Sraj 1371227730Sraj /* Process completed requests */ 1372227730Sraj cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_POSTREAD | 1373227730Sraj BUS_DMASYNC_POSTWRITE); 1374227730Sraj 1375227730Sraj STAILQ_FOREACH_SAFE(cr, &requests, cr_stq, tmp) { 1376227730Sraj bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, 1377227730Sraj BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1378227730Sraj 1379227730Sraj cr->cr_crp->crp_etype = sc->sc_error; 1380227730Sraj if (cr->cr_mac) 1381227730Sraj crypto_copyback(cr->cr_crp->crp_flags, 1382227730Sraj cr->cr_crp->crp_buf, cr->cr_mac->crd_inject, 1383227730Sraj cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash); 1384227730Sraj 1385227730Sraj crypto_done(cr->cr_crp); 1386227730Sraj cesa_free_request(sc, cr); 1387227730Sraj } 1388227730Sraj 1389227730Sraj cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_PREREAD | 1390227730Sraj BUS_DMASYNC_PREWRITE); 1391227730Sraj 1392227730Sraj sc->sc_error = 0; 1393227730Sraj 1394227730Sraj /* Unblock driver if it ran out of resources */ 1395227730Sraj CESA_LOCK(sc, sc); 1396227730Sraj blocked = sc->sc_blocked; 1397227730Sraj sc->sc_blocked = 0; 1398227730Sraj CESA_UNLOCK(sc, sc); 1399227730Sraj 1400227730Sraj if (blocked) 1401227730Sraj crypto_unblock(sc->sc_cid, blocked); 1402227730Sraj} 1403227730Sraj 1404227730Srajstatic int 1405227730Srajcesa_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri) 1406227730Sraj{ 1407227730Sraj struct cesa_session *cs; 1408227730Sraj struct cesa_softc *sc; 1409227730Sraj struct cryptoini *enc; 1410227730Sraj struct cryptoini *mac; 1411227730Sraj int error; 1412227730Sraj 1413227730Sraj sc = device_get_softc(dev); 1414227730Sraj enc = NULL; 1415227730Sraj mac = NULL; 1416227730Sraj error = 0; 1417227730Sraj 1418227730Sraj /* Check and parse input */ 1419227730Sraj if (cesa_is_hash(cri->cri_alg)) 1420227730Sraj mac = cri; 1421227730Sraj else 1422227730Sraj enc = cri; 1423227730Sraj 1424227730Sraj cri = cri->cri_next; 1425227730Sraj 1426227730Sraj if (cri) { 1427227730Sraj if (!enc && !cesa_is_hash(cri->cri_alg)) 1428227730Sraj enc = cri; 1429227730Sraj 1430227730Sraj if (!mac && cesa_is_hash(cri->cri_alg)) 1431227730Sraj mac = cri; 1432227730Sraj 1433227730Sraj if (cri->cri_next || !(enc && mac)) 1434227730Sraj return (EINVAL); 1435227730Sraj } 1436227730Sraj 1437227730Sraj if ((enc && (enc->cri_klen / 8) > CESA_MAX_KEY_LEN) || 1438227730Sraj (mac && (mac->cri_klen / 8) > CESA_MAX_MKEY_LEN)) 1439227730Sraj return (E2BIG); 1440227730Sraj 1441227730Sraj /* Allocate session */ 1442227730Sraj cs = cesa_alloc_session(sc); 1443227730Sraj if (!cs) 1444227730Sraj return (ENOMEM); 1445227730Sraj 1446227730Sraj /* Prepare CESA configuration */ 1447227730Sraj cs->cs_config = 0; 1448227730Sraj cs->cs_ivlen = 1; 1449227730Sraj cs->cs_mblen = 1; 1450227730Sraj 1451227730Sraj if (enc) { 1452227730Sraj switch (enc->cri_alg) { 1453227730Sraj case CRYPTO_AES_CBC: 1454227730Sraj cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC; 1455227730Sraj cs->cs_ivlen = AES_BLOCK_LEN; 1456227730Sraj break; 1457227730Sraj case CRYPTO_DES_CBC: 1458227730Sraj cs->cs_config |= CESA_CSHD_DES | CESA_CSHD_CBC; 1459227730Sraj cs->cs_ivlen = DES_BLOCK_LEN; 1460227730Sraj break; 1461227730Sraj case CRYPTO_3DES_CBC: 1462227730Sraj cs->cs_config |= CESA_CSHD_3DES | CESA_CSHD_3DES_EDE | 1463227730Sraj CESA_CSHD_CBC; 1464227730Sraj cs->cs_ivlen = DES3_BLOCK_LEN; 1465227730Sraj break; 1466227730Sraj default: 1467227730Sraj error = EINVAL; 1468227730Sraj break; 1469227730Sraj } 1470227730Sraj } 1471227730Sraj 1472227730Sraj if (!error && mac) { 1473227730Sraj switch (mac->cri_alg) { 1474227730Sraj case CRYPTO_MD5: 1475301223Szbb cs->cs_mblen = 1; 1476301223Szbb cs->cs_hlen = (mac->cri_mlen == 0) ? MD5_HASH_LEN : 1477301223Szbb mac->cri_mlen; 1478227730Sraj cs->cs_config |= CESA_CSHD_MD5; 1479227730Sraj break; 1480227730Sraj case CRYPTO_MD5_HMAC: 1481301223Szbb cs->cs_mblen = MD5_HMAC_BLOCK_LEN; 1482301223Szbb cs->cs_hlen = (mac->cri_mlen == 0) ? MD5_HASH_LEN : 1483301223Szbb mac->cri_mlen; 1484227730Sraj cs->cs_config |= CESA_CSHD_MD5_HMAC; 1485301223Szbb if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN) 1486301223Szbb cs->cs_config |= CESA_CSHD_96_BIT_HMAC; 1487227730Sraj break; 1488227730Sraj case CRYPTO_SHA1: 1489301223Szbb cs->cs_mblen = 1; 1490301223Szbb cs->cs_hlen = (mac->cri_mlen == 0) ? SHA1_HASH_LEN : 1491301223Szbb mac->cri_mlen; 1492227730Sraj cs->cs_config |= CESA_CSHD_SHA1; 1493227730Sraj break; 1494227730Sraj case CRYPTO_SHA1_HMAC: 1495301223Szbb cs->cs_mblen = SHA1_HMAC_BLOCK_LEN; 1496301223Szbb cs->cs_hlen = (mac->cri_mlen == 0) ? SHA1_HASH_LEN : 1497301223Szbb mac->cri_mlen; 1498227730Sraj cs->cs_config |= CESA_CSHD_SHA1_HMAC; 1499301223Szbb if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN) 1500301223Szbb cs->cs_config |= CESA_CSHD_96_BIT_HMAC; 1501227730Sraj break; 1502301224Szbb case CRYPTO_SHA2_256_HMAC: 1503301224Szbb cs->cs_mblen = SHA2_256_HMAC_BLOCK_LEN; 1504301224Szbb cs->cs_hlen = (mac->cri_mlen == 0) ? SHA2_256_HASH_LEN : 1505301224Szbb mac->cri_mlen; 1506301224Szbb cs->cs_config |= CESA_CSHD_SHA2_256_HMAC; 1507301224Szbb break; 1508227730Sraj default: 1509227730Sraj error = EINVAL; 1510227730Sraj break; 1511227730Sraj } 1512227730Sraj } 1513227730Sraj 1514227730Sraj /* Save cipher key */ 1515227730Sraj if (!error && enc && enc->cri_key) { 1516227730Sraj cs->cs_klen = enc->cri_klen / 8; 1517227730Sraj memcpy(cs->cs_key, enc->cri_key, cs->cs_klen); 1518227730Sraj if (enc->cri_alg == CRYPTO_AES_CBC) 1519227730Sraj error = cesa_prep_aes_key(cs); 1520227730Sraj } 1521227730Sraj 1522227730Sraj /* Save digest key */ 1523227730Sraj if (!error && mac && mac->cri_key) 1524227730Sraj error = cesa_set_mkey(cs, mac->cri_alg, mac->cri_key, 1525227730Sraj mac->cri_klen / 8); 1526227730Sraj 1527227730Sraj if (error) { 1528227730Sraj cesa_free_session(sc, cs); 1529227730Sraj return (EINVAL); 1530227730Sraj } 1531227730Sraj 1532227730Sraj *sidp = cs->cs_sid; 1533227730Sraj 1534227730Sraj return (0); 1535227730Sraj} 1536227730Sraj 1537227730Srajstatic int 1538227730Srajcesa_freesession(device_t dev, uint64_t tid) 1539227730Sraj{ 1540227730Sraj struct cesa_session *cs; 1541227730Sraj struct cesa_softc *sc; 1542227730Sraj 1543227730Sraj sc = device_get_softc(dev); 1544227730Sraj cs = cesa_get_session(sc, CRYPTO_SESID2LID(tid)); 1545227730Sraj if (!cs) 1546227730Sraj return (EINVAL); 1547227730Sraj 1548227730Sraj /* Free session */ 1549227730Sraj cesa_free_session(sc, cs); 1550227730Sraj 1551227730Sraj return (0); 1552227730Sraj} 1553227730Sraj 1554227730Srajstatic int 1555227730Srajcesa_process(device_t dev, struct cryptop *crp, int hint) 1556227730Sraj{ 1557227730Sraj struct cesa_request *cr; 1558227730Sraj struct cesa_session *cs; 1559227730Sraj struct cryptodesc *crd; 1560227730Sraj struct cryptodesc *enc; 1561227730Sraj struct cryptodesc *mac; 1562227730Sraj struct cesa_softc *sc; 1563227730Sraj int error; 1564227730Sraj 1565227730Sraj sc = device_get_softc(dev); 1566227730Sraj crd = crp->crp_desc; 1567227730Sraj enc = NULL; 1568227730Sraj mac = NULL; 1569227730Sraj error = 0; 1570227730Sraj 1571227730Sraj /* Check session ID */ 1572227730Sraj cs = cesa_get_session(sc, CRYPTO_SESID2LID(crp->crp_sid)); 1573227730Sraj if (!cs) { 1574227730Sraj crp->crp_etype = EINVAL; 1575227730Sraj crypto_done(crp); 1576227730Sraj return (0); 1577227730Sraj } 1578227730Sraj 1579227730Sraj /* Check and parse input */ 1580227730Sraj if (crp->crp_ilen > CESA_MAX_REQUEST_SIZE) { 1581227730Sraj crp->crp_etype = E2BIG; 1582227730Sraj crypto_done(crp); 1583227730Sraj return (0); 1584227730Sraj } 1585227730Sraj 1586227730Sraj if (cesa_is_hash(crd->crd_alg)) 1587227730Sraj mac = crd; 1588227730Sraj else 1589227730Sraj enc = crd; 1590227730Sraj 1591227730Sraj crd = crd->crd_next; 1592227730Sraj 1593227730Sraj if (crd) { 1594227730Sraj if (!enc && !cesa_is_hash(crd->crd_alg)) 1595227730Sraj enc = crd; 1596227730Sraj 1597227730Sraj if (!mac && cesa_is_hash(crd->crd_alg)) 1598227730Sraj mac = crd; 1599227730Sraj 1600227730Sraj if (crd->crd_next || !(enc && mac)) { 1601227730Sraj crp->crp_etype = EINVAL; 1602227730Sraj crypto_done(crp); 1603227730Sraj return (0); 1604227730Sraj } 1605227730Sraj } 1606227730Sraj 1607227730Sraj /* 1608227730Sraj * Get request descriptor. Block driver if there is no free 1609227730Sraj * descriptors in pool. 1610227730Sraj */ 1611227730Sraj cr = cesa_alloc_request(sc); 1612227730Sraj if (!cr) { 1613227730Sraj CESA_LOCK(sc, sc); 1614227730Sraj sc->sc_blocked = CRYPTO_SYMQ; 1615227730Sraj CESA_UNLOCK(sc, sc); 1616227730Sraj return (ERESTART); 1617227730Sraj } 1618227730Sraj 1619227730Sraj /* Prepare request */ 1620227730Sraj cr->cr_crp = crp; 1621227730Sraj cr->cr_enc = enc; 1622227730Sraj cr->cr_mac = mac; 1623227730Sraj cr->cr_cs = cs; 1624227730Sraj 1625227730Sraj CESA_LOCK(sc, sessions); 1626227730Sraj cesa_sync_desc(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1627227730Sraj 1628227730Sraj if (enc && enc->crd_flags & CRD_F_ENCRYPT) { 1629227730Sraj if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1630227730Sraj memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen); 1631227730Sraj else 1632227730Sraj arc4rand(cr->cr_csd->csd_iv, cs->cs_ivlen, 0); 1633227730Sraj 1634227730Sraj if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0) 1635227730Sraj crypto_copyback(crp->crp_flags, crp->crp_buf, 1636227730Sraj enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv); 1637227730Sraj } else if (enc) { 1638227730Sraj if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1639227730Sraj memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen); 1640227730Sraj else 1641227730Sraj crypto_copydata(crp->crp_flags, crp->crp_buf, 1642227730Sraj enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv); 1643227730Sraj } 1644227730Sraj 1645227730Sraj if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) { 1646227730Sraj if ((enc->crd_klen / 8) <= CESA_MAX_KEY_LEN) { 1647227730Sraj cs->cs_klen = enc->crd_klen / 8; 1648227730Sraj memcpy(cs->cs_key, enc->crd_key, cs->cs_klen); 1649227730Sraj if (enc->crd_alg == CRYPTO_AES_CBC) 1650227730Sraj error = cesa_prep_aes_key(cs); 1651227730Sraj } else 1652227730Sraj error = E2BIG; 1653227730Sraj } 1654227730Sraj 1655227730Sraj if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) { 1656227730Sraj if ((mac->crd_klen / 8) <= CESA_MAX_MKEY_LEN) 1657227730Sraj error = cesa_set_mkey(cs, mac->crd_alg, mac->crd_key, 1658227730Sraj mac->crd_klen / 8); 1659227730Sraj else 1660227730Sraj error = E2BIG; 1661227730Sraj } 1662227730Sraj 1663227730Sraj /* Convert request to chain of TDMA and SA descriptors */ 1664227730Sraj if (!error) 1665227730Sraj error = cesa_create_chain(sc, cr); 1666227730Sraj 1667227730Sraj cesa_sync_desc(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1668227730Sraj CESA_UNLOCK(sc, sessions); 1669227730Sraj 1670227730Sraj if (error) { 1671227730Sraj cesa_free_request(sc, cr); 1672227730Sraj crp->crp_etype = error; 1673227730Sraj crypto_done(crp); 1674227730Sraj return (0); 1675227730Sraj } 1676227730Sraj 1677227730Sraj bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, BUS_DMASYNC_PREREAD | 1678227730Sraj BUS_DMASYNC_PREWRITE); 1679227730Sraj 1680227730Sraj /* Enqueue request to execution */ 1681227730Sraj cesa_enqueue_request(sc, cr); 1682227730Sraj 1683227730Sraj /* Start execution, if we have no more requests in queue */ 1684227730Sraj if ((hint & CRYPTO_HINT_MORE) == 0) 1685227730Sraj cesa_execute(sc); 1686227730Sraj 1687227730Sraj return (0); 1688227730Sraj} 1689250291Sgber 1690250291Sgber/* 1691250291Sgber * Set CESA TDMA decode windows. 1692250291Sgber */ 1693250291Sgberstatic int 1694250291Sgberdecode_win_cesa_setup(struct cesa_softc *sc) 1695250291Sgber{ 1696250291Sgber struct mem_region availmem_regions[FDT_MEM_REGIONS]; 1697250291Sgber int availmem_regions_sz; 1698296189Swma uint32_t br, cr, i; 1699250291Sgber 1700250291Sgber /* Grab physical memory regions information from DTS */ 1701250291Sgber if (fdt_get_mem_regions(availmem_regions, &availmem_regions_sz, 1702296257Sandrew NULL) != 0) 1703250291Sgber return (ENXIO); 1704250291Sgber 1705250291Sgber if (availmem_regions_sz > MV_WIN_CESA_MAX) { 1706250291Sgber device_printf(sc->sc_dev, "Too much memory regions, cannot " 1707250291Sgber " set CESA windows to cover whole DRAM \n"); 1708250291Sgber return (ENXIO); 1709250291Sgber } 1710250291Sgber 1711250291Sgber /* Disable and clear all CESA windows */ 1712250291Sgber for (i = 0; i < MV_WIN_CESA_MAX; i++) { 1713301222Szbb CESA_TDMA_WRITE(sc, MV_WIN_CESA_BASE(i), 0); 1714301222Szbb CESA_TDMA_WRITE(sc, MV_WIN_CESA_CTRL(i), 0); 1715250291Sgber } 1716250291Sgber 1717250291Sgber /* Fill CESA TDMA decoding windows with information acquired from DTS */ 1718250291Sgber for (i = 0; i < availmem_regions_sz; i++) { 1719250291Sgber br = availmem_regions[i].mr_start; 1720250291Sgber cr = availmem_regions[i].mr_size; 1721250291Sgber 1722250291Sgber /* Don't add entries with size lower than 64KB */ 1723250291Sgber if (cr & 0xffff0000) { 1724250291Sgber cr = (((cr - 1) & 0xffff0000) | 1725250291Sgber (MV_WIN_DDR_ATTR(i) << MV_WIN_CPU_ATTR_SHIFT) | 1726250291Sgber (MV_WIN_DDR_TARGET << MV_WIN_CPU_TARGET_SHIFT) | 1727250291Sgber MV_WIN_CPU_ENABLE_BIT); 1728301222Szbb CESA_TDMA_WRITE(sc, MV_WIN_CESA_BASE(i), br); 1729301222Szbb CESA_TDMA_WRITE(sc, MV_WIN_CESA_CTRL(i), cr); 1730250291Sgber } 1731250291Sgber } 1732250291Sgber 1733250291Sgber return (0); 1734250291Sgber} 1735250291Sgber 1736