cesa.c revision 232883
1227730Sraj/*- 2227730Sraj * Copyright (C) 2009-2011 Semihalf. 3227730Sraj * All rights reserved. 4227730Sraj * 5227730Sraj * Redistribution and use in source and binary forms, with or without 6227730Sraj * modification, are permitted provided that the following conditions 7227730Sraj * are met: 8227730Sraj * 1. Redistributions of source code must retain the above copyright 9227730Sraj * notice, this list of conditions and the following disclaimer. 10227730Sraj * 2. Redistributions in binary form must reproduce the above copyright 11227730Sraj * notice, this list of conditions and the following disclaimer in the 12227730Sraj * documentation and/or other materials provided with the distribution. 13227730Sraj * 14227730Sraj * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15227730Sraj * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16227730Sraj * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17227730Sraj * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18227730Sraj * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19227730Sraj * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20227730Sraj * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21227730Sraj * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22227730Sraj * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23227730Sraj * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24227730Sraj * SUCH DAMAGE. 25227730Sraj */ 26227730Sraj 27227730Sraj/* 28227730Sraj * CESA SRAM Memory Map: 29227730Sraj * 30227730Sraj * +------------------------+ <= sc->sc_sram_base + CESA_SRAM_SIZE 31227730Sraj * | | 32227730Sraj * | DATA | 33227730Sraj * | | 34227730Sraj * +------------------------+ <= sc->sc_sram_base + CESA_DATA(0) 35227730Sraj * | struct cesa_sa_data | 36227730Sraj * +------------------------+ 37227730Sraj * | struct cesa_sa_hdesc | 38227730Sraj * +------------------------+ <= sc->sc_sram_base 39227730Sraj */ 40227730Sraj 41227730Sraj#include <sys/cdefs.h> 42227730Sraj__FBSDID("$FreeBSD: head/sys/dev/cesa/cesa.c 232883 2012-03-12 19:29:35Z scottl $"); 43227730Sraj 44227730Sraj#include <sys/param.h> 45227730Sraj#include <sys/systm.h> 46227730Sraj#include <sys/bus.h> 47227730Sraj#include <sys/endian.h> 48227730Sraj#include <sys/kernel.h> 49227730Sraj#include <sys/lock.h> 50227730Sraj#include <sys/mbuf.h> 51227730Sraj#include <sys/module.h> 52227730Sraj#include <sys/mutex.h> 53227730Sraj#include <sys/rman.h> 54227730Sraj 55227730Sraj#include <machine/bus.h> 56227730Sraj#include <machine/intr.h> 57227730Sraj#include <machine/resource.h> 58227730Sraj 59227730Sraj#include <dev/fdt/fdt_common.h> 60227730Sraj#include <dev/ofw/ofw_bus.h> 61227730Sraj#include <dev/ofw/ofw_bus_subr.h> 62227730Sraj 63227730Sraj#include <sys/md5.h> 64227730Sraj#include <crypto/sha1.h> 65227730Sraj#include <crypto/rijndael/rijndael.h> 66227730Sraj#include <opencrypto/cryptodev.h> 67227730Sraj#include "cryptodev_if.h" 68227730Sraj 69227730Sraj#include <arm/mv/mvreg.h> 70227730Sraj#include <arm/mv/mvwin.h> 71227730Sraj#include <arm/mv/mvvar.h> 72227730Sraj#include "cesa.h" 73227730Sraj 74227730Sraj#undef DEBUG 75227730Sraj 76227730Srajstatic int cesa_probe(device_t); 77227730Srajstatic int cesa_attach(device_t); 78227730Srajstatic int cesa_detach(device_t); 79227730Srajstatic void cesa_intr(void *); 80227730Srajstatic int cesa_newsession(device_t, u_int32_t *, struct cryptoini *); 81227730Srajstatic int cesa_freesession(device_t, u_int64_t); 82227730Srajstatic int cesa_process(device_t, struct cryptop *, int); 83227730Sraj 84227730Srajstatic struct resource_spec cesa_res_spec[] = { 85227730Sraj { SYS_RES_MEMORY, 0, RF_ACTIVE }, 86227730Sraj { SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE }, 87227730Sraj { -1, 0 } 88227730Sraj}; 89227730Sraj 90227730Srajstatic device_method_t cesa_methods[] = { 91227730Sraj /* Device interface */ 92227730Sraj DEVMETHOD(device_probe, cesa_probe), 93227730Sraj DEVMETHOD(device_attach, cesa_attach), 94227730Sraj DEVMETHOD(device_detach, cesa_detach), 95227730Sraj 96227730Sraj /* Crypto device methods */ 97227730Sraj DEVMETHOD(cryptodev_newsession, cesa_newsession), 98227730Sraj DEVMETHOD(cryptodev_freesession,cesa_freesession), 99227730Sraj DEVMETHOD(cryptodev_process, cesa_process), 100227730Sraj 101227843Smarius DEVMETHOD_END 102227730Sraj}; 103227730Sraj 104227730Srajstatic driver_t cesa_driver = { 105227730Sraj "cesa", 106227730Sraj cesa_methods, 107227730Sraj sizeof (struct cesa_softc) 108227730Sraj}; 109227730Srajstatic devclass_t cesa_devclass; 110227730Sraj 111227730SrajDRIVER_MODULE(cesa, simplebus, cesa_driver, cesa_devclass, 0, 0); 112227730SrajMODULE_DEPEND(cesa, crypto, 1, 1, 1); 113227730Sraj 114227730Srajstatic void 115227730Srajcesa_dump_cshd(struct cesa_softc *sc, struct cesa_sa_hdesc *cshd) 116227730Sraj{ 117227730Sraj#ifdef DEBUG 118227730Sraj device_t dev; 119227730Sraj 120227730Sraj dev = sc->sc_dev; 121227730Sraj device_printf(dev, "CESA SA Hardware Descriptor:\n"); 122227730Sraj device_printf(dev, "\t\tconfig: 0x%08X\n", cshd->cshd_config); 123227730Sraj device_printf(dev, "\t\te_src: 0x%08X\n", cshd->cshd_enc_src); 124227730Sraj device_printf(dev, "\t\te_dst: 0x%08X\n", cshd->cshd_enc_dst); 125227730Sraj device_printf(dev, "\t\te_dlen: 0x%08X\n", cshd->cshd_enc_dlen); 126227730Sraj device_printf(dev, "\t\te_key: 0x%08X\n", cshd->cshd_enc_key); 127227730Sraj device_printf(dev, "\t\te_iv_1: 0x%08X\n", cshd->cshd_enc_iv); 128227730Sraj device_printf(dev, "\t\te_iv_2: 0x%08X\n", cshd->cshd_enc_iv_buf); 129227730Sraj device_printf(dev, "\t\tm_src: 0x%08X\n", cshd->cshd_mac_src); 130227730Sraj device_printf(dev, "\t\tm_dst: 0x%08X\n", cshd->cshd_mac_dst); 131227730Sraj device_printf(dev, "\t\tm_dlen: 0x%08X\n", cshd->cshd_mac_dlen); 132227730Sraj device_printf(dev, "\t\tm_tlen: 0x%08X\n", cshd->cshd_mac_total_dlen); 133227730Sraj device_printf(dev, "\t\tm_iv_i: 0x%08X\n", cshd->cshd_mac_iv_in); 134227730Sraj device_printf(dev, "\t\tm_iv_o: 0x%08X\n", cshd->cshd_mac_iv_out); 135227730Sraj#endif 136227730Sraj} 137227730Sraj 138227730Srajstatic void 139227730Srajcesa_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 140227730Sraj{ 141227730Sraj struct cesa_dma_mem *cdm; 142227730Sraj 143227730Sraj if (error) 144227730Sraj return; 145227730Sraj 146227730Sraj KASSERT(nseg == 1, ("Got wrong number of DMA segments, should be 1.")); 147227730Sraj cdm = arg; 148227730Sraj cdm->cdm_paddr = segs->ds_addr; 149227730Sraj} 150227730Sraj 151227730Srajstatic int 152227730Srajcesa_alloc_dma_mem(struct cesa_softc *sc, struct cesa_dma_mem *cdm, 153227730Sraj bus_size_t size) 154227730Sraj{ 155227730Sraj int error; 156227730Sraj 157227730Sraj KASSERT(cdm->cdm_vaddr == NULL, 158227730Sraj ("%s(): DMA memory descriptor in use.", __func__)); 159227730Sraj 160232883Sscottl error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ 161227730Sraj PAGE_SIZE, 0, /* alignment, boundary */ 162227730Sraj BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 163227730Sraj BUS_SPACE_MAXADDR, /* highaddr */ 164227730Sraj NULL, NULL, /* filtfunc, filtfuncarg */ 165227730Sraj size, 1, /* maxsize, nsegments */ 166227730Sraj size, 0, /* maxsegsz, flags */ 167227730Sraj NULL, NULL, /* lockfunc, lockfuncarg */ 168227730Sraj &cdm->cdm_tag); /* dmat */ 169227730Sraj if (error) { 170227730Sraj device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 171227730Sraj " %i!\n", error); 172227730Sraj 173227730Sraj goto err1; 174227730Sraj } 175227730Sraj 176227730Sraj error = bus_dmamem_alloc(cdm->cdm_tag, &cdm->cdm_vaddr, 177227730Sraj BUS_DMA_NOWAIT | BUS_DMA_ZERO, &cdm->cdm_map); 178227730Sraj if (error) { 179227730Sraj device_printf(sc->sc_dev, "failed to allocate DMA safe" 180227730Sraj " memory, error %i!\n", error); 181227730Sraj 182227730Sraj goto err2; 183227730Sraj } 184227730Sraj 185227730Sraj error = bus_dmamap_load(cdm->cdm_tag, cdm->cdm_map, cdm->cdm_vaddr, 186227730Sraj size, cesa_alloc_dma_mem_cb, cdm, BUS_DMA_NOWAIT); 187227730Sraj if (error) { 188227730Sraj device_printf(sc->sc_dev, "cannot get address of the DMA" 189227730Sraj " memory, error %i\n", error); 190227730Sraj 191227730Sraj goto err3; 192227730Sraj } 193227730Sraj 194227730Sraj return (0); 195227730Srajerr3: 196227730Sraj bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); 197227730Srajerr2: 198227730Sraj bus_dma_tag_destroy(cdm->cdm_tag); 199227730Srajerr1: 200227730Sraj cdm->cdm_vaddr = NULL; 201227730Sraj return (error); 202227730Sraj} 203227730Sraj 204227730Srajstatic void 205227730Srajcesa_free_dma_mem(struct cesa_dma_mem *cdm) 206227730Sraj{ 207227730Sraj 208227730Sraj bus_dmamap_unload(cdm->cdm_tag, cdm->cdm_map); 209227730Sraj bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); 210227730Sraj bus_dma_tag_destroy(cdm->cdm_tag); 211227730Sraj cdm->cdm_vaddr = NULL; 212227730Sraj} 213227730Sraj 214227730Srajstatic void 215227730Srajcesa_sync_dma_mem(struct cesa_dma_mem *cdm, bus_dmasync_op_t op) 216227730Sraj{ 217227730Sraj 218227730Sraj /* Sync only if dma memory is valid */ 219227730Sraj if (cdm->cdm_vaddr != NULL) 220227730Sraj bus_dmamap_sync(cdm->cdm_tag, cdm->cdm_map, op); 221227730Sraj} 222227730Sraj 223227730Srajstatic void 224227730Srajcesa_sync_desc(struct cesa_softc *sc, bus_dmasync_op_t op) 225227730Sraj{ 226227730Sraj 227227730Sraj cesa_sync_dma_mem(&sc->sc_tdesc_cdm, op); 228227730Sraj cesa_sync_dma_mem(&sc->sc_sdesc_cdm, op); 229227730Sraj cesa_sync_dma_mem(&sc->sc_requests_cdm, op); 230227730Sraj} 231227730Sraj 232227730Srajstatic struct cesa_session * 233227730Srajcesa_alloc_session(struct cesa_softc *sc) 234227730Sraj{ 235227730Sraj struct cesa_session *cs; 236227730Sraj 237227730Sraj CESA_GENERIC_ALLOC_LOCKED(sc, cs, sessions); 238227730Sraj 239227730Sraj return (cs); 240227730Sraj} 241227730Sraj 242227730Srajstatic struct cesa_session * 243227730Srajcesa_get_session(struct cesa_softc *sc, uint32_t sid) 244227730Sraj{ 245227730Sraj 246227730Sraj if (sid >= CESA_SESSIONS) 247227730Sraj return (NULL); 248227730Sraj 249227730Sraj return (&sc->sc_sessions[sid]); 250227730Sraj} 251227730Sraj 252227730Srajstatic void 253227730Srajcesa_free_session(struct cesa_softc *sc, struct cesa_session *cs) 254227730Sraj{ 255227730Sraj 256227730Sraj CESA_GENERIC_FREE_LOCKED(sc, cs, sessions); 257227730Sraj} 258227730Sraj 259227730Srajstatic struct cesa_request * 260227730Srajcesa_alloc_request(struct cesa_softc *sc) 261227730Sraj{ 262227730Sraj struct cesa_request *cr; 263227730Sraj 264227730Sraj CESA_GENERIC_ALLOC_LOCKED(sc, cr, requests); 265227730Sraj if (!cr) 266227730Sraj return (NULL); 267227730Sraj 268227730Sraj STAILQ_INIT(&cr->cr_tdesc); 269227730Sraj STAILQ_INIT(&cr->cr_sdesc); 270227730Sraj 271227730Sraj return (cr); 272227730Sraj} 273227730Sraj 274227730Srajstatic void 275227730Srajcesa_free_request(struct cesa_softc *sc, struct cesa_request *cr) 276227730Sraj{ 277227730Sraj 278227730Sraj /* Free TDMA descriptors assigned to this request */ 279227730Sraj CESA_LOCK(sc, tdesc); 280227730Sraj STAILQ_CONCAT(&sc->sc_free_tdesc, &cr->cr_tdesc); 281227730Sraj CESA_UNLOCK(sc, tdesc); 282227730Sraj 283227730Sraj /* Free SA descriptors assigned to this request */ 284227730Sraj CESA_LOCK(sc, sdesc); 285227730Sraj STAILQ_CONCAT(&sc->sc_free_sdesc, &cr->cr_sdesc); 286227730Sraj CESA_UNLOCK(sc, sdesc); 287227730Sraj 288227730Sraj /* Unload DMA memory asociated with request */ 289227730Sraj if (cr->cr_dmap_loaded) { 290227730Sraj bus_dmamap_unload(sc->sc_data_dtag, cr->cr_dmap); 291227730Sraj cr->cr_dmap_loaded = 0; 292227730Sraj } 293227730Sraj 294227730Sraj CESA_GENERIC_FREE_LOCKED(sc, cr, requests); 295227730Sraj} 296227730Sraj 297227730Srajstatic void 298227730Srajcesa_enqueue_request(struct cesa_softc *sc, struct cesa_request *cr) 299227730Sraj{ 300227730Sraj 301227730Sraj CESA_LOCK(sc, requests); 302227730Sraj STAILQ_INSERT_TAIL(&sc->sc_ready_requests, cr, cr_stq); 303227730Sraj CESA_UNLOCK(sc, requests); 304227730Sraj} 305227730Sraj 306227730Srajstatic struct cesa_tdma_desc * 307227730Srajcesa_alloc_tdesc(struct cesa_softc *sc) 308227730Sraj{ 309227730Sraj struct cesa_tdma_desc *ctd; 310227730Sraj 311227730Sraj CESA_GENERIC_ALLOC_LOCKED(sc, ctd, tdesc); 312227730Sraj 313227730Sraj if (!ctd) 314227730Sraj device_printf(sc->sc_dev, "TDMA descriptors pool exhaused. " 315227730Sraj "Consider increasing CESA_TDMA_DESCRIPTORS.\n"); 316227730Sraj 317227730Sraj return (ctd); 318227730Sraj} 319227730Sraj 320227730Srajstatic struct cesa_sa_desc * 321227730Srajcesa_alloc_sdesc(struct cesa_softc *sc, struct cesa_request *cr) 322227730Sraj{ 323227730Sraj struct cesa_sa_desc *csd; 324227730Sraj 325227730Sraj CESA_GENERIC_ALLOC_LOCKED(sc, csd, sdesc); 326227730Sraj if (!csd) { 327227730Sraj device_printf(sc->sc_dev, "SA descriptors pool exhaused. " 328227730Sraj "Consider increasing CESA_SA_DESCRIPTORS.\n"); 329227730Sraj return (NULL); 330227730Sraj } 331227730Sraj 332227730Sraj STAILQ_INSERT_TAIL(&cr->cr_sdesc, csd, csd_stq); 333227730Sraj 334227730Sraj /* Fill-in SA descriptor with default values */ 335227730Sraj csd->csd_cshd->cshd_enc_key = CESA_SA_DATA(csd_key); 336227730Sraj csd->csd_cshd->cshd_enc_iv = CESA_SA_DATA(csd_iv); 337227730Sraj csd->csd_cshd->cshd_enc_iv_buf = CESA_SA_DATA(csd_iv); 338227730Sraj csd->csd_cshd->cshd_enc_src = 0; 339227730Sraj csd->csd_cshd->cshd_enc_dst = 0; 340227730Sraj csd->csd_cshd->cshd_enc_dlen = 0; 341227730Sraj csd->csd_cshd->cshd_mac_dst = CESA_SA_DATA(csd_hash); 342227730Sraj csd->csd_cshd->cshd_mac_iv_in = CESA_SA_DATA(csd_hiv_in); 343227730Sraj csd->csd_cshd->cshd_mac_iv_out = CESA_SA_DATA(csd_hiv_out); 344227730Sraj csd->csd_cshd->cshd_mac_src = 0; 345227730Sraj csd->csd_cshd->cshd_mac_dlen = 0; 346227730Sraj 347227730Sraj return (csd); 348227730Sraj} 349227730Sraj 350227730Srajstatic struct cesa_tdma_desc * 351227730Srajcesa_tdma_copy(struct cesa_softc *sc, bus_addr_t dst, bus_addr_t src, 352227730Sraj bus_size_t size) 353227730Sraj{ 354227730Sraj struct cesa_tdma_desc *ctd; 355227730Sraj 356227730Sraj ctd = cesa_alloc_tdesc(sc); 357227730Sraj if (!ctd) 358227730Sraj return (NULL); 359227730Sraj 360227730Sraj ctd->ctd_cthd->cthd_dst = dst; 361227730Sraj ctd->ctd_cthd->cthd_src = src; 362227730Sraj ctd->ctd_cthd->cthd_byte_count = size; 363227730Sraj 364227730Sraj /* Handle special control packet */ 365227730Sraj if (size != 0) 366227730Sraj ctd->ctd_cthd->cthd_flags = CESA_CTHD_OWNED; 367227730Sraj else 368227730Sraj ctd->ctd_cthd->cthd_flags = 0; 369227730Sraj 370227730Sraj return (ctd); 371227730Sraj} 372227730Sraj 373227730Srajstatic struct cesa_tdma_desc * 374227730Srajcesa_tdma_copyin_sa_data(struct cesa_softc *sc, struct cesa_request *cr) 375227730Sraj{ 376227730Sraj 377227730Sraj return (cesa_tdma_copy(sc, sc->sc_sram_base + 378227730Sraj sizeof(struct cesa_sa_hdesc), cr->cr_csd_paddr, 379227730Sraj sizeof(struct cesa_sa_data))); 380227730Sraj} 381227730Sraj 382227730Srajstatic struct cesa_tdma_desc * 383227730Srajcesa_tdma_copyout_sa_data(struct cesa_softc *sc, struct cesa_request *cr) 384227730Sraj{ 385227730Sraj 386227730Sraj return (cesa_tdma_copy(sc, cr->cr_csd_paddr, sc->sc_sram_base + 387227730Sraj sizeof(struct cesa_sa_hdesc), sizeof(struct cesa_sa_data))); 388227730Sraj} 389227730Sraj 390227730Srajstatic struct cesa_tdma_desc * 391227730Srajcesa_tdma_copy_sdesc(struct cesa_softc *sc, struct cesa_sa_desc *csd) 392227730Sraj{ 393227730Sraj 394227730Sraj return (cesa_tdma_copy(sc, sc->sc_sram_base, csd->csd_cshd_paddr, 395227730Sraj sizeof(struct cesa_sa_hdesc))); 396227730Sraj} 397227730Sraj 398227730Srajstatic void 399227730Srajcesa_append_tdesc(struct cesa_request *cr, struct cesa_tdma_desc *ctd) 400227730Sraj{ 401227730Sraj struct cesa_tdma_desc *ctd_prev; 402227730Sraj 403227730Sraj if (!STAILQ_EMPTY(&cr->cr_tdesc)) { 404227730Sraj ctd_prev = STAILQ_LAST(&cr->cr_tdesc, cesa_tdma_desc, ctd_stq); 405227730Sraj ctd_prev->ctd_cthd->cthd_next = ctd->ctd_cthd_paddr; 406227730Sraj } 407227730Sraj 408227730Sraj ctd->ctd_cthd->cthd_next = 0; 409227730Sraj STAILQ_INSERT_TAIL(&cr->cr_tdesc, ctd, ctd_stq); 410227730Sraj} 411227730Sraj 412227730Srajstatic int 413227730Srajcesa_append_packet(struct cesa_softc *sc, struct cesa_request *cr, 414227730Sraj struct cesa_packet *cp, struct cesa_sa_desc *csd) 415227730Sraj{ 416227730Sraj struct cesa_tdma_desc *ctd, *tmp; 417227730Sraj 418227730Sraj /* Copy SA descriptor for this packet */ 419227730Sraj ctd = cesa_tdma_copy_sdesc(sc, csd); 420227730Sraj if (!ctd) 421227730Sraj return (ENOMEM); 422227730Sraj 423227730Sraj cesa_append_tdesc(cr, ctd); 424227730Sraj 425227730Sraj /* Copy data to be processed */ 426227730Sraj STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyin, ctd_stq, tmp) 427227730Sraj cesa_append_tdesc(cr, ctd); 428227730Sraj STAILQ_INIT(&cp->cp_copyin); 429227730Sraj 430227730Sraj /* Insert control descriptor */ 431227730Sraj ctd = cesa_tdma_copy(sc, 0, 0, 0); 432227730Sraj if (!ctd) 433227730Sraj return (ENOMEM); 434227730Sraj 435227730Sraj cesa_append_tdesc(cr, ctd); 436227730Sraj 437227730Sraj /* Copy back results */ 438227730Sraj STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyout, ctd_stq, tmp) 439227730Sraj cesa_append_tdesc(cr, ctd); 440227730Sraj STAILQ_INIT(&cp->cp_copyout); 441227730Sraj 442227730Sraj return (0); 443227730Sraj} 444227730Sraj 445227730Srajstatic int 446227730Srajcesa_set_mkey(struct cesa_session *cs, int alg, const uint8_t *mkey, int mklen) 447227730Sraj{ 448227730Sraj uint8_t ipad[CESA_MAX_HMAC_BLOCK_LEN]; 449227730Sraj uint8_t opad[CESA_MAX_HMAC_BLOCK_LEN]; 450227730Sraj SHA1_CTX sha1ctx; 451227730Sraj MD5_CTX md5ctx; 452227730Sraj uint32_t *hout; 453227730Sraj uint32_t *hin; 454227730Sraj int i; 455227730Sraj 456227730Sraj memset(ipad, HMAC_IPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN); 457227730Sraj memset(opad, HMAC_OPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN); 458227730Sraj for (i = 0; i < mklen; i++) { 459227730Sraj ipad[i] ^= mkey[i]; 460227730Sraj opad[i] ^= mkey[i]; 461227730Sraj } 462227730Sraj 463227730Sraj hin = (uint32_t *)cs->cs_hiv_in; 464227730Sraj hout = (uint32_t *)cs->cs_hiv_out; 465227730Sraj 466227730Sraj switch (alg) { 467227730Sraj case CRYPTO_MD5_HMAC: 468227730Sraj MD5Init(&md5ctx); 469227730Sraj MD5Update(&md5ctx, ipad, MD5_HMAC_BLOCK_LEN); 470227730Sraj memcpy(hin, md5ctx.state, sizeof(md5ctx.state)); 471227730Sraj MD5Init(&md5ctx); 472227730Sraj MD5Update(&md5ctx, opad, MD5_HMAC_BLOCK_LEN); 473227730Sraj memcpy(hout, md5ctx.state, sizeof(md5ctx.state)); 474227730Sraj break; 475227730Sraj case CRYPTO_SHA1_HMAC: 476227730Sraj SHA1Init(&sha1ctx); 477227730Sraj SHA1Update(&sha1ctx, ipad, SHA1_HMAC_BLOCK_LEN); 478227730Sraj memcpy(hin, sha1ctx.h.b32, sizeof(sha1ctx.h.b32)); 479227730Sraj SHA1Init(&sha1ctx); 480227730Sraj SHA1Update(&sha1ctx, opad, SHA1_HMAC_BLOCK_LEN); 481227730Sraj memcpy(hout, sha1ctx.h.b32, sizeof(sha1ctx.h.b32)); 482227730Sraj break; 483227730Sraj default: 484227730Sraj return (EINVAL); 485227730Sraj } 486227730Sraj 487227730Sraj for (i = 0; i < CESA_MAX_HASH_LEN / sizeof(uint32_t); i++) { 488227730Sraj hin[i] = htobe32(hin[i]); 489227730Sraj hout[i] = htobe32(hout[i]); 490227730Sraj } 491227730Sraj 492227730Sraj return (0); 493227730Sraj} 494227730Sraj 495227730Srajstatic int 496227730Srajcesa_prep_aes_key(struct cesa_session *cs) 497227730Sraj{ 498227730Sraj uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)]; 499227730Sraj uint32_t *dkey; 500227730Sraj int i; 501227730Sraj 502227730Sraj rijndaelKeySetupEnc(ek, cs->cs_key, cs->cs_klen * 8); 503227730Sraj 504227730Sraj cs->cs_config &= ~CESA_CSH_AES_KLEN_MASK; 505227730Sraj dkey = (uint32_t *)cs->cs_aes_dkey; 506227730Sraj 507227730Sraj switch (cs->cs_klen) { 508227730Sraj case 16: 509227730Sraj cs->cs_config |= CESA_CSH_AES_KLEN_128; 510227730Sraj for (i = 0; i < 4; i++) 511227730Sraj *dkey++ = htobe32(ek[4 * 10 + i]); 512227730Sraj break; 513227730Sraj case 24: 514227730Sraj cs->cs_config |= CESA_CSH_AES_KLEN_192; 515227730Sraj for (i = 0; i < 4; i++) 516227730Sraj *dkey++ = htobe32(ek[4 * 12 + i]); 517227730Sraj for (i = 0; i < 2; i++) 518227730Sraj *dkey++ = htobe32(ek[4 * 11 + 2 + i]); 519227730Sraj break; 520227730Sraj case 32: 521227730Sraj cs->cs_config |= CESA_CSH_AES_KLEN_256; 522227730Sraj for (i = 0; i < 4; i++) 523227730Sraj *dkey++ = htobe32(ek[4 * 14 + i]); 524227730Sraj for (i = 0; i < 4; i++) 525227730Sraj *dkey++ = htobe32(ek[4 * 13 + i]); 526227730Sraj break; 527227730Sraj default: 528227730Sraj return (EINVAL); 529227730Sraj } 530227730Sraj 531227730Sraj return (0); 532227730Sraj} 533227730Sraj 534227730Srajstatic int 535227730Srajcesa_is_hash(int alg) 536227730Sraj{ 537227730Sraj 538227730Sraj switch (alg) { 539227730Sraj case CRYPTO_MD5: 540227730Sraj case CRYPTO_MD5_HMAC: 541227730Sraj case CRYPTO_SHA1: 542227730Sraj case CRYPTO_SHA1_HMAC: 543227730Sraj return (1); 544227730Sraj default: 545227730Sraj return (0); 546227730Sraj } 547227730Sraj} 548227730Sraj 549227730Srajstatic void 550227730Srajcesa_start_packet(struct cesa_packet *cp, unsigned int size) 551227730Sraj{ 552227730Sraj 553227730Sraj cp->cp_size = size; 554227730Sraj cp->cp_offset = 0; 555227730Sraj STAILQ_INIT(&cp->cp_copyin); 556227730Sraj STAILQ_INIT(&cp->cp_copyout); 557227730Sraj} 558227730Sraj 559227730Srajstatic int 560227730Srajcesa_fill_packet(struct cesa_softc *sc, struct cesa_packet *cp, 561227730Sraj bus_dma_segment_t *seg) 562227730Sraj{ 563227730Sraj struct cesa_tdma_desc *ctd; 564227730Sraj unsigned int bsize; 565227730Sraj 566227730Sraj /* Calculate size of block copy */ 567227730Sraj bsize = MIN(seg->ds_len, cp->cp_size - cp->cp_offset); 568227730Sraj 569227730Sraj if (bsize > 0) { 570227730Sraj ctd = cesa_tdma_copy(sc, sc->sc_sram_base + 571227730Sraj CESA_DATA(cp->cp_offset), seg->ds_addr, bsize); 572227730Sraj if (!ctd) 573227730Sraj return (-ENOMEM); 574227730Sraj 575227730Sraj STAILQ_INSERT_TAIL(&cp->cp_copyin, ctd, ctd_stq); 576227730Sraj 577227730Sraj ctd = cesa_tdma_copy(sc, seg->ds_addr, sc->sc_sram_base + 578227730Sraj CESA_DATA(cp->cp_offset), bsize); 579227730Sraj if (!ctd) 580227730Sraj return (-ENOMEM); 581227730Sraj 582227730Sraj STAILQ_INSERT_TAIL(&cp->cp_copyout, ctd, ctd_stq); 583227730Sraj 584227730Sraj seg->ds_len -= bsize; 585227730Sraj seg->ds_addr += bsize; 586227730Sraj cp->cp_offset += bsize; 587227730Sraj } 588227730Sraj 589227730Sraj return (bsize); 590227730Sraj} 591227730Sraj 592227730Srajstatic void 593227730Srajcesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 594227730Sraj{ 595227730Sraj unsigned int mpsize, fragmented; 596227730Sraj unsigned int mlen, mskip, tmlen; 597227730Sraj struct cesa_chain_info *cci; 598227730Sraj unsigned int elen, eskip; 599227730Sraj unsigned int skip, len; 600227730Sraj struct cesa_sa_desc *csd; 601227730Sraj struct cesa_request *cr; 602227730Sraj struct cesa_softc *sc; 603227730Sraj struct cesa_packet cp; 604227730Sraj bus_dma_segment_t seg; 605227730Sraj uint32_t config; 606227730Sraj int size; 607227730Sraj 608227730Sraj cci = arg; 609227730Sraj sc = cci->cci_sc; 610227730Sraj cr = cci->cci_cr; 611227730Sraj 612227730Sraj if (error) { 613227730Sraj cci->cci_error = error; 614227730Sraj return; 615227730Sraj } 616227730Sraj 617227730Sraj elen = cci->cci_enc ? cci->cci_enc->crd_len : 0; 618227730Sraj eskip = cci->cci_enc ? cci->cci_enc->crd_skip : 0; 619227730Sraj mlen = cci->cci_mac ? cci->cci_mac->crd_len : 0; 620227730Sraj mskip = cci->cci_mac ? cci->cci_mac->crd_skip : 0; 621227730Sraj 622227730Sraj if (elen && mlen && 623227730Sraj ((eskip > mskip && ((eskip - mskip) & (cr->cr_cs->cs_ivlen - 1))) || 624227730Sraj (mskip > eskip && ((mskip - eskip) & (cr->cr_cs->cs_mblen - 1))) || 625227730Sraj (eskip > (mskip + mlen)) || (mskip > (eskip + elen)))) { 626227730Sraj /* 627227730Sraj * Data alignment in the request does not meet CESA requiremnts 628227730Sraj * for combined encryption/decryption and hashing. We have to 629227730Sraj * split the request to separate operations and process them 630227730Sraj * one by one. 631227730Sraj */ 632227730Sraj config = cci->cci_config; 633227730Sraj if ((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC) { 634227730Sraj config &= ~CESA_CSHD_OP_MASK; 635227730Sraj 636227730Sraj cci->cci_config = config | CESA_CSHD_MAC; 637227730Sraj cci->cci_enc = NULL; 638227730Sraj cci->cci_mac = cr->cr_mac; 639227730Sraj cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 640227730Sraj 641227730Sraj cci->cci_config = config | CESA_CSHD_ENC; 642227730Sraj cci->cci_enc = cr->cr_enc; 643227730Sraj cci->cci_mac = NULL; 644227730Sraj cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 645227730Sraj } else { 646227730Sraj config &= ~CESA_CSHD_OP_MASK; 647227730Sraj 648227730Sraj cci->cci_config = config | CESA_CSHD_ENC; 649227730Sraj cci->cci_enc = cr->cr_enc; 650227730Sraj cci->cci_mac = NULL; 651227730Sraj cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 652227730Sraj 653227730Sraj cci->cci_config = config | CESA_CSHD_MAC; 654227730Sraj cci->cci_enc = NULL; 655227730Sraj cci->cci_mac = cr->cr_mac; 656227730Sraj cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 657227730Sraj } 658227730Sraj 659227730Sraj return; 660227730Sraj } 661227730Sraj 662227730Sraj tmlen = mlen; 663227730Sraj fragmented = 0; 664227730Sraj mpsize = CESA_MAX_PACKET_SIZE; 665227730Sraj mpsize &= ~((cr->cr_cs->cs_ivlen - 1) | (cr->cr_cs->cs_mblen - 1)); 666227730Sraj 667227730Sraj if (elen && mlen) { 668227730Sraj skip = MIN(eskip, mskip); 669227730Sraj len = MAX(elen + eskip, mlen + mskip) - skip; 670227730Sraj } else if (elen) { 671227730Sraj skip = eskip; 672227730Sraj len = elen; 673227730Sraj } else { 674227730Sraj skip = mskip; 675227730Sraj len = mlen; 676227730Sraj } 677227730Sraj 678227730Sraj /* Start first packet in chain */ 679227730Sraj cesa_start_packet(&cp, MIN(mpsize, len)); 680227730Sraj 681227730Sraj while (nseg-- && len > 0) { 682227730Sraj seg = *(segs++); 683227730Sraj 684227730Sraj /* 685227730Sraj * Skip data in buffer on which neither ENC nor MAC operation 686227730Sraj * is requested. 687227730Sraj */ 688227730Sraj if (skip > 0) { 689227730Sraj size = MIN(skip, seg.ds_len); 690227730Sraj skip -= size; 691227730Sraj 692227730Sraj seg.ds_addr += size; 693227730Sraj seg.ds_len -= size; 694227730Sraj 695227730Sraj if (eskip > 0) 696227730Sraj eskip -= size; 697227730Sraj 698227730Sraj if (mskip > 0) 699227730Sraj mskip -= size; 700227730Sraj 701227730Sraj if (seg.ds_len == 0) 702227730Sraj continue; 703227730Sraj } 704227730Sraj 705227730Sraj while (1) { 706227730Sraj /* 707227730Sraj * Fill in current packet with data. Break if there is 708227730Sraj * no more data in current DMA segment or an error 709227730Sraj * occured. 710227730Sraj */ 711227730Sraj size = cesa_fill_packet(sc, &cp, &seg); 712227730Sraj if (size <= 0) { 713227730Sraj error = -size; 714227730Sraj break; 715227730Sraj } 716227730Sraj 717227730Sraj len -= size; 718227730Sraj 719227730Sraj /* If packet is full, append it to the chain */ 720227730Sraj if (cp.cp_size == cp.cp_offset) { 721227730Sraj csd = cesa_alloc_sdesc(sc, cr); 722227730Sraj if (!csd) { 723227730Sraj error = ENOMEM; 724227730Sraj break; 725227730Sraj } 726227730Sraj 727227730Sraj /* Create SA descriptor for this packet */ 728227730Sraj csd->csd_cshd->cshd_config = cci->cci_config; 729227730Sraj csd->csd_cshd->cshd_mac_total_dlen = tmlen; 730227730Sraj 731227730Sraj /* 732227730Sraj * Enable fragmentation if request will not fit 733227730Sraj * into one packet. 734227730Sraj */ 735227730Sraj if (len > 0) { 736227730Sraj if (!fragmented) { 737227730Sraj fragmented = 1; 738227730Sraj csd->csd_cshd->cshd_config |= 739227730Sraj CESA_CSHD_FRAG_FIRST; 740227730Sraj } else 741227730Sraj csd->csd_cshd->cshd_config |= 742227730Sraj CESA_CSHD_FRAG_MIDDLE; 743227730Sraj } else if (fragmented) 744227730Sraj csd->csd_cshd->cshd_config |= 745227730Sraj CESA_CSHD_FRAG_LAST; 746227730Sraj 747227730Sraj if (eskip < cp.cp_size && elen > 0) { 748227730Sraj csd->csd_cshd->cshd_enc_src = 749227730Sraj CESA_DATA(eskip); 750227730Sraj csd->csd_cshd->cshd_enc_dst = 751227730Sraj CESA_DATA(eskip); 752227730Sraj csd->csd_cshd->cshd_enc_dlen = 753227730Sraj MIN(elen, cp.cp_size - eskip); 754227730Sraj } 755227730Sraj 756227730Sraj if (mskip < cp.cp_size && mlen > 0) { 757227730Sraj csd->csd_cshd->cshd_mac_src = 758227730Sraj CESA_DATA(mskip); 759227730Sraj csd->csd_cshd->cshd_mac_dlen = 760227730Sraj MIN(mlen, cp.cp_size - mskip); 761227730Sraj } 762227730Sraj 763227730Sraj elen -= csd->csd_cshd->cshd_enc_dlen; 764227730Sraj eskip -= MIN(eskip, cp.cp_size); 765227730Sraj mlen -= csd->csd_cshd->cshd_mac_dlen; 766227730Sraj mskip -= MIN(mskip, cp.cp_size); 767227730Sraj 768227730Sraj cesa_dump_cshd(sc, csd->csd_cshd); 769227730Sraj 770227730Sraj /* Append packet to the request */ 771227730Sraj error = cesa_append_packet(sc, cr, &cp, csd); 772227730Sraj if (error) 773227730Sraj break; 774227730Sraj 775227730Sraj /* Start a new packet, as current is full */ 776227730Sraj cesa_start_packet(&cp, MIN(mpsize, len)); 777227730Sraj } 778227730Sraj } 779227730Sraj 780227730Sraj if (error) 781227730Sraj break; 782227730Sraj } 783227730Sraj 784227730Sraj if (error) { 785227730Sraj /* 786227730Sraj * Move all allocated resources to the request. They will be 787227730Sraj * freed later. 788227730Sraj */ 789227730Sraj STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyin); 790227730Sraj STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyout); 791227730Sraj cci->cci_error = error; 792227730Sraj } 793227730Sraj} 794227730Sraj 795227730Srajstatic void 796227730Srajcesa_create_chain_cb2(void *arg, bus_dma_segment_t *segs, int nseg, 797227730Sraj bus_size_t size, int error) 798227730Sraj{ 799227730Sraj 800227730Sraj cesa_create_chain_cb(arg, segs, nseg, error); 801227730Sraj} 802227730Sraj 803227730Srajstatic int 804227730Srajcesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr) 805227730Sraj{ 806227730Sraj struct cesa_chain_info cci; 807227730Sraj struct cesa_tdma_desc *ctd; 808227730Sraj uint32_t config; 809227730Sraj int error; 810227730Sraj 811227730Sraj error = 0; 812227730Sraj CESA_LOCK_ASSERT(sc, sessions); 813227730Sraj 814227730Sraj /* Create request metadata */ 815227730Sraj if (cr->cr_enc) { 816227730Sraj if (cr->cr_enc->crd_alg == CRYPTO_AES_CBC && 817227730Sraj (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0) 818227730Sraj memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_aes_dkey, 819227730Sraj cr->cr_cs->cs_klen); 820227730Sraj else 821227730Sraj memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_key, 822227730Sraj cr->cr_cs->cs_klen); 823227730Sraj } 824227730Sraj 825227730Sraj if (cr->cr_mac) { 826227730Sraj memcpy(cr->cr_csd->csd_hiv_in, cr->cr_cs->cs_hiv_in, 827227730Sraj CESA_MAX_HASH_LEN); 828227730Sraj memcpy(cr->cr_csd->csd_hiv_out, cr->cr_cs->cs_hiv_out, 829227730Sraj CESA_MAX_HASH_LEN); 830227730Sraj } 831227730Sraj 832227730Sraj ctd = cesa_tdma_copyin_sa_data(sc, cr); 833227730Sraj if (!ctd) 834227730Sraj return (ENOMEM); 835227730Sraj 836227730Sraj cesa_append_tdesc(cr, ctd); 837227730Sraj 838227730Sraj /* Prepare SA configuration */ 839227730Sraj config = cr->cr_cs->cs_config; 840227730Sraj 841227730Sraj if (cr->cr_enc && (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0) 842227730Sraj config |= CESA_CSHD_DECRYPT; 843227730Sraj if (cr->cr_enc && !cr->cr_mac) 844227730Sraj config |= CESA_CSHD_ENC; 845227730Sraj if (!cr->cr_enc && cr->cr_mac) 846227730Sraj config |= CESA_CSHD_MAC; 847227730Sraj if (cr->cr_enc && cr->cr_mac) 848227730Sraj config |= (config & CESA_CSHD_DECRYPT) ? CESA_CSHD_MAC_AND_ENC : 849227730Sraj CESA_CSHD_ENC_AND_MAC; 850227730Sraj 851227730Sraj /* Create data packets */ 852227730Sraj cci.cci_sc = sc; 853227730Sraj cci.cci_cr = cr; 854227730Sraj cci.cci_enc = cr->cr_enc; 855227730Sraj cci.cci_mac = cr->cr_mac; 856227730Sraj cci.cci_config = config; 857227730Sraj cci.cci_error = 0; 858227730Sraj 859227730Sraj if (cr->cr_crp->crp_flags & CRYPTO_F_IOV) 860227730Sraj error = bus_dmamap_load_uio(sc->sc_data_dtag, 861227730Sraj cr->cr_dmap, (struct uio *)cr->cr_crp->crp_buf, 862227730Sraj cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT); 863227730Sraj else if (cr->cr_crp->crp_flags & CRYPTO_F_IMBUF) 864227730Sraj error = bus_dmamap_load_mbuf(sc->sc_data_dtag, 865227730Sraj cr->cr_dmap, (struct mbuf *)cr->cr_crp->crp_buf, 866227730Sraj cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT); 867227730Sraj else 868227730Sraj error = bus_dmamap_load(sc->sc_data_dtag, 869227730Sraj cr->cr_dmap, cr->cr_crp->crp_buf, 870227730Sraj cr->cr_crp->crp_ilen, cesa_create_chain_cb, &cci, 871227730Sraj BUS_DMA_NOWAIT); 872227730Sraj 873227730Sraj if (!error) 874227730Sraj cr->cr_dmap_loaded = 1; 875227730Sraj 876227730Sraj if (cci.cci_error) 877227730Sraj error = cci.cci_error; 878227730Sraj 879227730Sraj if (error) 880227730Sraj return (error); 881227730Sraj 882227730Sraj /* Read back request metadata */ 883227730Sraj ctd = cesa_tdma_copyout_sa_data(sc, cr); 884227730Sraj if (!ctd) 885227730Sraj return (ENOMEM); 886227730Sraj 887227730Sraj cesa_append_tdesc(cr, ctd); 888227730Sraj 889227730Sraj return (0); 890227730Sraj} 891227730Sraj 892227730Srajstatic void 893227730Srajcesa_execute(struct cesa_softc *sc) 894227730Sraj{ 895227730Sraj struct cesa_tdma_desc *prev_ctd, *ctd; 896227730Sraj struct cesa_request *prev_cr, *cr; 897227730Sraj 898227730Sraj CESA_LOCK(sc, requests); 899227730Sraj 900227730Sraj /* 901227730Sraj * If ready list is empty, there is nothing to execute. If queued list 902227730Sraj * is not empty, the hardware is busy and we cannot start another 903227730Sraj * execution. 904227730Sraj */ 905227730Sraj if (STAILQ_EMPTY(&sc->sc_ready_requests) || 906227730Sraj !STAILQ_EMPTY(&sc->sc_queued_requests)) { 907227730Sraj CESA_UNLOCK(sc, requests); 908227730Sraj return; 909227730Sraj } 910227730Sraj 911227730Sraj /* Move all ready requests to queued list */ 912227730Sraj STAILQ_CONCAT(&sc->sc_queued_requests, &sc->sc_ready_requests); 913227730Sraj STAILQ_INIT(&sc->sc_ready_requests); 914227730Sraj 915227730Sraj /* Create one execution chain from all requests on the list */ 916227730Sraj if (STAILQ_FIRST(&sc->sc_queued_requests) != 917227730Sraj STAILQ_LAST(&sc->sc_queued_requests, cesa_request, cr_stq)) { 918227730Sraj prev_cr = NULL; 919227730Sraj cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_POSTREAD | 920227730Sraj BUS_DMASYNC_POSTWRITE); 921227730Sraj 922227730Sraj STAILQ_FOREACH(cr, &sc->sc_queued_requests, cr_stq) { 923227730Sraj if (prev_cr) { 924227730Sraj ctd = STAILQ_FIRST(&cr->cr_tdesc); 925227730Sraj prev_ctd = STAILQ_LAST(&prev_cr->cr_tdesc, 926227730Sraj cesa_tdma_desc, ctd_stq); 927227730Sraj 928227730Sraj prev_ctd->ctd_cthd->cthd_next = 929227730Sraj ctd->ctd_cthd_paddr; 930227730Sraj } 931227730Sraj 932227730Sraj prev_cr = cr; 933227730Sraj } 934227730Sraj 935227730Sraj cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_PREREAD | 936227730Sraj BUS_DMASYNC_PREWRITE); 937227730Sraj } 938227730Sraj 939227730Sraj /* Start chain execution in hardware */ 940227730Sraj cr = STAILQ_FIRST(&sc->sc_queued_requests); 941227730Sraj ctd = STAILQ_FIRST(&cr->cr_tdesc); 942227730Sraj 943227730Sraj CESA_WRITE(sc, CESA_TDMA_ND, ctd->ctd_cthd_paddr); 944227730Sraj CESA_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE); 945227730Sraj 946227730Sraj CESA_UNLOCK(sc, requests); 947227730Sraj} 948227730Sraj 949227730Srajstatic int 950227730Srajcesa_setup_sram(struct cesa_softc *sc) 951227730Sraj{ 952227730Sraj phandle_t sram_node; 953227730Sraj ihandle_t sram_ihandle; 954227730Sraj pcell_t sram_handle, sram_reg; 955227730Sraj 956227730Sraj if (OF_getprop(ofw_bus_get_node(sc->sc_dev), "sram-handle", 957227730Sraj (void *)&sram_handle, sizeof(sram_handle)) <= 0) 958227730Sraj return (ENXIO); 959227730Sraj 960227730Sraj sram_ihandle = (ihandle_t)sram_handle; 961227730Sraj sram_ihandle = fdt32_to_cpu(sram_ihandle); 962227730Sraj sram_node = OF_instance_to_package(sram_ihandle); 963227730Sraj 964227730Sraj if (OF_getprop(sram_node, "reg", (void *)&sram_reg, 965227730Sraj sizeof(sram_reg)) <= 0) 966227730Sraj return (ENXIO); 967227730Sraj 968227730Sraj sc->sc_sram_base = fdt32_to_cpu(sram_reg); 969227730Sraj 970227730Sraj return (0); 971227730Sraj} 972227730Sraj 973227730Srajstatic int 974227730Srajcesa_probe(device_t dev) 975227730Sraj{ 976227730Sraj if (!ofw_bus_is_compatible(dev, "mrvl,cesa")) 977227730Sraj return (ENXIO); 978227730Sraj 979227730Sraj device_set_desc(dev, "Marvell Cryptographic Engine and Security " 980227730Sraj "Accelerator"); 981227730Sraj 982227730Sraj return (BUS_PROBE_DEFAULT); 983227730Sraj} 984227730Sraj 985227730Srajstatic int 986227730Srajcesa_attach(device_t dev) 987227730Sraj{ 988227730Sraj struct cesa_softc *sc; 989227730Sraj uint32_t d, r; 990227730Sraj int error; 991227730Sraj int i; 992227730Sraj 993227730Sraj sc = device_get_softc(dev); 994227730Sraj sc->sc_blocked = 0; 995227730Sraj sc->sc_error = 0; 996227730Sraj sc->sc_dev = dev; 997227730Sraj 998227730Sraj error = cesa_setup_sram(sc); 999227730Sraj if (error) { 1000227730Sraj device_printf(dev, "could not setup SRAM\n"); 1001227730Sraj return (error); 1002227730Sraj } 1003227730Sraj 1004227730Sraj soc_id(&d, &r); 1005227730Sraj 1006227730Sraj switch (d) { 1007227730Sraj case MV_DEV_88F6281: 1008227730Sraj sc->sc_tperr = 0; 1009227730Sraj break; 1010227730Sraj case MV_DEV_MV78100: 1011227730Sraj case MV_DEV_MV78100_Z0: 1012227730Sraj sc->sc_tperr = CESA_ICR_TPERR; 1013227730Sraj break; 1014227730Sraj default: 1015227730Sraj return (ENXIO); 1016227730Sraj } 1017227730Sraj 1018227730Sraj /* Initialize mutexes */ 1019227730Sraj mtx_init(&sc->sc_sc_lock, device_get_nameunit(dev), 1020227730Sraj "CESA Shared Data", MTX_DEF); 1021227730Sraj mtx_init(&sc->sc_tdesc_lock, device_get_nameunit(dev), 1022227730Sraj "CESA TDMA Descriptors Pool", MTX_DEF); 1023227730Sraj mtx_init(&sc->sc_sdesc_lock, device_get_nameunit(dev), 1024227730Sraj "CESA SA Descriptors Pool", MTX_DEF); 1025227730Sraj mtx_init(&sc->sc_requests_lock, device_get_nameunit(dev), 1026227730Sraj "CESA Requests Pool", MTX_DEF); 1027227730Sraj mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev), 1028227730Sraj "CESA Sessions Pool", MTX_DEF); 1029227730Sraj 1030227730Sraj /* Allocate I/O and IRQ resources */ 1031227730Sraj error = bus_alloc_resources(dev, cesa_res_spec, sc->sc_res); 1032227730Sraj if (error) { 1033227730Sraj device_printf(dev, "could not allocate resources\n"); 1034227730Sraj goto err0; 1035227730Sraj } 1036227730Sraj 1037227730Sraj sc->sc_bsh = rman_get_bushandle(*(sc->sc_res)); 1038227730Sraj sc->sc_bst = rman_get_bustag(*(sc->sc_res)); 1039227730Sraj 1040227730Sraj /* Setup interrupt handler */ 1041227730Sraj error = bus_setup_intr(dev, sc->sc_res[1], INTR_TYPE_NET | INTR_MPSAFE, 1042227730Sraj NULL, cesa_intr, sc, &(sc->sc_icookie)); 1043227730Sraj if (error) { 1044227730Sraj device_printf(dev, "could not setup engine completion irq\n"); 1045227730Sraj goto err1; 1046227730Sraj } 1047227730Sraj 1048227730Sraj /* Create DMA tag for processed data */ 1049232883Sscottl error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 1050227730Sraj 1, 0, /* alignment, boundary */ 1051227730Sraj BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 1052227730Sraj BUS_SPACE_MAXADDR, /* highaddr */ 1053227730Sraj NULL, NULL, /* filtfunc, filtfuncarg */ 1054227730Sraj CESA_MAX_REQUEST_SIZE, /* maxsize */ 1055227730Sraj CESA_MAX_FRAGMENTS, /* nsegments */ 1056227730Sraj CESA_MAX_REQUEST_SIZE, 0, /* maxsegsz, flags */ 1057227730Sraj NULL, NULL, /* lockfunc, lockfuncarg */ 1058227730Sraj &sc->sc_data_dtag); /* dmat */ 1059227730Sraj if (error) 1060227730Sraj goto err2; 1061227730Sraj 1062227730Sraj /* Initialize data structures: TDMA Descriptors Pool */ 1063227730Sraj error = cesa_alloc_dma_mem(sc, &sc->sc_tdesc_cdm, 1064227730Sraj CESA_TDMA_DESCRIPTORS * sizeof(struct cesa_tdma_hdesc)); 1065227730Sraj if (error) 1066227730Sraj goto err3; 1067227730Sraj 1068227730Sraj STAILQ_INIT(&sc->sc_free_tdesc); 1069227730Sraj for (i = 0; i < CESA_TDMA_DESCRIPTORS; i++) { 1070227730Sraj sc->sc_tdesc[i].ctd_cthd = 1071227730Sraj (struct cesa_tdma_hdesc *)(sc->sc_tdesc_cdm.cdm_vaddr) + i; 1072227730Sraj sc->sc_tdesc[i].ctd_cthd_paddr = sc->sc_tdesc_cdm.cdm_paddr + 1073227730Sraj (i * sizeof(struct cesa_tdma_hdesc)); 1074227730Sraj STAILQ_INSERT_TAIL(&sc->sc_free_tdesc, &sc->sc_tdesc[i], 1075227730Sraj ctd_stq); 1076227730Sraj } 1077227730Sraj 1078227730Sraj /* Initialize data structures: SA Descriptors Pool */ 1079227730Sraj error = cesa_alloc_dma_mem(sc, &sc->sc_sdesc_cdm, 1080227730Sraj CESA_SA_DESCRIPTORS * sizeof(struct cesa_sa_hdesc)); 1081227730Sraj if (error) 1082227730Sraj goto err4; 1083227730Sraj 1084227730Sraj STAILQ_INIT(&sc->sc_free_sdesc); 1085227730Sraj for (i = 0; i < CESA_SA_DESCRIPTORS; i++) { 1086227730Sraj sc->sc_sdesc[i].csd_cshd = 1087227730Sraj (struct cesa_sa_hdesc *)(sc->sc_sdesc_cdm.cdm_vaddr) + i; 1088227730Sraj sc->sc_sdesc[i].csd_cshd_paddr = sc->sc_sdesc_cdm.cdm_paddr + 1089227730Sraj (i * sizeof(struct cesa_sa_hdesc)); 1090227730Sraj STAILQ_INSERT_TAIL(&sc->sc_free_sdesc, &sc->sc_sdesc[i], 1091227730Sraj csd_stq); 1092227730Sraj } 1093227730Sraj 1094227730Sraj /* Initialize data structures: Requests Pool */ 1095227730Sraj error = cesa_alloc_dma_mem(sc, &sc->sc_requests_cdm, 1096227730Sraj CESA_REQUESTS * sizeof(struct cesa_sa_data)); 1097227730Sraj if (error) 1098227730Sraj goto err5; 1099227730Sraj 1100227730Sraj STAILQ_INIT(&sc->sc_free_requests); 1101227730Sraj STAILQ_INIT(&sc->sc_ready_requests); 1102227730Sraj STAILQ_INIT(&sc->sc_queued_requests); 1103227730Sraj for (i = 0; i < CESA_REQUESTS; i++) { 1104227730Sraj sc->sc_requests[i].cr_csd = 1105227730Sraj (struct cesa_sa_data *)(sc->sc_requests_cdm.cdm_vaddr) + i; 1106227730Sraj sc->sc_requests[i].cr_csd_paddr = 1107227730Sraj sc->sc_requests_cdm.cdm_paddr + 1108227730Sraj (i * sizeof(struct cesa_sa_data)); 1109227730Sraj 1110227730Sraj /* Preallocate DMA maps */ 1111227730Sraj error = bus_dmamap_create(sc->sc_data_dtag, 0, 1112227730Sraj &sc->sc_requests[i].cr_dmap); 1113227730Sraj if (error && i > 0) { 1114227730Sraj i--; 1115227730Sraj do { 1116227730Sraj bus_dmamap_destroy(sc->sc_data_dtag, 1117227730Sraj sc->sc_requests[i].cr_dmap); 1118227730Sraj } while (i--); 1119227730Sraj 1120227730Sraj goto err6; 1121227730Sraj } 1122227730Sraj 1123227730Sraj STAILQ_INSERT_TAIL(&sc->sc_free_requests, &sc->sc_requests[i], 1124227730Sraj cr_stq); 1125227730Sraj } 1126227730Sraj 1127227730Sraj /* Initialize data structures: Sessions Pool */ 1128227730Sraj STAILQ_INIT(&sc->sc_free_sessions); 1129227730Sraj for (i = 0; i < CESA_SESSIONS; i++) { 1130227730Sraj sc->sc_sessions[i].cs_sid = i; 1131227730Sraj STAILQ_INSERT_TAIL(&sc->sc_free_sessions, &sc->sc_sessions[i], 1132227730Sraj cs_stq); 1133227730Sraj } 1134227730Sraj 1135227730Sraj /* 1136227730Sraj * Initialize TDMA: 1137227730Sraj * - Burst limit: 128 bytes, 1138227730Sraj * - Outstanding reads enabled, 1139227730Sraj * - No byte-swap. 1140227730Sraj */ 1141227730Sraj CESA_WRITE(sc, CESA_TDMA_CR, CESA_TDMA_CR_DBL128 | CESA_TDMA_CR_SBL128 | 1142227730Sraj CESA_TDMA_CR_ORDEN | CESA_TDMA_CR_NBS | CESA_TDMA_CR_ENABLE); 1143227730Sraj 1144227730Sraj /* 1145227730Sraj * Initialize SA: 1146227730Sraj * - SA descriptor is present at beginning of CESA SRAM, 1147227730Sraj * - Multi-packet chain mode, 1148227730Sraj * - Cooperation with TDMA enabled. 1149227730Sraj */ 1150227730Sraj CESA_WRITE(sc, CESA_SA_DPR, 0); 1151227730Sraj CESA_WRITE(sc, CESA_SA_CR, CESA_SA_CR_ACTIVATE_TDMA | 1152227730Sraj CESA_SA_CR_WAIT_FOR_TDMA | CESA_SA_CR_MULTI_MODE); 1153227730Sraj 1154227730Sraj /* Unmask interrupts */ 1155227730Sraj CESA_WRITE(sc, CESA_ICR, 0); 1156227730Sraj CESA_WRITE(sc, CESA_ICM, CESA_ICM_ACCTDMA | sc->sc_tperr); 1157227730Sraj CESA_WRITE(sc, CESA_TDMA_ECR, 0); 1158227730Sraj CESA_WRITE(sc, CESA_TDMA_EMR, CESA_TDMA_EMR_MISS | 1159227730Sraj CESA_TDMA_EMR_DOUBLE_HIT | CESA_TDMA_EMR_BOTH_HIT | 1160227730Sraj CESA_TDMA_EMR_DATA_ERROR); 1161227730Sraj 1162227730Sraj /* Register in OCF */ 1163227730Sraj sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); 1164227730Sraj if (sc->sc_cid) { 1165227730Sraj device_printf(dev, "could not get crypto driver id\n"); 1166227730Sraj goto err7; 1167227730Sraj } 1168227730Sraj 1169227730Sraj crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); 1170227730Sraj crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); 1171227730Sraj crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); 1172227730Sraj crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0); 1173227730Sraj crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 1174227730Sraj crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0); 1175227730Sraj crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 1176227730Sraj 1177227730Sraj return (0); 1178227730Srajerr7: 1179227730Sraj for (i = 0; i < CESA_REQUESTS; i++) 1180227730Sraj bus_dmamap_destroy(sc->sc_data_dtag, 1181227730Sraj sc->sc_requests[i].cr_dmap); 1182227730Srajerr6: 1183227730Sraj cesa_free_dma_mem(&sc->sc_requests_cdm); 1184227730Srajerr5: 1185227730Sraj cesa_free_dma_mem(&sc->sc_sdesc_cdm); 1186227730Srajerr4: 1187227730Sraj cesa_free_dma_mem(&sc->sc_tdesc_cdm); 1188227730Srajerr3: 1189227730Sraj bus_dma_tag_destroy(sc->sc_data_dtag); 1190227730Srajerr2: 1191227730Sraj bus_teardown_intr(dev, sc->sc_res[1], sc->sc_icookie); 1192227730Srajerr1: 1193227730Sraj bus_release_resources(dev, cesa_res_spec, sc->sc_res); 1194227730Srajerr0: 1195227730Sraj mtx_destroy(&sc->sc_sessions_lock); 1196227730Sraj mtx_destroy(&sc->sc_requests_lock); 1197227730Sraj mtx_destroy(&sc->sc_sdesc_lock); 1198227730Sraj mtx_destroy(&sc->sc_tdesc_lock); 1199227730Sraj mtx_destroy(&sc->sc_sc_lock); 1200227730Sraj return (ENXIO); 1201227730Sraj} 1202227730Sraj 1203227730Srajstatic int 1204227730Srajcesa_detach(device_t dev) 1205227730Sraj{ 1206227730Sraj struct cesa_softc *sc; 1207227730Sraj int i; 1208227730Sraj 1209227730Sraj sc = device_get_softc(dev); 1210227730Sraj 1211227730Sraj /* TODO: Wait for queued requests completion before shutdown. */ 1212227730Sraj 1213227730Sraj /* Mask interrupts */ 1214227730Sraj CESA_WRITE(sc, CESA_ICM, 0); 1215227730Sraj CESA_WRITE(sc, CESA_TDMA_EMR, 0); 1216227730Sraj 1217227730Sraj /* Unregister from OCF */ 1218227730Sraj crypto_unregister_all(sc->sc_cid); 1219227730Sraj 1220227730Sraj /* Free DMA Maps */ 1221227730Sraj for (i = 0; i < CESA_REQUESTS; i++) 1222227730Sraj bus_dmamap_destroy(sc->sc_data_dtag, 1223227730Sraj sc->sc_requests[i].cr_dmap); 1224227730Sraj 1225227730Sraj /* Free DMA Memory */ 1226227730Sraj cesa_free_dma_mem(&sc->sc_requests_cdm); 1227227730Sraj cesa_free_dma_mem(&sc->sc_sdesc_cdm); 1228227730Sraj cesa_free_dma_mem(&sc->sc_tdesc_cdm); 1229227730Sraj 1230227730Sraj /* Free DMA Tag */ 1231227730Sraj bus_dma_tag_destroy(sc->sc_data_dtag); 1232227730Sraj 1233227730Sraj /* Stop interrupt */ 1234227730Sraj bus_teardown_intr(dev, sc->sc_res[1], sc->sc_icookie); 1235227730Sraj 1236227730Sraj /* Relase I/O and IRQ resources */ 1237227730Sraj bus_release_resources(dev, cesa_res_spec, sc->sc_res); 1238227730Sraj 1239227730Sraj /* Destory mutexes */ 1240227730Sraj mtx_destroy(&sc->sc_sessions_lock); 1241227730Sraj mtx_destroy(&sc->sc_requests_lock); 1242227730Sraj mtx_destroy(&sc->sc_sdesc_lock); 1243227730Sraj mtx_destroy(&sc->sc_tdesc_lock); 1244227730Sraj mtx_destroy(&sc->sc_sc_lock); 1245227730Sraj 1246227730Sraj return (0); 1247227730Sraj} 1248227730Sraj 1249227730Srajstatic void 1250227730Srajcesa_intr(void *arg) 1251227730Sraj{ 1252227730Sraj STAILQ_HEAD(, cesa_request) requests; 1253227730Sraj struct cesa_request *cr, *tmp; 1254227730Sraj struct cesa_softc *sc; 1255227730Sraj uint32_t ecr, icr; 1256227730Sraj int blocked; 1257227730Sraj 1258227730Sraj sc = arg; 1259227730Sraj 1260227730Sraj /* Ack interrupt */ 1261227730Sraj ecr = CESA_READ(sc, CESA_TDMA_ECR); 1262227730Sraj CESA_WRITE(sc, CESA_TDMA_ECR, 0); 1263227730Sraj icr = CESA_READ(sc, CESA_ICR); 1264227730Sraj CESA_WRITE(sc, CESA_ICR, 0); 1265227730Sraj 1266227730Sraj /* Check for TDMA errors */ 1267227730Sraj if (ecr & CESA_TDMA_ECR_MISS) { 1268227730Sraj device_printf(sc->sc_dev, "TDMA Miss error detected!\n"); 1269227730Sraj sc->sc_error = EIO; 1270227730Sraj } 1271227730Sraj 1272227730Sraj if (ecr & CESA_TDMA_ECR_DOUBLE_HIT) { 1273227730Sraj device_printf(sc->sc_dev, "TDMA Double Hit error detected!\n"); 1274227730Sraj sc->sc_error = EIO; 1275227730Sraj } 1276227730Sraj 1277227730Sraj if (ecr & CESA_TDMA_ECR_BOTH_HIT) { 1278227730Sraj device_printf(sc->sc_dev, "TDMA Both Hit error detected!\n"); 1279227730Sraj sc->sc_error = EIO; 1280227730Sraj } 1281227730Sraj 1282227730Sraj if (ecr & CESA_TDMA_ECR_DATA_ERROR) { 1283227730Sraj device_printf(sc->sc_dev, "TDMA Data error detected!\n"); 1284227730Sraj sc->sc_error = EIO; 1285227730Sraj } 1286227730Sraj 1287227730Sraj /* Check for CESA errors */ 1288227730Sraj if (icr & sc->sc_tperr) { 1289227730Sraj device_printf(sc->sc_dev, "CESA SRAM Parity error detected!\n"); 1290227730Sraj sc->sc_error = EIO; 1291227730Sraj } 1292227730Sraj 1293227730Sraj /* If there is nothing more to do, return */ 1294227730Sraj if ((icr & CESA_ICR_ACCTDMA) == 0) 1295227730Sraj return; 1296227730Sraj 1297227730Sraj /* Get all finished requests */ 1298227730Sraj CESA_LOCK(sc, requests); 1299227730Sraj STAILQ_INIT(&requests); 1300227730Sraj STAILQ_CONCAT(&requests, &sc->sc_queued_requests); 1301227730Sraj STAILQ_INIT(&sc->sc_queued_requests); 1302227730Sraj CESA_UNLOCK(sc, requests); 1303227730Sraj 1304227730Sraj /* Execute all ready requests */ 1305227730Sraj cesa_execute(sc); 1306227730Sraj 1307227730Sraj /* Process completed requests */ 1308227730Sraj cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_POSTREAD | 1309227730Sraj BUS_DMASYNC_POSTWRITE); 1310227730Sraj 1311227730Sraj STAILQ_FOREACH_SAFE(cr, &requests, cr_stq, tmp) { 1312227730Sraj bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, 1313227730Sraj BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1314227730Sraj 1315227730Sraj cr->cr_crp->crp_etype = sc->sc_error; 1316227730Sraj if (cr->cr_mac) 1317227730Sraj crypto_copyback(cr->cr_crp->crp_flags, 1318227730Sraj cr->cr_crp->crp_buf, cr->cr_mac->crd_inject, 1319227730Sraj cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash); 1320227730Sraj 1321227730Sraj crypto_done(cr->cr_crp); 1322227730Sraj cesa_free_request(sc, cr); 1323227730Sraj } 1324227730Sraj 1325227730Sraj cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_PREREAD | 1326227730Sraj BUS_DMASYNC_PREWRITE); 1327227730Sraj 1328227730Sraj sc->sc_error = 0; 1329227730Sraj 1330227730Sraj /* Unblock driver if it ran out of resources */ 1331227730Sraj CESA_LOCK(sc, sc); 1332227730Sraj blocked = sc->sc_blocked; 1333227730Sraj sc->sc_blocked = 0; 1334227730Sraj CESA_UNLOCK(sc, sc); 1335227730Sraj 1336227730Sraj if (blocked) 1337227730Sraj crypto_unblock(sc->sc_cid, blocked); 1338227730Sraj} 1339227730Sraj 1340227730Srajstatic int 1341227730Srajcesa_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri) 1342227730Sraj{ 1343227730Sraj struct cesa_session *cs; 1344227730Sraj struct cesa_softc *sc; 1345227730Sraj struct cryptoini *enc; 1346227730Sraj struct cryptoini *mac; 1347227730Sraj int error; 1348227730Sraj 1349227730Sraj sc = device_get_softc(dev); 1350227730Sraj enc = NULL; 1351227730Sraj mac = NULL; 1352227730Sraj error = 0; 1353227730Sraj 1354227730Sraj /* Check and parse input */ 1355227730Sraj if (cesa_is_hash(cri->cri_alg)) 1356227730Sraj mac = cri; 1357227730Sraj else 1358227730Sraj enc = cri; 1359227730Sraj 1360227730Sraj cri = cri->cri_next; 1361227730Sraj 1362227730Sraj if (cri) { 1363227730Sraj if (!enc && !cesa_is_hash(cri->cri_alg)) 1364227730Sraj enc = cri; 1365227730Sraj 1366227730Sraj if (!mac && cesa_is_hash(cri->cri_alg)) 1367227730Sraj mac = cri; 1368227730Sraj 1369227730Sraj if (cri->cri_next || !(enc && mac)) 1370227730Sraj return (EINVAL); 1371227730Sraj } 1372227730Sraj 1373227730Sraj if ((enc && (enc->cri_klen / 8) > CESA_MAX_KEY_LEN) || 1374227730Sraj (mac && (mac->cri_klen / 8) > CESA_MAX_MKEY_LEN)) 1375227730Sraj return (E2BIG); 1376227730Sraj 1377227730Sraj /* Allocate session */ 1378227730Sraj cs = cesa_alloc_session(sc); 1379227730Sraj if (!cs) 1380227730Sraj return (ENOMEM); 1381227730Sraj 1382227730Sraj /* Prepare CESA configuration */ 1383227730Sraj cs->cs_config = 0; 1384227730Sraj cs->cs_ivlen = 1; 1385227730Sraj cs->cs_mblen = 1; 1386227730Sraj 1387227730Sraj if (enc) { 1388227730Sraj switch (enc->cri_alg) { 1389227730Sraj case CRYPTO_AES_CBC: 1390227730Sraj cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC; 1391227730Sraj cs->cs_ivlen = AES_BLOCK_LEN; 1392227730Sraj break; 1393227730Sraj case CRYPTO_DES_CBC: 1394227730Sraj cs->cs_config |= CESA_CSHD_DES | CESA_CSHD_CBC; 1395227730Sraj cs->cs_ivlen = DES_BLOCK_LEN; 1396227730Sraj break; 1397227730Sraj case CRYPTO_3DES_CBC: 1398227730Sraj cs->cs_config |= CESA_CSHD_3DES | CESA_CSHD_3DES_EDE | 1399227730Sraj CESA_CSHD_CBC; 1400227730Sraj cs->cs_ivlen = DES3_BLOCK_LEN; 1401227730Sraj break; 1402227730Sraj default: 1403227730Sraj error = EINVAL; 1404227730Sraj break; 1405227730Sraj } 1406227730Sraj } 1407227730Sraj 1408227730Sraj if (!error && mac) { 1409227730Sraj switch (mac->cri_alg) { 1410227730Sraj case CRYPTO_MD5: 1411227730Sraj cs->cs_config |= CESA_CSHD_MD5; 1412227730Sraj cs->cs_mblen = 1; 1413227730Sraj cs->cs_hlen = MD5_HASH_LEN; 1414227730Sraj break; 1415227730Sraj case CRYPTO_MD5_HMAC: 1416227730Sraj cs->cs_config |= CESA_CSHD_MD5_HMAC; 1417227730Sraj cs->cs_mblen = MD5_HMAC_BLOCK_LEN; 1418227730Sraj cs->cs_hlen = CESA_HMAC_HASH_LENGTH; 1419227730Sraj break; 1420227730Sraj case CRYPTO_SHA1: 1421227730Sraj cs->cs_config |= CESA_CSHD_SHA1; 1422227730Sraj cs->cs_mblen = 1; 1423227730Sraj cs->cs_hlen = SHA1_HASH_LEN; 1424227730Sraj break; 1425227730Sraj case CRYPTO_SHA1_HMAC: 1426227730Sraj cs->cs_config |= CESA_CSHD_SHA1_HMAC; 1427227730Sraj cs->cs_mblen = SHA1_HMAC_BLOCK_LEN; 1428227730Sraj cs->cs_hlen = CESA_HMAC_HASH_LENGTH; 1429227730Sraj break; 1430227730Sraj default: 1431227730Sraj error = EINVAL; 1432227730Sraj break; 1433227730Sraj } 1434227730Sraj } 1435227730Sraj 1436227730Sraj /* Save cipher key */ 1437227730Sraj if (!error && enc && enc->cri_key) { 1438227730Sraj cs->cs_klen = enc->cri_klen / 8; 1439227730Sraj memcpy(cs->cs_key, enc->cri_key, cs->cs_klen); 1440227730Sraj if (enc->cri_alg == CRYPTO_AES_CBC) 1441227730Sraj error = cesa_prep_aes_key(cs); 1442227730Sraj } 1443227730Sraj 1444227730Sraj /* Save digest key */ 1445227730Sraj if (!error && mac && mac->cri_key) 1446227730Sraj error = cesa_set_mkey(cs, mac->cri_alg, mac->cri_key, 1447227730Sraj mac->cri_klen / 8); 1448227730Sraj 1449227730Sraj if (error) { 1450227730Sraj cesa_free_session(sc, cs); 1451227730Sraj return (EINVAL); 1452227730Sraj } 1453227730Sraj 1454227730Sraj *sidp = cs->cs_sid; 1455227730Sraj 1456227730Sraj return (0); 1457227730Sraj} 1458227730Sraj 1459227730Srajstatic int 1460227730Srajcesa_freesession(device_t dev, uint64_t tid) 1461227730Sraj{ 1462227730Sraj struct cesa_session *cs; 1463227730Sraj struct cesa_softc *sc; 1464227730Sraj 1465227730Sraj sc = device_get_softc(dev); 1466227730Sraj cs = cesa_get_session(sc, CRYPTO_SESID2LID(tid)); 1467227730Sraj if (!cs) 1468227730Sraj return (EINVAL); 1469227730Sraj 1470227730Sraj /* Free session */ 1471227730Sraj cesa_free_session(sc, cs); 1472227730Sraj 1473227730Sraj return (0); 1474227730Sraj} 1475227730Sraj 1476227730Srajstatic int 1477227730Srajcesa_process(device_t dev, struct cryptop *crp, int hint) 1478227730Sraj{ 1479227730Sraj struct cesa_request *cr; 1480227730Sraj struct cesa_session *cs; 1481227730Sraj struct cryptodesc *crd; 1482227730Sraj struct cryptodesc *enc; 1483227730Sraj struct cryptodesc *mac; 1484227730Sraj struct cesa_softc *sc; 1485227730Sraj int error; 1486227730Sraj 1487227730Sraj sc = device_get_softc(dev); 1488227730Sraj crd = crp->crp_desc; 1489227730Sraj enc = NULL; 1490227730Sraj mac = NULL; 1491227730Sraj error = 0; 1492227730Sraj 1493227730Sraj /* Check session ID */ 1494227730Sraj cs = cesa_get_session(sc, CRYPTO_SESID2LID(crp->crp_sid)); 1495227730Sraj if (!cs) { 1496227730Sraj crp->crp_etype = EINVAL; 1497227730Sraj crypto_done(crp); 1498227730Sraj return (0); 1499227730Sraj } 1500227730Sraj 1501227730Sraj /* Check and parse input */ 1502227730Sraj if (crp->crp_ilen > CESA_MAX_REQUEST_SIZE) { 1503227730Sraj crp->crp_etype = E2BIG; 1504227730Sraj crypto_done(crp); 1505227730Sraj return (0); 1506227730Sraj } 1507227730Sraj 1508227730Sraj if (cesa_is_hash(crd->crd_alg)) 1509227730Sraj mac = crd; 1510227730Sraj else 1511227730Sraj enc = crd; 1512227730Sraj 1513227730Sraj crd = crd->crd_next; 1514227730Sraj 1515227730Sraj if (crd) { 1516227730Sraj if (!enc && !cesa_is_hash(crd->crd_alg)) 1517227730Sraj enc = crd; 1518227730Sraj 1519227730Sraj if (!mac && cesa_is_hash(crd->crd_alg)) 1520227730Sraj mac = crd; 1521227730Sraj 1522227730Sraj if (crd->crd_next || !(enc && mac)) { 1523227730Sraj crp->crp_etype = EINVAL; 1524227730Sraj crypto_done(crp); 1525227730Sraj return (0); 1526227730Sraj } 1527227730Sraj } 1528227730Sraj 1529227730Sraj /* 1530227730Sraj * Get request descriptor. Block driver if there is no free 1531227730Sraj * descriptors in pool. 1532227730Sraj */ 1533227730Sraj cr = cesa_alloc_request(sc); 1534227730Sraj if (!cr) { 1535227730Sraj CESA_LOCK(sc, sc); 1536227730Sraj sc->sc_blocked = CRYPTO_SYMQ; 1537227730Sraj CESA_UNLOCK(sc, sc); 1538227730Sraj return (ERESTART); 1539227730Sraj } 1540227730Sraj 1541227730Sraj /* Prepare request */ 1542227730Sraj cr->cr_crp = crp; 1543227730Sraj cr->cr_enc = enc; 1544227730Sraj cr->cr_mac = mac; 1545227730Sraj cr->cr_cs = cs; 1546227730Sraj 1547227730Sraj CESA_LOCK(sc, sessions); 1548227730Sraj cesa_sync_desc(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1549227730Sraj 1550227730Sraj if (enc && enc->crd_flags & CRD_F_ENCRYPT) { 1551227730Sraj if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1552227730Sraj memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen); 1553227730Sraj else 1554227730Sraj arc4rand(cr->cr_csd->csd_iv, cs->cs_ivlen, 0); 1555227730Sraj 1556227730Sraj if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0) 1557227730Sraj crypto_copyback(crp->crp_flags, crp->crp_buf, 1558227730Sraj enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv); 1559227730Sraj } else if (enc) { 1560227730Sraj if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1561227730Sraj memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen); 1562227730Sraj else 1563227730Sraj crypto_copydata(crp->crp_flags, crp->crp_buf, 1564227730Sraj enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv); 1565227730Sraj } 1566227730Sraj 1567227730Sraj if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) { 1568227730Sraj if ((enc->crd_klen / 8) <= CESA_MAX_KEY_LEN) { 1569227730Sraj cs->cs_klen = enc->crd_klen / 8; 1570227730Sraj memcpy(cs->cs_key, enc->crd_key, cs->cs_klen); 1571227730Sraj if (enc->crd_alg == CRYPTO_AES_CBC) 1572227730Sraj error = cesa_prep_aes_key(cs); 1573227730Sraj } else 1574227730Sraj error = E2BIG; 1575227730Sraj } 1576227730Sraj 1577227730Sraj if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) { 1578227730Sraj if ((mac->crd_klen / 8) <= CESA_MAX_MKEY_LEN) 1579227730Sraj error = cesa_set_mkey(cs, mac->crd_alg, mac->crd_key, 1580227730Sraj mac->crd_klen / 8); 1581227730Sraj else 1582227730Sraj error = E2BIG; 1583227730Sraj } 1584227730Sraj 1585227730Sraj /* Convert request to chain of TDMA and SA descriptors */ 1586227730Sraj if (!error) 1587227730Sraj error = cesa_create_chain(sc, cr); 1588227730Sraj 1589227730Sraj cesa_sync_desc(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1590227730Sraj CESA_UNLOCK(sc, sessions); 1591227730Sraj 1592227730Sraj if (error) { 1593227730Sraj cesa_free_request(sc, cr); 1594227730Sraj crp->crp_etype = error; 1595227730Sraj crypto_done(crp); 1596227730Sraj return (0); 1597227730Sraj } 1598227730Sraj 1599227730Sraj bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, BUS_DMASYNC_PREREAD | 1600227730Sraj BUS_DMASYNC_PREWRITE); 1601227730Sraj 1602227730Sraj /* Enqueue request to execution */ 1603227730Sraj cesa_enqueue_request(sc, cr); 1604227730Sraj 1605227730Sraj /* Start execution, if we have no more requests in queue */ 1606227730Sraj if ((hint & CRYPTO_HINT_MORE) == 0) 1607227730Sraj cesa_execute(sc); 1608227730Sraj 1609227730Sraj return (0); 1610227730Sraj} 1611