1233541Sjchandra/*- 2233541Sjchandra * Copyright (c) 2003-2012 Broadcom Corporation 3233541Sjchandra * All Rights Reserved 4233541Sjchandra * 5233541Sjchandra * Redistribution and use in source and binary forms, with or without 6233541Sjchandra * modification, are permitted provided that the following conditions 7233541Sjchandra * are met: 8233541Sjchandra * 9233541Sjchandra * 1. Redistributions of source code must retain the above copyright 10233541Sjchandra * notice, this list of conditions and the following disclaimer. 11233541Sjchandra * 2. Redistributions in binary form must reproduce the above copyright 12233541Sjchandra * notice, this list of conditions and the following disclaimer in 13233541Sjchandra * the documentation and/or other materials provided with the 14233541Sjchandra * distribution. 15233541Sjchandra * 16233541Sjchandra * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR 17233541Sjchandra * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18233541Sjchandra * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19233541Sjchandra * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE 20233541Sjchandra * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21233541Sjchandra * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22233541Sjchandra * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 23233541Sjchandra * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24233541Sjchandra * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 25233541Sjchandra * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 26233541Sjchandra * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27233541Sjchandra */ 28233541Sjchandra 29233541Sjchandra#include <sys/cdefs.h> 30233541Sjchandra__FBSDID("$FreeBSD$"); 31233541Sjchandra 32233541Sjchandra#include <sys/cdefs.h> 33233541Sjchandra#include <sys/param.h> 34233541Sjchandra#include <sys/systm.h> 35233541Sjchandra#include <sys/proc.h> 36233541Sjchandra#include <sys/errno.h> 37233541Sjchandra#include <sys/malloc.h> 38233541Sjchandra#include <sys/kernel.h> 39233541Sjchandra#include <sys/module.h> 40233541Sjchandra#include <sys/mbuf.h> 41233541Sjchandra#include <sys/lock.h> 42233541Sjchandra#include <sys/mutex.h> 43233541Sjchandra#include <sys/sysctl.h> 44233541Sjchandra#include <sys/bus.h> 45233541Sjchandra#include <sys/random.h> 46233541Sjchandra#include <sys/rman.h> 47233541Sjchandra#include <sys/uio.h> 48233541Sjchandra#include <sys/kobj.h> 49233541Sjchandra 50233541Sjchandra#include <dev/pci/pcivar.h> 51233541Sjchandra 52233541Sjchandra#include <opencrypto/cryptodev.h> 53233541Sjchandra 54233541Sjchandra#include "cryptodev_if.h" 55233541Sjchandra 56233541Sjchandra#include <vm/vm.h> 57233541Sjchandra#include <vm/pmap.h> 58233541Sjchandra 59233541Sjchandra#include <mips/nlm/hal/haldefs.h> 60233541Sjchandra#include <mips/nlm/hal/iomap.h> 61233541Sjchandra#include <mips/nlm/xlp.h> 62233541Sjchandra#include <mips/nlm/hal/sys.h> 63233541Sjchandra#include <mips/nlm/hal/fmn.h> 64233541Sjchandra#include <mips/nlm/hal/nlmsaelib.h> 65233541Sjchandra#include <mips/nlm/dev/sec/nlmseclib.h> 66233541Sjchandra#include <mips/nlm/hal/cop2.h> 67233541Sjchandra#include <mips/nlm/hal/mips-extns.h> 68233541Sjchandra#include <mips/nlm/msgring.h> 69233541Sjchandra 70233541Sjchandraunsigned int creditleft; 71233541Sjchandra 72233541Sjchandravoid xlp_sec_print_data(struct cryptop *crp); 73233541Sjchandra 74233541Sjchandrastatic int xlp_sec_init(struct xlp_sec_softc *sc); 75233541Sjchandrastatic int xlp_sec_newsession(device_t , uint32_t *, struct cryptoini *); 76233541Sjchandrastatic int xlp_sec_freesession(device_t , uint64_t); 77233541Sjchandrastatic int xlp_sec_process(device_t , struct cryptop *, int); 78233541Sjchandrastatic int xlp_copyiv(struct xlp_sec_softc *, struct xlp_sec_command *, 79233541Sjchandra struct cryptodesc *enccrd); 80233541Sjchandrastatic int xlp_get_nsegs(struct cryptop *, unsigned int *); 81233541Sjchandrastatic int xlp_alloc_cmd_params(struct xlp_sec_command *, unsigned int); 82233541Sjchandrastatic void xlp_free_cmd_params(struct xlp_sec_command *); 83233541Sjchandra 84233541Sjchandrastatic int xlp_sec_probe(device_t); 85233541Sjchandrastatic int xlp_sec_attach(device_t); 86233541Sjchandrastatic int xlp_sec_detach(device_t); 87233541Sjchandra 88233541Sjchandrastatic device_method_t xlp_sec_methods[] = { 89233541Sjchandra /* device interface */ 90233541Sjchandra DEVMETHOD(device_probe, xlp_sec_probe), 91233541Sjchandra DEVMETHOD(device_attach, xlp_sec_attach), 92233541Sjchandra DEVMETHOD(device_detach, xlp_sec_detach), 93233541Sjchandra 94233541Sjchandra /* bus interface */ 95233541Sjchandra DEVMETHOD(bus_print_child, bus_generic_print_child), 96233541Sjchandra DEVMETHOD(bus_driver_added, bus_generic_driver_added), 97233541Sjchandra 98233541Sjchandra /* crypto device methods */ 99233541Sjchandra DEVMETHOD(cryptodev_newsession, xlp_sec_newsession), 100233541Sjchandra DEVMETHOD(cryptodev_freesession,xlp_sec_freesession), 101233541Sjchandra DEVMETHOD(cryptodev_process, xlp_sec_process), 102233541Sjchandra 103233541Sjchandra DEVMETHOD_END 104233541Sjchandra}; 105233541Sjchandra 106233541Sjchandrastatic driver_t xlp_sec_driver = { 107233541Sjchandra "nlmsec", 108233541Sjchandra xlp_sec_methods, 109233541Sjchandra sizeof(struct xlp_sec_softc) 110233541Sjchandra}; 111233541Sjchandrastatic devclass_t xlp_sec_devclass; 112233541Sjchandra 113233541SjchandraDRIVER_MODULE(nlmsec, pci, xlp_sec_driver, xlp_sec_devclass, 0, 0); 114233541SjchandraMODULE_DEPEND(nlmsec, crypto, 1, 1, 1); 115233541Sjchandra 116233541Sjchandravoid 117233541Sjchandranlm_xlpsec_msgring_handler(int vc, int size, int code, int src_id, 118233541Sjchandra struct nlm_fmn_msg *msg, void *data); 119233541Sjchandra 120233541Sjchandra#ifdef NLM_SEC_DEBUG 121233541Sjchandra 122233541Sjchandra#define extract_bits(x, bitshift, bitcnt) \ 123233541Sjchandra (((unsigned long long)x >> bitshift) & ((1ULL << bitcnt) - 1)) 124233541Sjchandra 125233541Sjchandravoid 126233541Sjchandraprint_crypto_params(struct xlp_sec_command *cmd, struct nlm_fmn_msg m) 127233541Sjchandra{ 128233541Sjchandra unsigned long long msg0,msg1,msg2,msg3,msg4,msg5,msg6,msg7,msg8; 129233541Sjchandra 130233541Sjchandra msg0 = cmd->ctrlp->desc0; 131233541Sjchandra msg1 = cmd->paramp->desc0; 132233541Sjchandra msg2 = cmd->paramp->desc1; 133233541Sjchandra msg3 = cmd->paramp->desc2; 134233541Sjchandra msg4 = cmd->paramp->desc3; 135233541Sjchandra msg5 = cmd->paramp->segment[0][0]; 136233541Sjchandra msg6 = cmd->paramp->segment[0][1]; 137233541Sjchandra msg7 = m.msg[0]; 138233541Sjchandra msg8 = m.msg[1]; 139233541Sjchandra 140233541Sjchandra printf("msg0 %llx msg1 %llx msg2 %llx msg3 %llx msg4 %llx msg5 %llx" 141233541Sjchandra "msg6 %llx msg7 %llx msg8 %llx\n", msg0, msg1, msg2, msg3, msg4, 142233541Sjchandra msg5, msg6, msg7, msg8); 143233541Sjchandra 144233541Sjchandra printf("c0: hmac %d htype %d hmode %d ctype %d cmode %d arc4 %x\n", 145233541Sjchandra (unsigned int)extract_bits(msg0, 61, 1), 146233541Sjchandra (unsigned int)extract_bits(msg0, 52, 8), 147233541Sjchandra (unsigned int)extract_bits(msg0, 43, 8), 148233541Sjchandra (unsigned int)extract_bits(msg0, 34, 8), 149233541Sjchandra (unsigned int)extract_bits(msg0, 25, 8), 150233541Sjchandra (unsigned int)extract_bits(msg0, 0, 23)); 151233541Sjchandra 152233541Sjchandra printf("p0: tls %d hsrc %d hl3 %d enc %d ivl %d hd %llx\n", 153233541Sjchandra (unsigned int)extract_bits(msg1, 63, 1), 154233541Sjchandra (unsigned int)extract_bits(msg1,62,1), 155233541Sjchandra (unsigned int)extract_bits(msg1,60,1), 156233541Sjchandra (unsigned int)extract_bits(msg1,59,1), 157233541Sjchandra (unsigned int)extract_bits(msg1,41,16), extract_bits(msg1,0,40)); 158233541Sjchandra 159233541Sjchandra printf("p1: clen %u hl %u\n", (unsigned int)extract_bits(msg2, 32, 32), 160233541Sjchandra (unsigned int)extract_bits(msg2,0,32)); 161233541Sjchandra 162233541Sjchandra printf("p2: ivoff %d cbit %d coff %d hbit %d hclb %d hoff %d\n", 163233541Sjchandra (unsigned int)extract_bits(msg3, 45, 17), 164233541Sjchandra (unsigned int)extract_bits(msg3, 42,3), 165233541Sjchandra (unsigned int)extract_bits(msg3, 22,16), 166233541Sjchandra (unsigned int)extract_bits(msg3, 19,3), 167233541Sjchandra (unsigned int)extract_bits(msg3, 18,1), 168233541Sjchandra (unsigned int)extract_bits(msg3, 0, 16)); 169233541Sjchandra 170233541Sjchandra printf("p3: desfbid %d tlen %d arc4 %x hmacpad %d\n", 171233541Sjchandra (unsigned int)extract_bits(msg4, 48,16), 172233541Sjchandra (unsigned int)extract_bits(msg4,11,16), 173233541Sjchandra (unsigned int)extract_bits(msg4,6,3), 174233541Sjchandra (unsigned int)extract_bits(msg4,5,1)); 175233541Sjchandra 176233541Sjchandra printf("p4: sflen %d sddr %llx \n", 177233541Sjchandra (unsigned int)extract_bits(msg5, 48, 16),extract_bits(msg5, 0, 40)); 178233541Sjchandra 179233541Sjchandra printf("p5: dflen %d cl3 %d cclob %d cdest %llx \n", 180233541Sjchandra (unsigned int)extract_bits(msg6, 48, 16), 181233541Sjchandra (unsigned int)extract_bits(msg6, 46, 1), 182233541Sjchandra (unsigned int)extract_bits(msg6, 41, 1), extract_bits(msg6, 0, 40)); 183233541Sjchandra 184233541Sjchandra printf("fmn0: fbid %d dfrlen %d dfrv %d cklen %d cdescaddr %llx\n", 185233541Sjchandra (unsigned int)extract_bits(msg7, 48, 16), 186233541Sjchandra (unsigned int)extract_bits(msg7,46,2), 187233541Sjchandra (unsigned int)extract_bits(msg7,45,1), 188233541Sjchandra (unsigned int)extract_bits(msg7,40,5), 189233541Sjchandra (extract_bits(msg7,0,34)<< 6)); 190233541Sjchandra 191233541Sjchandra printf("fmn1: arc4 %d hklen %d pdesclen %d pktdescad %llx\n", 192233541Sjchandra (unsigned int)extract_bits(msg8, 63, 1), 193233541Sjchandra (unsigned int)extract_bits(msg8,56,5), 194233541Sjchandra (unsigned int)extract_bits(msg8,43,12), 195233541Sjchandra (extract_bits(msg8,0,34) << 6)); 196233541Sjchandra 197233541Sjchandra return; 198233541Sjchandra} 199233541Sjchandra 200233541Sjchandravoid 201233541Sjchandraxlp_sec_print_data(struct cryptop *crp) 202233541Sjchandra{ 203233541Sjchandra int i, key_len; 204233541Sjchandra struct cryptodesc *crp_desc; 205233541Sjchandra 206233541Sjchandra printf("session id = 0x%llx, crp_ilen = %d, crp_olen=%d \n", 207233541Sjchandra crp->crp_sid, crp->crp_ilen, crp->crp_olen); 208233541Sjchandra 209233541Sjchandra printf("crp_flags = 0x%x\n", crp->crp_flags); 210233541Sjchandra 211233541Sjchandra printf("crp buf:\n"); 212233541Sjchandra for (i = 0; i < crp->crp_ilen; i++) { 213233541Sjchandra printf("%c ", crp->crp_buf[i]); 214233541Sjchandra if (i % 10 == 0) 215233541Sjchandra printf("\n"); 216233541Sjchandra } 217233541Sjchandra 218233541Sjchandra printf("\n"); 219233541Sjchandra printf("****************** desc ****************\n"); 220233541Sjchandra crp_desc = crp->crp_desc; 221233541Sjchandra printf("crd_skip=%d, crd_len=%d, crd_flags=0x%x, crd_alg=%d\n", 222233541Sjchandra crp_desc->crd_skip, crp_desc->crd_len, crp_desc->crd_flags, 223233541Sjchandra crp_desc->crd_alg); 224233541Sjchandra 225233541Sjchandra key_len = crp_desc->crd_klen / 8; 226233541Sjchandra printf("key(%d) :\n", key_len); 227233541Sjchandra for (i = 0; i < key_len; i++) 228233541Sjchandra printf("%d", crp_desc->crd_key[i]); 229233541Sjchandra printf("\n"); 230233541Sjchandra 231233541Sjchandra printf(" IV : \n"); 232233541Sjchandra for (i = 0; i < EALG_MAX_BLOCK_LEN; i++) 233233541Sjchandra printf("%d", crp_desc->crd_iv[i]); 234233541Sjchandra printf("\n"); 235233541Sjchandra 236233541Sjchandra printf("crd_next=%p\n", crp_desc->crd_next); 237233541Sjchandra return; 238233541Sjchandra} 239233541Sjchandra 240233541Sjchandravoid 241233541Sjchandraprint_cmd(struct xlp_sec_command *cmd) 242233541Sjchandra{ 243233541Sjchandra printf("session_num :%d\n",cmd->session_num); 244233541Sjchandra printf("crp :0x%x\n",(uint32_t)cmd->crp); 245233541Sjchandra printf("enccrd :0x%x\n",(uint32_t)cmd->enccrd); 246233541Sjchandra printf("maccrd :0x%x\n",(uint32_t)cmd->maccrd); 247233541Sjchandra printf("ses :%d\n",(uint32_t)cmd->ses); 248233541Sjchandra printf("ctrlp :0x%x\n",(uint32_t)cmd->ctrlp); 249233541Sjchandra printf("paramp :0x%x\n",(uint32_t)cmd->paramp); 250233541Sjchandra printf("hashdest :0x%x\n",(uint32_t)cmd->hashdest); 251233541Sjchandra printf("hashsrc :%d\n",cmd->hashsrc); 252233541Sjchandra printf("hmacpad :%d\n",cmd->hmacpad); 253233541Sjchandra printf("hashoff :%d\n",cmd->hashoff); 254233541Sjchandra printf("hashlen :%d\n",cmd->hashlen); 255233541Sjchandra printf("cipheroff :%d\n",cmd->cipheroff); 256233541Sjchandra printf("cipherlen :%d\n",cmd->cipherlen); 257233541Sjchandra printf("ivoff :%d\n",cmd->ivoff); 258233541Sjchandra printf("ivlen :%d\n",cmd->ivlen); 259233541Sjchandra printf("hashalg :%d\n",cmd->hashalg); 260233541Sjchandra printf("hashmode :%d\n",cmd->hashmode); 261233541Sjchandra printf("cipheralg :%d\n",cmd->cipheralg); 262233541Sjchandra printf("ciphermode :%d\n",cmd->ciphermode); 263233541Sjchandra printf("nsegs :%d\n",cmd->nsegs); 264233541Sjchandra printf("hash_dst_len :%d\n",cmd->hash_dst_len); 265233541Sjchandra} 266233541Sjchandra#endif /* NLM_SEC_DEBUG */ 267233541Sjchandra 268233541Sjchandrastatic int 269233541Sjchandraxlp_sec_init(struct xlp_sec_softc *sc) 270233541Sjchandra{ 271233541Sjchandra 272233541Sjchandra /* Register interrupt handler for the SEC CMS messages */ 273233541Sjchandra if (register_msgring_handler(sc->sec_vc_start, 274233541Sjchandra sc->sec_vc_end, nlm_xlpsec_msgring_handler, sc) != 0) { 275233541Sjchandra printf("Couldn't register sec msgring handler\n"); 276233541Sjchandra return (-1); 277233541Sjchandra } 278233541Sjchandra 279233541Sjchandra /* Do the CMS credit initialization */ 280233541Sjchandra /* Currently it is configured by default to 50 when kernel comes up */ 281233541Sjchandra 282233541Sjchandra return (0); 283233541Sjchandra} 284233541Sjchandra 285233541Sjchandra/* This function is called from an interrupt handler */ 286233541Sjchandravoid 287233541Sjchandranlm_xlpsec_msgring_handler(int vc, int size, int code, int src_id, 288233541Sjchandra struct nlm_fmn_msg *msg, void *data) 289233541Sjchandra{ 290233541Sjchandra struct xlp_sec_command *cmd = NULL; 291233541Sjchandra struct xlp_sec_softc *sc = NULL; 292233541Sjchandra struct cryptodesc *crd = NULL; 293233541Sjchandra unsigned int ivlen = 0; 294233541Sjchandra 295233541Sjchandra KASSERT(code == FMN_SWCODE_CRYPTO, 296233541Sjchandra ("%s: bad code = %d, expected code = %d\n", __FUNCTION__, 297233541Sjchandra code, FMN_SWCODE_CRYPTO)); 298233541Sjchandra 299233541Sjchandra sc = (struct xlp_sec_softc *)data; 300233541Sjchandra KASSERT(src_id >= sc->sec_vc_start && src_id <= sc->sec_vc_end, 301233541Sjchandra ("%s: bad src_id = %d, expect %d - %d\n", __FUNCTION__, 302233541Sjchandra src_id, sc->sec_vc_start, sc->sec_vc_end)); 303233541Sjchandra 304233541Sjchandra cmd = (struct xlp_sec_command *)(uintptr_t)msg->msg[0]; 305233541Sjchandra KASSERT(cmd != NULL && cmd->crp != NULL, 306233541Sjchandra ("%s :cmd not received properly\n",__FUNCTION__)); 307233541Sjchandra 308233541Sjchandra KASSERT(CRYPTO_ERROR(msg->msg[1]) == 0, 309233541Sjchandra ("%s: Message rcv msg0 %llx msg1 %llx err %x \n", __FUNCTION__, 310233541Sjchandra (unsigned long long)msg->msg[0], (unsigned long long)msg->msg[1], 311233541Sjchandra (int)CRYPTO_ERROR(msg->msg[1]))); 312233541Sjchandra 313233541Sjchandra crd = cmd->enccrd; 314233541Sjchandra /* Copy the last 8 or 16 bytes to the session iv, so that in few 315233541Sjchandra * cases this will be used as IV for the next request 316233541Sjchandra */ 317233541Sjchandra if (crd != NULL) { 318233541Sjchandra if ((crd->crd_alg == CRYPTO_DES_CBC || 319233541Sjchandra crd->crd_alg == CRYPTO_3DES_CBC || 320233541Sjchandra crd->crd_alg == CRYPTO_AES_CBC) && 321233541Sjchandra (crd->crd_flags & CRD_F_ENCRYPT)) { 322233541Sjchandra ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ? 323233541Sjchandra XLP_SEC_AES_IV_LENGTH : XLP_SEC_DES_IV_LENGTH); 324233541Sjchandra crypto_copydata(cmd->crp->crp_flags, cmd->crp->crp_buf, 325233541Sjchandra crd->crd_skip + crd->crd_len - ivlen, ivlen, 326233541Sjchandra sc->sc_sessions[cmd->session_num].ses_iv); 327233541Sjchandra } 328233541Sjchandra } 329233541Sjchandra 330233541Sjchandra /* If there are not enough credits to send, then send request 331233541Sjchandra * will fail with ERESTART and the driver will be blocked until it is 332233541Sjchandra * unblocked here after knowing that there are sufficient credits to 333233541Sjchandra * send the request again. 334233541Sjchandra */ 335233541Sjchandra if (sc->sc_needwakeup) { 336233541Sjchandra atomic_add_int(&creditleft, sc->sec_msgsz); 337233541Sjchandra if (creditleft >= (NLM_CRYPTO_LEFT_REQS)) { 338233541Sjchandra crypto_unblock(sc->sc_cid, sc->sc_needwakeup); 339233541Sjchandra sc->sc_needwakeup &= (~(CRYPTO_SYMQ | CRYPTO_ASYMQ)); 340233541Sjchandra } 341233541Sjchandra } 342233541Sjchandra if(cmd->maccrd) { 343233541Sjchandra crypto_copyback(cmd->crp->crp_flags, 344233541Sjchandra cmd->crp->crp_buf, cmd->maccrd->crd_inject, 345233541Sjchandra cmd->hash_dst_len, cmd->hashdest); 346233541Sjchandra } 347233541Sjchandra 348233541Sjchandra /* This indicates completion of the crypto operation */ 349233541Sjchandra crypto_done(cmd->crp); 350233541Sjchandra 351233541Sjchandra xlp_free_cmd_params(cmd); 352233541Sjchandra 353233541Sjchandra return; 354233541Sjchandra} 355233541Sjchandra 356233541Sjchandrastatic int 357233541Sjchandraxlp_sec_probe(device_t dev) 358233541Sjchandra{ 359233541Sjchandra struct xlp_sec_softc *sc; 360233541Sjchandra 361233541Sjchandra if (pci_get_vendor(dev) == PCI_VENDOR_NETLOGIC && 362233541Sjchandra pci_get_device(dev) == PCI_DEVICE_ID_NLM_SAE) { 363233541Sjchandra sc = device_get_softc(dev); 364233541Sjchandra return (BUS_PROBE_DEFAULT); 365233541Sjchandra } 366233541Sjchandra return (ENXIO); 367233541Sjchandra} 368233541Sjchandra 369233541Sjchandra/* 370233541Sjchandra * Attach an interface that successfully probed. 371233541Sjchandra */ 372233541Sjchandrastatic int 373233541Sjchandraxlp_sec_attach(device_t dev) 374233541Sjchandra{ 375233541Sjchandra struct xlp_sec_softc *sc = device_get_softc(dev); 376233541Sjchandra uint64_t base; 377233541Sjchandra int qstart, qnum; 378233541Sjchandra int freq, node; 379233541Sjchandra 380233541Sjchandra sc->sc_dev = dev; 381233541Sjchandra 382233541Sjchandra node = nlm_get_device_node(pci_get_slot(dev)); 383233541Sjchandra freq = nlm_set_device_frequency(node, DFS_DEVICE_SAE, 250); 384233541Sjchandra if (bootverbose) 385233541Sjchandra device_printf(dev, "SAE Freq: %dMHz\n", freq); 386233541Sjchandra if(pci_get_device(dev) == PCI_DEVICE_ID_NLM_SAE) { 387233541Sjchandra device_set_desc(dev, "XLP Security Accelerator"); 388233541Sjchandra sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); 389233541Sjchandra if (sc->sc_cid < 0) { 390233541Sjchandra printf("xlp_sec - error : could not get the driver" 391233541Sjchandra " id\n"); 392233541Sjchandra goto error_exit; 393233541Sjchandra } 394233541Sjchandra if (crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0) != 0) 395233541Sjchandra printf("register failed for CRYPTO_DES_CBC\n"); 396233541Sjchandra 397233541Sjchandra if (crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0) != 0) 398233541Sjchandra printf("register failed for CRYPTO_3DES_CBC\n"); 399233541Sjchandra 400233541Sjchandra if (crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0) != 0) 401233541Sjchandra printf("register failed for CRYPTO_AES_CBC\n"); 402233541Sjchandra 403233541Sjchandra if (crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0) != 0) 404233541Sjchandra printf("register failed for CRYPTO_ARC4\n"); 405233541Sjchandra 406233541Sjchandra if (crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0) != 0) 407233541Sjchandra printf("register failed for CRYPTO_MD5\n"); 408233541Sjchandra 409233541Sjchandra if (crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0) != 0) 410233541Sjchandra printf("register failed for CRYPTO_SHA1\n"); 411233541Sjchandra 412233541Sjchandra if (crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0) != 0) 413233541Sjchandra printf("register failed for CRYPTO_MD5_HMAC\n"); 414233541Sjchandra 415233541Sjchandra if (crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0) != 0) 416233541Sjchandra printf("register failed for CRYPTO_SHA1_HMAC\n"); 417233541Sjchandra 418233541Sjchandra base = nlm_get_sec_pcibase(node); 419233541Sjchandra qstart = nlm_qidstart(base); 420233541Sjchandra qnum = nlm_qnum(base); 421233541Sjchandra sc->sec_vc_start = qstart; 422233541Sjchandra sc->sec_vc_end = qstart + qnum - 1; 423233541Sjchandra } 424233541Sjchandra 425233541Sjchandra if (xlp_sec_init(sc) != 0) 426233541Sjchandra goto error_exit; 427233541Sjchandra if (bootverbose) 428233541Sjchandra device_printf(dev, "SEC Initialization complete!\n"); 429233541Sjchandra return (0); 430233541Sjchandra 431233541Sjchandraerror_exit: 432233541Sjchandra return (ENXIO); 433233541Sjchandra 434233541Sjchandra} 435233541Sjchandra 436233541Sjchandra/* 437233541Sjchandra * Detach an interface that successfully probed. 438233541Sjchandra */ 439233541Sjchandrastatic int 440233541Sjchandraxlp_sec_detach(device_t dev) 441233541Sjchandra{ 442233541Sjchandra return (0); 443233541Sjchandra} 444233541Sjchandra 445233541Sjchandra/* 446233541Sjchandra * Allocate a new 'session' and return an encoded session id. 'sidp' 447233541Sjchandra * contains our registration id, and should contain an encoded session 448233541Sjchandra * id on successful allocation. 449233541Sjchandra */ 450233541Sjchandrastatic int 451233541Sjchandraxlp_sec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri) 452233541Sjchandra{ 453233541Sjchandra struct cryptoini *c; 454233541Sjchandra struct xlp_sec_softc *sc = device_get_softc(dev); 455233541Sjchandra int mac = 0, cry = 0, sesn; 456233541Sjchandra struct xlp_sec_session *ses = NULL; 457233541Sjchandra struct xlp_sec_command *cmd = NULL; 458233541Sjchandra 459233541Sjchandra if (sidp == NULL || cri == NULL || sc == NULL) 460233541Sjchandra return (EINVAL); 461233541Sjchandra 462233541Sjchandra if (sc->sc_sessions == NULL) { 463233541Sjchandra ses = sc->sc_sessions = malloc(sizeof(struct xlp_sec_session), 464233541Sjchandra M_DEVBUF, M_NOWAIT); 465233541Sjchandra if (ses == NULL) 466233541Sjchandra return (ENOMEM); 467233541Sjchandra sesn = 0; 468233541Sjchandra sc->sc_nsessions = 1; 469233541Sjchandra } else { 470233541Sjchandra for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 471233541Sjchandra if (!sc->sc_sessions[sesn].hs_used) { 472233541Sjchandra ses = &sc->sc_sessions[sesn]; 473233541Sjchandra break; 474233541Sjchandra } 475233541Sjchandra } 476233541Sjchandra 477233541Sjchandra if (ses == NULL) { 478233541Sjchandra sesn = sc->sc_nsessions; 479233541Sjchandra ses = malloc((sesn + 1)*sizeof(struct xlp_sec_session), 480233541Sjchandra M_DEVBUF, M_NOWAIT); 481233541Sjchandra if (ses == NULL) 482233541Sjchandra return (ENOMEM); 483233541Sjchandra bcopy(sc->sc_sessions, ses, sesn * sizeof(*ses)); 484233541Sjchandra bzero(sc->sc_sessions, sesn * sizeof(*ses)); 485233541Sjchandra free(sc->sc_sessions, M_DEVBUF); 486233541Sjchandra sc->sc_sessions = ses; 487233541Sjchandra ses = &sc->sc_sessions[sesn]; 488233541Sjchandra sc->sc_nsessions++; 489233541Sjchandra } 490233541Sjchandra } 491233541Sjchandra bzero(ses, sizeof(*ses)); 492233541Sjchandra ses->sessionid = sesn; 493233541Sjchandra cmd = &ses->cmd; 494233541Sjchandra ses->hs_used = 1; 495233541Sjchandra 496233541Sjchandra for (c = cri; c != NULL; c = c->cri_next) { 497233541Sjchandra switch (c->cri_alg) { 498233541Sjchandra case CRYPTO_MD5: 499233541Sjchandra case CRYPTO_SHA1: 500233541Sjchandra case CRYPTO_MD5_HMAC: 501233541Sjchandra case CRYPTO_SHA1_HMAC: 502233541Sjchandra if (mac) 503233541Sjchandra return (EINVAL); 504233541Sjchandra mac = 1; 505233541Sjchandra ses->hs_mlen = c->cri_mlen; 506233541Sjchandra if (ses->hs_mlen == 0) { 507233541Sjchandra switch (c->cri_alg) { 508233541Sjchandra case CRYPTO_MD5: 509233541Sjchandra case CRYPTO_MD5_HMAC: 510233541Sjchandra ses->hs_mlen = 16; 511233541Sjchandra break; 512233541Sjchandra case CRYPTO_SHA1: 513233541Sjchandra case CRYPTO_SHA1_HMAC: 514233541Sjchandra ses->hs_mlen = 20; 515233541Sjchandra break; 516233541Sjchandra } 517233541Sjchandra } 518233541Sjchandra break; 519233541Sjchandra case CRYPTO_DES_CBC: 520233541Sjchandra case CRYPTO_3DES_CBC: 521233541Sjchandra case CRYPTO_AES_CBC: 522233541Sjchandra /* XXX this may read fewer, does it matter? */ 523233541Sjchandra read_random(ses->ses_iv, c->cri_alg == 524233541Sjchandra CRYPTO_AES_CBC ? XLP_SEC_AES_IV_LENGTH : 525233541Sjchandra XLP_SEC_DES_IV_LENGTH); 526233541Sjchandra /* FALLTHROUGH */ 527233541Sjchandra case CRYPTO_ARC4: 528233541Sjchandra if (cry) 529233541Sjchandra return (EINVAL); 530233541Sjchandra cry = 1; 531233541Sjchandra break; 532233541Sjchandra default: 533233541Sjchandra return (EINVAL); 534233541Sjchandra } 535233541Sjchandra } 536233541Sjchandra if (mac == 0 && cry == 0) 537233541Sjchandra return (EINVAL); 538233541Sjchandra 539233541Sjchandra cmd->hash_dst_len = ses->hs_mlen; 540233541Sjchandra *sidp = XLP_SEC_SID(device_get_unit(sc->sc_dev), sesn); 541233541Sjchandra return (0); 542233541Sjchandra} 543233541Sjchandra 544233541Sjchandra/* 545233541Sjchandra * Deallocate a session. 546233541Sjchandra * XXX this routine should run a zero'd mac/encrypt key into context ram. 547233541Sjchandra * XXX to blow away any keys already stored there. 548233541Sjchandra */ 549233541Sjchandrastatic int 550233541Sjchandraxlp_sec_freesession(device_t dev, u_int64_t tid) 551233541Sjchandra{ 552233541Sjchandra struct xlp_sec_softc *sc = device_get_softc(dev); 553233541Sjchandra int session; 554233541Sjchandra u_int32_t sid = CRYPTO_SESID2LID(tid); 555233541Sjchandra 556233541Sjchandra if (sc == NULL) 557233541Sjchandra return (EINVAL); 558233541Sjchandra 559233541Sjchandra session = XLP_SEC_SESSION(sid); 560233541Sjchandra if (session >= sc->sc_nsessions) 561233541Sjchandra return (EINVAL); 562233541Sjchandra 563233541Sjchandra sc->sc_sessions[session].hs_used = 0; 564233541Sjchandra return (0); 565233541Sjchandra} 566233541Sjchandra 567233541Sjchandrastatic int 568233541Sjchandraxlp_copyiv(struct xlp_sec_softc *sc, struct xlp_sec_command *cmd, 569233541Sjchandra struct cryptodesc *enccrd) 570233541Sjchandra{ 571233541Sjchandra unsigned int ivlen = 0; 572233541Sjchandra int session; 573233541Sjchandra struct cryptop *crp = NULL; 574233541Sjchandra 575233541Sjchandra crp = cmd->crp; 576233541Sjchandra session = cmd->session_num; 577233541Sjchandra 578233541Sjchandra if (enccrd->crd_alg != CRYPTO_ARC4) { 579233541Sjchandra ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ? 580233541Sjchandra XLP_SEC_AES_IV_LENGTH : XLP_SEC_DES_IV_LENGTH); 581233541Sjchandra if (enccrd->crd_flags & CRD_F_ENCRYPT) { 582233541Sjchandra if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) { 583233541Sjchandra bcopy(enccrd->crd_iv, cmd->iv, ivlen); 584233541Sjchandra } else { 585233541Sjchandra bcopy(sc->sc_sessions[session].ses_iv, cmd->iv, 586233541Sjchandra ivlen); 587233541Sjchandra } 588233541Sjchandra if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { 589233541Sjchandra crypto_copyback(crp->crp_flags, 590233541Sjchandra crp->crp_buf, enccrd->crd_inject, 591233541Sjchandra ivlen, cmd->iv); 592233541Sjchandra } 593233541Sjchandra } else { 594233541Sjchandra if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) { 595233541Sjchandra bcopy(enccrd->crd_iv, cmd->iv, ivlen); 596233541Sjchandra } else { 597233541Sjchandra crypto_copydata(crp->crp_flags, crp->crp_buf, 598233541Sjchandra enccrd->crd_inject, ivlen, cmd->iv); 599233541Sjchandra } 600233541Sjchandra } 601233541Sjchandra } 602233541Sjchandra return (0); 603233541Sjchandra} 604233541Sjchandra 605233541Sjchandrastatic int 606233541Sjchandraxlp_get_nsegs(struct cryptop *crp, unsigned int *nsegs) 607233541Sjchandra{ 608233541Sjchandra 609233541Sjchandra if (crp->crp_flags & CRYPTO_F_IMBUF) { 610233541Sjchandra struct mbuf *m = NULL; 611233541Sjchandra 612233541Sjchandra m = (struct mbuf *)crp->crp_buf; 613233541Sjchandra while (m != NULL) { 614233541Sjchandra *nsegs += NLM_CRYPTO_NUM_SEGS_REQD(m->m_len); 615233541Sjchandra m = m->m_next; 616233541Sjchandra } 617233541Sjchandra } else if (crp->crp_flags & CRYPTO_F_IOV) { 618233541Sjchandra struct uio *uio = NULL; 619233541Sjchandra struct iovec *iov = NULL; 620233541Sjchandra int iol = 0; 621233541Sjchandra 622233541Sjchandra uio = (struct uio *)crp->crp_buf; 623233541Sjchandra iov = (struct iovec *)uio->uio_iov; 624233541Sjchandra iol = uio->uio_iovcnt; 625233541Sjchandra while (iol > 0) { 626233541Sjchandra *nsegs += NLM_CRYPTO_NUM_SEGS_REQD(iov->iov_len); 627233541Sjchandra iol--; 628233541Sjchandra iov++; 629233541Sjchandra } 630233541Sjchandra } else { 631233541Sjchandra *nsegs = NLM_CRYPTO_NUM_SEGS_REQD(crp->crp_ilen); 632233541Sjchandra } 633233541Sjchandra return (0); 634233541Sjchandra} 635233541Sjchandra 636233541Sjchandrastatic int 637233541Sjchandraxlp_alloc_cmd_params(struct xlp_sec_command *cmd, unsigned int nsegs) 638233541Sjchandra{ 639233541Sjchandra int err = 0; 640233541Sjchandra 641233541Sjchandra if(cmd == NULL) { 642233541Sjchandra err = EINVAL; 643233541Sjchandra goto error; 644233541Sjchandra } 645233541Sjchandra if ((cmd->ctrlp = malloc(sizeof(struct nlm_crypto_pkt_ctrl), M_DEVBUF, 646233541Sjchandra M_NOWAIT | M_ZERO)) == NULL) { 647233541Sjchandra err = ENOMEM; 648233541Sjchandra goto error; 649233541Sjchandra } 650233541Sjchandra if (((uintptr_t)cmd->ctrlp & (XLP_L2L3_CACHELINE_SIZE - 1))) { 651233541Sjchandra err = EINVAL; 652233541Sjchandra goto error; 653233541Sjchandra } 654233541Sjchandra /* (nsegs - 1) because one seg is part of the structure already */ 655233541Sjchandra if ((cmd->paramp = malloc(sizeof(struct nlm_crypto_pkt_param) + 656233541Sjchandra (16 * (nsegs - 1)), M_DEVBUF, M_NOWAIT | M_ZERO)) == NULL) { 657233541Sjchandra err = ENOMEM; 658233541Sjchandra goto error; 659233541Sjchandra } 660233541Sjchandra if (((uintptr_t)cmd->paramp & (XLP_L2L3_CACHELINE_SIZE - 1))) { 661233541Sjchandra err = EINVAL; 662233541Sjchandra goto error; 663233541Sjchandra } 664233541Sjchandra if ((cmd->iv = malloc(EALG_MAX_BLOCK_LEN, M_DEVBUF, 665233541Sjchandra M_NOWAIT | M_ZERO)) == NULL) { 666233541Sjchandra err = ENOMEM; 667233541Sjchandra goto error; 668233541Sjchandra } 669233541Sjchandra if ((cmd->hashdest = malloc(HASH_MAX_LEN, M_DEVBUF, 670233541Sjchandra M_NOWAIT | M_ZERO)) == NULL) { 671233541Sjchandra err = ENOMEM; 672233541Sjchandra goto error; 673233541Sjchandra } 674233541Sjchandraerror: 675233541Sjchandra return (err); 676233541Sjchandra} 677233541Sjchandra 678233541Sjchandrastatic void 679233541Sjchandraxlp_free_cmd_params(struct xlp_sec_command *cmd) 680233541Sjchandra{ 681233541Sjchandra if (cmd->ctrlp != NULL) 682233541Sjchandra free(cmd->ctrlp, M_DEVBUF); 683233541Sjchandra if (cmd->paramp != NULL) 684233541Sjchandra free(cmd->paramp, M_DEVBUF); 685233541Sjchandra if (cmd->iv != NULL) 686233541Sjchandra free(cmd->iv, M_DEVBUF); 687233541Sjchandra if (cmd->hashdest != NULL) 688233541Sjchandra free(cmd->hashdest, M_DEVBUF); 689233541Sjchandra if (cmd != NULL) 690233541Sjchandra free(cmd, M_DEVBUF); 691233541Sjchandra return; 692233541Sjchandra} 693233541Sjchandra 694233541Sjchandrastatic int 695233541Sjchandraxlp_sec_process(device_t dev, struct cryptop *crp, int hint) 696233541Sjchandra{ 697233541Sjchandra struct xlp_sec_softc *sc = device_get_softc(dev); 698233541Sjchandra struct xlp_sec_command *cmd = NULL; 699233541Sjchandra int session, err = -1, ret = 0; 700233541Sjchandra struct cryptodesc *crd1, *crd2; 701233541Sjchandra struct xlp_sec_session *ses; 702233541Sjchandra unsigned int nsegs = 0; 703233541Sjchandra 704233541Sjchandra if (crp == NULL || crp->crp_callback == NULL) { 705233541Sjchandra return (EINVAL); 706233541Sjchandra } 707233541Sjchandra session = XLP_SEC_SESSION(crp->crp_sid); 708233541Sjchandra if (sc == NULL || session >= sc->sc_nsessions) { 709233541Sjchandra err = EINVAL; 710233541Sjchandra goto errout; 711233541Sjchandra } 712233541Sjchandra ses = &sc->sc_sessions[session]; 713233541Sjchandra 714233541Sjchandra if ((cmd = malloc(sizeof(struct xlp_sec_command), M_DEVBUF, 715233541Sjchandra M_NOWAIT | M_ZERO)) == NULL) { 716233541Sjchandra err = ENOMEM; 717233541Sjchandra goto errout; 718233541Sjchandra } 719233541Sjchandra 720233541Sjchandra cmd->crp = crp; 721233541Sjchandra cmd->session_num = session; 722233541Sjchandra cmd->hash_dst_len = ses->hs_mlen; 723233541Sjchandra 724233541Sjchandra if ((crd1 = crp->crp_desc) == NULL) { 725233541Sjchandra err = EINVAL; 726233541Sjchandra goto errout; 727233541Sjchandra } 728233541Sjchandra crd2 = crd1->crd_next; 729233541Sjchandra 730233541Sjchandra if ((ret = xlp_get_nsegs(crp, &nsegs)) != 0) { 731233541Sjchandra err = EINVAL; 732233541Sjchandra goto errout; 733233541Sjchandra } 734233541Sjchandra if (((crd1 != NULL) && (crd1->crd_flags & CRD_F_IV_EXPLICIT)) || 735233541Sjchandra ((crd2 != NULL) && (crd2->crd_flags & CRD_F_IV_EXPLICIT))) { 736233541Sjchandra /* Since IV is given as separate segment to avoid copy */ 737233541Sjchandra nsegs += 1; 738233541Sjchandra } 739233541Sjchandra cmd->nsegs = nsegs; 740233541Sjchandra 741233541Sjchandra if ((err = xlp_alloc_cmd_params(cmd, nsegs)) != 0) 742233541Sjchandra goto errout; 743233541Sjchandra 744233541Sjchandra if ((crd1 != NULL) && (crd2 == NULL)) { 745233541Sjchandra if (crd1->crd_alg == CRYPTO_DES_CBC || 746233541Sjchandra crd1->crd_alg == CRYPTO_3DES_CBC || 747233541Sjchandra crd1->crd_alg == CRYPTO_AES_CBC || 748233541Sjchandra crd1->crd_alg == CRYPTO_ARC4) { 749233541Sjchandra cmd->enccrd = crd1; 750233541Sjchandra cmd->maccrd = NULL; 751233541Sjchandra if ((ret = nlm_get_cipher_param(cmd)) != 0) { 752233541Sjchandra err = EINVAL; 753233541Sjchandra goto errout; 754233541Sjchandra } 755233541Sjchandra if (crd1->crd_flags & CRD_F_IV_EXPLICIT) 756233541Sjchandra cmd->cipheroff = cmd->ivlen; 757233541Sjchandra else 758233541Sjchandra cmd->cipheroff = cmd->enccrd->crd_skip; 759233541Sjchandra cmd->cipherlen = cmd->enccrd->crd_len; 760233541Sjchandra if (crd1->crd_flags & CRD_F_IV_PRESENT) 761233541Sjchandra cmd->ivoff = 0; 762233541Sjchandra else 763233541Sjchandra cmd->ivoff = cmd->enccrd->crd_inject; 764233541Sjchandra if ((err = xlp_copyiv(sc, cmd, cmd->enccrd)) != 0) 765233541Sjchandra goto errout; 766233541Sjchandra if ((err = nlm_crypto_do_cipher(sc, cmd)) != 0) 767233541Sjchandra goto errout; 768233541Sjchandra } else if (crd1->crd_alg == CRYPTO_MD5_HMAC || 769233541Sjchandra crd1->crd_alg == CRYPTO_SHA1_HMAC || 770233541Sjchandra crd1->crd_alg == CRYPTO_SHA1 || 771233541Sjchandra crd1->crd_alg == CRYPTO_MD5) { 772233541Sjchandra cmd->enccrd = NULL; 773233541Sjchandra cmd->maccrd = crd1; 774233541Sjchandra if ((ret = nlm_get_digest_param(cmd)) != 0) { 775233541Sjchandra err = EINVAL; 776233541Sjchandra goto errout; 777233541Sjchandra } 778233541Sjchandra cmd->hashoff = cmd->maccrd->crd_skip; 779233541Sjchandra cmd->hashlen = cmd->maccrd->crd_len; 780233541Sjchandra cmd->hmacpad = 0; 781233541Sjchandra cmd->hashsrc = 0; 782233541Sjchandra if ((err = nlm_crypto_do_digest(sc, cmd)) != 0) 783233541Sjchandra goto errout; 784233541Sjchandra } else { 785233541Sjchandra err = EINVAL; 786233541Sjchandra goto errout; 787233541Sjchandra } 788233541Sjchandra } else if( (crd1 != NULL) && (crd2 != NULL) ) { 789233541Sjchandra if ((crd1->crd_alg == CRYPTO_MD5_HMAC || 790233541Sjchandra crd1->crd_alg == CRYPTO_SHA1_HMAC || 791233541Sjchandra crd1->crd_alg == CRYPTO_MD5 || 792233541Sjchandra crd1->crd_alg == CRYPTO_SHA1) && 793233541Sjchandra (crd2->crd_alg == CRYPTO_DES_CBC || 794233541Sjchandra crd2->crd_alg == CRYPTO_3DES_CBC || 795233541Sjchandra crd2->crd_alg == CRYPTO_AES_CBC || 796233541Sjchandra crd2->crd_alg == CRYPTO_ARC4)) { 797233541Sjchandra cmd->maccrd = crd1; 798233541Sjchandra cmd->enccrd = crd2; 799233541Sjchandra } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 800233541Sjchandra crd1->crd_alg == CRYPTO_ARC4 || 801233541Sjchandra crd1->crd_alg == CRYPTO_3DES_CBC || 802233541Sjchandra crd1->crd_alg == CRYPTO_AES_CBC) && 803233541Sjchandra (crd2->crd_alg == CRYPTO_MD5_HMAC || 804233541Sjchandra crd2->crd_alg == CRYPTO_SHA1_HMAC || 805233541Sjchandra crd2->crd_alg == CRYPTO_MD5 || 806233541Sjchandra crd2->crd_alg == CRYPTO_SHA1)) { 807233541Sjchandra cmd->enccrd = crd1; 808233541Sjchandra cmd->maccrd = crd2; 809233541Sjchandra } else { 810233541Sjchandra err = EINVAL; 811233541Sjchandra goto errout; 812233541Sjchandra } 813233541Sjchandra if ((ret = nlm_get_cipher_param(cmd)) != 0) { 814233541Sjchandra err = EINVAL; 815233541Sjchandra goto errout; 816233541Sjchandra } 817233541Sjchandra if ((ret = nlm_get_digest_param(cmd)) != 0) { 818233541Sjchandra err = EINVAL; 819233541Sjchandra goto errout; 820233541Sjchandra } 821233541Sjchandra cmd->ivoff = cmd->enccrd->crd_inject; 822233541Sjchandra cmd->hashoff = cmd->maccrd->crd_skip; 823233541Sjchandra cmd->hashlen = cmd->maccrd->crd_len; 824233541Sjchandra cmd->hmacpad = 0; 825233541Sjchandra if (cmd->enccrd->crd_flags & CRD_F_ENCRYPT) 826233541Sjchandra cmd->hashsrc = 1; 827233541Sjchandra else 828233541Sjchandra cmd->hashsrc = 0; 829233541Sjchandra cmd->cipheroff = cmd->enccrd->crd_skip; 830233541Sjchandra cmd->cipherlen = cmd->enccrd->crd_len; 831233541Sjchandra if ((err = xlp_copyiv(sc, cmd, cmd->enccrd)) != 0) 832233541Sjchandra goto errout; 833233541Sjchandra if ((err = nlm_crypto_do_cipher_digest(sc, cmd)) != 0) 834233541Sjchandra goto errout; 835233541Sjchandra } else { 836233541Sjchandra err = EINVAL; 837233541Sjchandra goto errout; 838233541Sjchandra } 839233541Sjchandra return (0); 840233541Sjchandraerrout: 841233541Sjchandra xlp_free_cmd_params(cmd); 842233541Sjchandra if (err == ERESTART) { 843233541Sjchandra sc->sc_needwakeup |= CRYPTO_SYMQ; 844233541Sjchandra creditleft = 0; 845233541Sjchandra return (err); 846233541Sjchandra } 847233541Sjchandra crp->crp_etype = err; 848233541Sjchandra crypto_done(crp); 849233541Sjchandra return (err); 850233541Sjchandra} 851