1/*- 2 * Copyright (c) 2017 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: John Baldwin <jhb@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD$"); 30 31#include <sys/types.h> 32#include <sys/bus.h> 33#include <sys/lock.h> 34#include <sys/malloc.h> 35#include <sys/mutex.h> 36#include <sys/module.h> 37#include <sys/sglist.h> 38 39#include <opencrypto/cryptodev.h> 40#include <opencrypto/xform.h> 41 42#include "cryptodev_if.h" 43 44#include "common/common.h" 45#include "crypto/t4_crypto.h" 46 47/* 48 * Requests consist of: 49 * 50 * +-------------------------------+ 51 * | struct fw_crypto_lookaside_wr | 52 * +-------------------------------+ 53 * | struct ulp_txpkt | 54 * +-------------------------------+ 55 * | struct ulptx_idata | 56 * +-------------------------------+ 57 * | struct cpl_tx_sec_pdu | 58 * +-------------------------------+ 59 * | struct cpl_tls_tx_scmd_fmt | 60 * +-------------------------------+ 61 * | key context header | 62 * +-------------------------------+ 63 * | AES key | ----- For requests with AES 64 * +-------------------------------+ 65 * | Hash state | ----- For hash-only requests 66 * +-------------------------------+ - 67 * | IPAD (16-byte aligned) | \ 68 * +-------------------------------+ +---- For requests with HMAC 69 * | OPAD (16-byte aligned) | / 70 * +-------------------------------+ - 71 * | GMAC H | ----- For AES-GCM 72 * +-------------------------------+ - 73 * | struct cpl_rx_phys_dsgl | \ 74 * +-------------------------------+ +---- Destination buffer for 75 * | PHYS_DSGL entries | / non-hash-only requests 76 * +-------------------------------+ - 77 * | 16 dummy bytes | ----- Only for HMAC/hash-only requests 78 * +-------------------------------+ 79 * | IV | ----- If immediate IV 80 * +-------------------------------+ 81 * | Payload | ----- If immediate Payload 82 * +-------------------------------+ - 83 * | struct ulptx_sgl | \ 84 * +-------------------------------+ +---- If payload via SGL 85 * | SGL entries | / 86 * +-------------------------------+ - 87 * 88 * Note that the key context must be padded to ensure 16-byte alignment. 89 * For HMAC requests, the key consists of the partial hash of the IPAD 90 * followed by the partial hash of the OPAD. 91 * 92 * Replies consist of: 93 * 94 * +-------------------------------+ 95 * | struct cpl_fw6_pld | 96 * +-------------------------------+ 97 * | hash digest | ----- For HMAC request with 98 * +-------------------------------+ 'hash_size' set in work request 99 * 100 * A 32-bit big-endian error status word is supplied in the last 4 101 * bytes of data[0] in the CPL_FW6_PLD message. bit 0 indicates a 102 * "MAC" error and bit 1 indicates a "PAD" error. 103 * 104 * The 64-bit 'cookie' field from the fw_crypto_lookaside_wr message 105 * in the request is returned in data[1] of the CPL_FW6_PLD message. 106 * 107 * For block cipher replies, the updated IV is supplied in data[2] and 108 * data[3] of the CPL_FW6_PLD message. 109 * 110 * For hash replies where the work request set 'hash_size' to request 111 * a copy of the hash in the reply, the hash digest is supplied 112 * immediately following the CPL_FW6_PLD message. 113 */ 114 115/* 116 * The crypto engine supports a maximum AAD size of 511 bytes. 117 */ 118#define MAX_AAD_LEN 511 119 120/* 121 * The documentation for CPL_RX_PHYS_DSGL claims a maximum of 32 SG 122 * entries. While the CPL includes a 16-bit length field, the T6 can 123 * sometimes hang if an error occurs while processing a request with a 124 * single DSGL entry larger than 2k. 125 */ 126#define MAX_RX_PHYS_DSGL_SGE 32 127#define DSGL_SGE_MAXLEN 2048 128 129/* 130 * The adapter only supports requests with a total input or output 131 * length of 64k-1 or smaller. Longer requests either result in hung 132 * requests or incorrect results. 133 */ 134#define MAX_REQUEST_SIZE 65535 135 136static MALLOC_DEFINE(M_CCR, "ccr", "Chelsio T6 crypto"); 137 138struct ccr_session_hmac { 139 struct auth_hash *auth_hash; 140 int hash_len; 141 unsigned int partial_digest_len; 142 unsigned int auth_mode; 143 unsigned int mk_size; 144 char pads[CHCR_HASH_MAX_BLOCK_SIZE_128 * 2]; 145}; 146 147struct ccr_session_gmac { 148 int hash_len; 149 char ghash_h[GMAC_BLOCK_LEN]; 150}; 151 152struct ccr_session_ccm_mac { 153 int hash_len; 154}; 155 156struct ccr_session_blkcipher { 157 unsigned int cipher_mode; 158 unsigned int key_len; 159 unsigned int iv_len; 160 __be32 key_ctx_hdr; 161 char enckey[CHCR_AES_MAX_KEY_LEN]; 162 char deckey[CHCR_AES_MAX_KEY_LEN]; 163}; 164 165struct ccr_port { 166 struct sge_wrq *txq; 167 struct sge_rxq *rxq; 168 int rx_channel_id; 169 int tx_channel_id; 170 u_int active_sessions; 171 172 counter_u64_t stats_queued; 173 counter_u64_t stats_completed; 174}; 175 176struct ccr_session { 177#ifdef INVARIANTS 178 int pending; 179#endif 180 enum { HASH, HMAC, BLKCIPHER, ETA, GCM, CCM } mode; 181 struct ccr_port *port; 182 union { 183 struct ccr_session_hmac hmac; 184 struct ccr_session_gmac gmac; 185 struct ccr_session_ccm_mac ccm_mac; 186 }; 187 struct ccr_session_blkcipher blkcipher; 188 struct mtx lock; 189 190 /* 191 * Pre-allocate S/G lists used when preparing a work request. 192 * 'sg_input' contains an sglist describing the entire input 193 * buffer for a 'struct cryptop'. 'sg_output' contains an 194 * sglist describing the entire output buffer. 'sg_ulptx' is 195 * used to describe the data the engine should DMA as input 196 * via ULPTX_SGL. 'sg_dsgl' is used to describe the 197 * destination that cipher text and a tag should be written 198 * to. 199 */ 200 struct sglist *sg_input; 201 struct sglist *sg_output; 202 struct sglist *sg_ulptx; 203 struct sglist *sg_dsgl; 204}; 205 206struct ccr_softc { 207 struct adapter *adapter; 208 device_t dev; 209 uint32_t cid; 210 struct mtx lock; 211 bool detaching; 212 struct ccr_port ports[MAX_NPORTS]; 213 u_int port_mask; 214 int first_rxq_id; 215 216 /* 217 * Pre-allocate a dummy output buffer for the IV and AAD for 218 * AEAD requests. 219 */ 220 char *iv_aad_buf; 221 struct sglist *sg_iv_aad; 222 223 /* Statistics. */ 224 counter_u64_t stats_blkcipher_encrypt; 225 counter_u64_t stats_blkcipher_decrypt; 226 counter_u64_t stats_hash; 227 counter_u64_t stats_hmac; 228 counter_u64_t stats_eta_encrypt; 229 counter_u64_t stats_eta_decrypt; 230 counter_u64_t stats_gcm_encrypt; 231 counter_u64_t stats_gcm_decrypt; 232 counter_u64_t stats_ccm_encrypt; 233 counter_u64_t stats_ccm_decrypt; 234 counter_u64_t stats_wr_nomem; 235 counter_u64_t stats_inflight; 236 counter_u64_t stats_mac_error; 237 counter_u64_t stats_pad_error; 238 counter_u64_t stats_sglist_error; 239 counter_u64_t stats_process_error; 240 counter_u64_t stats_sw_fallback; 241}; 242 243/* 244 * Crypto requests involve two kind of scatter/gather lists. 245 * 246 * Non-hash-only requests require a PHYS_DSGL that describes the 247 * location to store the results of the encryption or decryption 248 * operation. This SGL uses a different format (PHYS_DSGL) and should 249 * exclude the skip bytes at the start of the data as well as any AAD 250 * or IV. For authenticated encryption requests it should include the 251 * destination of the hash or tag. 252 * 253 * The input payload may either be supplied inline as immediate data, 254 * or via a standard ULP_TX SGL. This SGL should include AAD, 255 * ciphertext, and the hash or tag for authenticated decryption 256 * requests. 257 * 258 * These scatter/gather lists can describe different subsets of the 259 * buffers described by the crypto operation. ccr_populate_sglist() 260 * generates a scatter/gather list that covers an entire crypto 261 * operation buffer that is then used to construct the other 262 * scatter/gather lists. 263 */ 264static int 265ccr_populate_sglist(struct sglist *sg, struct crypto_buffer *cb) 266{ 267 int error; 268 269 sglist_reset(sg); 270 switch (cb->cb_type) { 271 case CRYPTO_BUF_MBUF: 272 error = sglist_append_mbuf(sg, cb->cb_mbuf); 273 break; 274 case CRYPTO_BUF_UIO: 275 error = sglist_append_uio(sg, cb->cb_uio); 276 break; 277 case CRYPTO_BUF_CONTIG: 278 error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len); 279 break; 280 case CRYPTO_BUF_VMPAGE: 281 error = sglist_append_vmpages(sg, cb->cb_vm_page, 282 cb->cb_vm_page_len, cb->cb_vm_page_offset); 283 break; 284 default: 285 error = EINVAL; 286 } 287 return (error); 288} 289 290/* 291 * Segments in 'sg' larger than 'maxsegsize' are counted as multiple 292 * segments. 293 */ 294static int 295ccr_count_sgl(struct sglist *sg, int maxsegsize) 296{ 297 int i, nsegs; 298 299 nsegs = 0; 300 for (i = 0; i < sg->sg_nseg; i++) 301 nsegs += howmany(sg->sg_segs[i].ss_len, maxsegsize); 302 return (nsegs); 303} 304 305/* These functions deal with PHYS_DSGL for the reply buffer. */ 306static inline int 307ccr_phys_dsgl_len(int nsegs) 308{ 309 int len; 310 311 len = (nsegs / 8) * sizeof(struct phys_sge_pairs); 312 if ((nsegs % 8) != 0) { 313 len += sizeof(uint16_t) * 8; 314 len += roundup2(nsegs % 8, 2) * sizeof(uint64_t); 315 } 316 return (len); 317} 318 319static void 320ccr_write_phys_dsgl(struct ccr_session *s, void *dst, int nsegs) 321{ 322 struct sglist *sg; 323 struct cpl_rx_phys_dsgl *cpl; 324 struct phys_sge_pairs *sgl; 325 vm_paddr_t paddr; 326 size_t seglen; 327 u_int i, j; 328 329 sg = s->sg_dsgl; 330 cpl = dst; 331 cpl->op_to_tid = htobe32(V_CPL_RX_PHYS_DSGL_OPCODE(CPL_RX_PHYS_DSGL) | 332 V_CPL_RX_PHYS_DSGL_ISRDMA(0)); 333 cpl->pcirlxorder_to_noofsgentr = htobe32( 334 V_CPL_RX_PHYS_DSGL_PCIRLXORDER(0) | 335 V_CPL_RX_PHYS_DSGL_PCINOSNOOP(0) | 336 V_CPL_RX_PHYS_DSGL_PCITPHNTENB(0) | V_CPL_RX_PHYS_DSGL_DCAID(0) | 337 V_CPL_RX_PHYS_DSGL_NOOFSGENTR(nsegs)); 338 cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; 339 cpl->rss_hdr_int.qid = htobe16(s->port->rxq->iq.abs_id); 340 cpl->rss_hdr_int.hash_val = 0; 341 cpl->rss_hdr_int.channel = s->port->rx_channel_id; 342 sgl = (struct phys_sge_pairs *)(cpl + 1); 343 j = 0; 344 for (i = 0; i < sg->sg_nseg; i++) { 345 seglen = sg->sg_segs[i].ss_len; 346 paddr = sg->sg_segs[i].ss_paddr; 347 do { 348 sgl->addr[j] = htobe64(paddr); 349 if (seglen > DSGL_SGE_MAXLEN) { 350 sgl->len[j] = htobe16(DSGL_SGE_MAXLEN); 351 paddr += DSGL_SGE_MAXLEN; 352 seglen -= DSGL_SGE_MAXLEN; 353 } else { 354 sgl->len[j] = htobe16(seglen); 355 seglen = 0; 356 } 357 j++; 358 if (j == 8) { 359 sgl++; 360 j = 0; 361 } 362 } while (seglen != 0); 363 } 364 MPASS(j + 8 * (sgl - (struct phys_sge_pairs *)(cpl + 1)) == nsegs); 365} 366 367/* These functions deal with the ULPTX_SGL for input payload. */ 368static inline int 369ccr_ulptx_sgl_len(int nsegs) 370{ 371 u_int n; 372 373 nsegs--; /* first segment is part of ulptx_sgl */ 374 n = sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); 375 return (roundup2(n, 16)); 376} 377 378static void 379ccr_write_ulptx_sgl(struct ccr_session *s, void *dst, int nsegs) 380{ 381 struct ulptx_sgl *usgl; 382 struct sglist *sg; 383 struct sglist_seg *ss; 384 int i; 385 386 sg = s->sg_ulptx; 387 MPASS(nsegs == sg->sg_nseg); 388 ss = &sg->sg_segs[0]; 389 usgl = dst; 390 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 391 V_ULPTX_NSGE(nsegs)); 392 usgl->len0 = htobe32(ss->ss_len); 393 usgl->addr0 = htobe64(ss->ss_paddr); 394 ss++; 395 for (i = 0; i < sg->sg_nseg - 1; i++) { 396 usgl->sge[i / 2].len[i & 1] = htobe32(ss->ss_len); 397 usgl->sge[i / 2].addr[i & 1] = htobe64(ss->ss_paddr); 398 ss++; 399 } 400} 401 402static bool 403ccr_use_imm_data(u_int transhdr_len, u_int input_len) 404{ 405 406 if (input_len > CRYPTO_MAX_IMM_TX_PKT_LEN) 407 return (false); 408 if (roundup2(transhdr_len, 16) + roundup2(input_len, 16) > 409 SGE_MAX_WR_LEN) 410 return (false); 411 return (true); 412} 413 414static void 415ccr_populate_wreq(struct ccr_softc *sc, struct ccr_session *s, 416 struct chcr_wr *crwr, u_int kctx_len, u_int wr_len, u_int imm_len, 417 u_int sgl_len, u_int hash_size, struct cryptop *crp) 418{ 419 u_int cctx_size, idata_len; 420 421 cctx_size = sizeof(struct _key_ctx) + kctx_len; 422 crwr->wreq.op_to_cctx_size = htobe32( 423 V_FW_CRYPTO_LOOKASIDE_WR_OPCODE(FW_CRYPTO_LOOKASIDE_WR) | 424 V_FW_CRYPTO_LOOKASIDE_WR_COMPL(0) | 425 V_FW_CRYPTO_LOOKASIDE_WR_IMM_LEN(imm_len) | 426 V_FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC(1) | 427 V_FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE(cctx_size >> 4)); 428 crwr->wreq.len16_pkd = htobe32( 429 V_FW_CRYPTO_LOOKASIDE_WR_LEN16(wr_len / 16)); 430 crwr->wreq.session_id = 0; 431 crwr->wreq.rx_chid_to_rx_q_id = htobe32( 432 V_FW_CRYPTO_LOOKASIDE_WR_RX_CHID(s->port->rx_channel_id) | 433 V_FW_CRYPTO_LOOKASIDE_WR_LCB(0) | 434 V_FW_CRYPTO_LOOKASIDE_WR_PHASH(0) | 435 V_FW_CRYPTO_LOOKASIDE_WR_IV(IV_NOP) | 436 V_FW_CRYPTO_LOOKASIDE_WR_FQIDX(0) | 437 V_FW_CRYPTO_LOOKASIDE_WR_TX_CH(0) | /* unused in firmware */ 438 V_FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID(s->port->rxq->iq.abs_id)); 439 crwr->wreq.key_addr = 0; 440 crwr->wreq.pld_size_hash_size = htobe32( 441 V_FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE(sgl_len) | 442 V_FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE(hash_size)); 443 crwr->wreq.cookie = htobe64((uintptr_t)crp); 444 445 crwr->ulptx.cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) | 446 V_ULP_TXPKT_DATAMODIFY(0) | 447 V_ULP_TXPKT_CHANNELID(s->port->tx_channel_id) | 448 V_ULP_TXPKT_DEST(0) | 449 V_ULP_TXPKT_FID(sc->first_rxq_id) | V_ULP_TXPKT_RO(1)); 450 crwr->ulptx.len = htobe32( 451 ((wr_len - sizeof(struct fw_crypto_lookaside_wr)) / 16)); 452 453 crwr->sc_imm.cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | 454 V_ULP_TX_SC_MORE(sgl_len != 0 ? 1 : 0)); 455 idata_len = wr_len - offsetof(struct chcr_wr, sec_cpl) - sgl_len; 456 if (imm_len % 16 != 0) 457 idata_len -= 16 - imm_len % 16; 458 crwr->sc_imm.len = htobe32(idata_len); 459} 460 461static int 462ccr_hash(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) 463{ 464 struct chcr_wr *crwr; 465 struct wrqe *wr; 466 struct auth_hash *axf; 467 char *dst; 468 u_int hash_size_in_response, kctx_flits, kctx_len, transhdr_len, wr_len; 469 u_int hmac_ctrl, imm_len, iopad_size; 470 int error, sgl_nsegs, sgl_len, use_opad; 471 472 /* Reject requests with too large of an input buffer. */ 473 if (crp->crp_payload_length > MAX_REQUEST_SIZE) 474 return (EFBIG); 475 476 axf = s->hmac.auth_hash; 477 478 if (s->mode == HMAC) { 479 use_opad = 1; 480 hmac_ctrl = SCMD_HMAC_CTRL_NO_TRUNC; 481 } else { 482 use_opad = 0; 483 hmac_ctrl = SCMD_HMAC_CTRL_NOP; 484 } 485 486 /* PADs must be 128-bit aligned. */ 487 iopad_size = roundup2(s->hmac.partial_digest_len, 16); 488 489 /* 490 * The 'key' part of the context includes the aligned IPAD and 491 * OPAD. 492 */ 493 kctx_len = iopad_size; 494 if (use_opad) 495 kctx_len += iopad_size; 496 hash_size_in_response = axf->hashsize; 497 transhdr_len = HASH_TRANSHDR_SIZE(kctx_len); 498 499 if (crp->crp_payload_length == 0) { 500 imm_len = axf->blocksize; 501 sgl_nsegs = 0; 502 sgl_len = 0; 503 } else if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length)) { 504 imm_len = crp->crp_payload_length; 505 sgl_nsegs = 0; 506 sgl_len = 0; 507 } else { 508 imm_len = 0; 509 sglist_reset(s->sg_ulptx); 510 error = sglist_append_sglist(s->sg_ulptx, s->sg_input, 511 crp->crp_payload_start, crp->crp_payload_length); 512 if (error) 513 return (error); 514 sgl_nsegs = s->sg_ulptx->sg_nseg; 515 sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); 516 } 517 518 wr_len = roundup2(transhdr_len, 16) + roundup2(imm_len, 16) + sgl_len; 519 if (wr_len > SGE_MAX_WR_LEN) 520 return (EFBIG); 521 wr = alloc_wrqe(wr_len, s->port->txq); 522 if (wr == NULL) { 523 counter_u64_add(sc->stats_wr_nomem, 1); 524 return (ENOMEM); 525 } 526 crwr = wrtod(wr); 527 memset(crwr, 0, wr_len); 528 529 ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len, 530 hash_size_in_response, crp); 531 532 crwr->sec_cpl.op_ivinsrtofst = htobe32( 533 V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | 534 V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) | 535 V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | 536 V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | 537 V_CPL_TX_SEC_PDU_IVINSRTOFST(0)); 538 539 crwr->sec_cpl.pldlen = htobe32(crp->crp_payload_length == 0 ? 540 axf->blocksize : crp->crp_payload_length); 541 542 crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( 543 V_CPL_TX_SEC_PDU_AUTHSTART(1) | V_CPL_TX_SEC_PDU_AUTHSTOP(0)); 544 545 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ 546 crwr->sec_cpl.seqno_numivs = htobe32( 547 V_SCMD_SEQ_NO_CTRL(0) | 548 V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | 549 V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_NOP) | 550 V_SCMD_AUTH_MODE(s->hmac.auth_mode) | 551 V_SCMD_HMAC_CTRL(hmac_ctrl)); 552 crwr->sec_cpl.ivgen_hdrlen = htobe32( 553 V_SCMD_LAST_FRAG(0) | 554 V_SCMD_MORE_FRAGS(crp->crp_payload_length == 0 ? 1 : 0) | 555 V_SCMD_MAC_ONLY(1)); 556 557 memcpy(crwr->key_ctx.key, s->hmac.pads, kctx_len); 558 559 /* XXX: F_KEY_CONTEXT_SALT_PRESENT set, but 'salt' not set. */ 560 kctx_flits = (sizeof(struct _key_ctx) + kctx_len) / 16; 561 crwr->key_ctx.ctx_hdr = htobe32(V_KEY_CONTEXT_CTX_LEN(kctx_flits) | 562 V_KEY_CONTEXT_OPAD_PRESENT(use_opad) | 563 V_KEY_CONTEXT_SALT_PRESENT(1) | 564 V_KEY_CONTEXT_CK_SIZE(CHCR_KEYCTX_NO_KEY) | 565 V_KEY_CONTEXT_MK_SIZE(s->hmac.mk_size) | V_KEY_CONTEXT_VALID(1)); 566 567 dst = (char *)(crwr + 1) + kctx_len + DUMMY_BYTES; 568 if (crp->crp_payload_length == 0) { 569 dst[0] = 0x80; 570 if (s->mode == HMAC) 571 *(uint64_t *)(dst + axf->blocksize - sizeof(uint64_t)) = 572 htobe64(axf->blocksize << 3); 573 } else if (imm_len != 0) 574 crypto_copydata(crp, crp->crp_payload_start, 575 crp->crp_payload_length, dst); 576 else 577 ccr_write_ulptx_sgl(s, dst, sgl_nsegs); 578 579 /* XXX: TODO backpressure */ 580 t4_wrq_tx(sc->adapter, wr); 581 582 return (0); 583} 584 585static int 586ccr_hash_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, 587 const struct cpl_fw6_pld *cpl, int error) 588{ 589 uint8_t hash[HASH_MAX_LEN]; 590 591 if (error) 592 return (error); 593 594 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 595 crypto_copydata(crp, crp->crp_digest_start, s->hmac.hash_len, 596 hash); 597 if (timingsafe_bcmp((cpl + 1), hash, s->hmac.hash_len) != 0) 598 return (EBADMSG); 599 } else 600 crypto_copyback(crp, crp->crp_digest_start, s->hmac.hash_len, 601 (cpl + 1)); 602 return (0); 603} 604 605static int 606ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) 607{ 608 char iv[CHCR_MAX_CRYPTO_IV_LEN]; 609 struct chcr_wr *crwr; 610 struct wrqe *wr; 611 char *dst; 612 u_int kctx_len, key_half, op_type, transhdr_len, wr_len; 613 u_int imm_len, iv_len; 614 int dsgl_nsegs, dsgl_len; 615 int sgl_nsegs, sgl_len; 616 int error; 617 618 if (s->blkcipher.key_len == 0 || crp->crp_payload_length == 0) 619 return (EINVAL); 620 if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_CBC && 621 (crp->crp_payload_length % AES_BLOCK_LEN) != 0) 622 return (EINVAL); 623 624 /* Reject requests with too large of an input buffer. */ 625 if (crp->crp_payload_length > MAX_REQUEST_SIZE) 626 return (EFBIG); 627 628 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 629 op_type = CHCR_ENCRYPT_OP; 630 else 631 op_type = CHCR_DECRYPT_OP; 632 633 sglist_reset(s->sg_dsgl); 634 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) 635 error = sglist_append_sglist(s->sg_dsgl, s->sg_output, 636 crp->crp_payload_output_start, crp->crp_payload_length); 637 else 638 error = sglist_append_sglist(s->sg_dsgl, s->sg_input, 639 crp->crp_payload_start, crp->crp_payload_length); 640 if (error) 641 return (error); 642 dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN); 643 if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) 644 return (EFBIG); 645 dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); 646 647 /* The 'key' must be 128-bit aligned. */ 648 kctx_len = roundup2(s->blkcipher.key_len, 16); 649 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len); 650 651 /* For AES-XTS we send a 16-byte IV in the work request. */ 652 if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS) 653 iv_len = AES_BLOCK_LEN; 654 else 655 iv_len = s->blkcipher.iv_len; 656 657 if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length + iv_len)) { 658 imm_len = crp->crp_payload_length; 659 sgl_nsegs = 0; 660 sgl_len = 0; 661 } else { 662 imm_len = 0; 663 sglist_reset(s->sg_ulptx); 664 error = sglist_append_sglist(s->sg_ulptx, s->sg_input, 665 crp->crp_payload_start, crp->crp_payload_length); 666 if (error) 667 return (error); 668 sgl_nsegs = s->sg_ulptx->sg_nseg; 669 sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); 670 } 671 672 wr_len = roundup2(transhdr_len, 16) + iv_len + 673 roundup2(imm_len, 16) + sgl_len; 674 if (wr_len > SGE_MAX_WR_LEN) 675 return (EFBIG); 676 wr = alloc_wrqe(wr_len, s->port->txq); 677 if (wr == NULL) { 678 counter_u64_add(sc->stats_wr_nomem, 1); 679 return (ENOMEM); 680 } 681 crwr = wrtod(wr); 682 memset(crwr, 0, wr_len); 683 684 crypto_read_iv(crp, iv); 685 686 /* Zero the remainder of the IV for AES-XTS. */ 687 memset(iv + s->blkcipher.iv_len, 0, iv_len - s->blkcipher.iv_len); 688 689 ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len, 0, 690 crp); 691 692 crwr->sec_cpl.op_ivinsrtofst = htobe32( 693 V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | 694 V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) | 695 V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | 696 V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | 697 V_CPL_TX_SEC_PDU_IVINSRTOFST(1)); 698 699 crwr->sec_cpl.pldlen = htobe32(iv_len + crp->crp_payload_length); 700 701 crwr->sec_cpl.aadstart_cipherstop_hi = htobe32( 702 V_CPL_TX_SEC_PDU_CIPHERSTART(iv_len + 1) | 703 V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0)); 704 crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( 705 V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0)); 706 707 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ 708 crwr->sec_cpl.seqno_numivs = htobe32( 709 V_SCMD_SEQ_NO_CTRL(0) | 710 V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | 711 V_SCMD_ENC_DEC_CTRL(op_type) | 712 V_SCMD_CIPH_MODE(s->blkcipher.cipher_mode) | 713 V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_NOP) | 714 V_SCMD_HMAC_CTRL(SCMD_HMAC_CTRL_NOP) | 715 V_SCMD_IV_SIZE(iv_len / 2) | 716 V_SCMD_NUM_IVS(0)); 717 crwr->sec_cpl.ivgen_hdrlen = htobe32( 718 V_SCMD_IV_GEN_CTRL(0) | 719 V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) | 720 V_SCMD_AADIVDROP(1) | V_SCMD_HDR_LEN(dsgl_len)); 721 722 crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr; 723 switch (s->blkcipher.cipher_mode) { 724 case SCMD_CIPH_MODE_AES_CBC: 725 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 726 memcpy(crwr->key_ctx.key, s->blkcipher.enckey, 727 s->blkcipher.key_len); 728 else 729 memcpy(crwr->key_ctx.key, s->blkcipher.deckey, 730 s->blkcipher.key_len); 731 break; 732 case SCMD_CIPH_MODE_AES_CTR: 733 memcpy(crwr->key_ctx.key, s->blkcipher.enckey, 734 s->blkcipher.key_len); 735 break; 736 case SCMD_CIPH_MODE_AES_XTS: 737 key_half = s->blkcipher.key_len / 2; 738 memcpy(crwr->key_ctx.key, s->blkcipher.enckey + key_half, 739 key_half); 740 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 741 memcpy(crwr->key_ctx.key + key_half, 742 s->blkcipher.enckey, key_half); 743 else 744 memcpy(crwr->key_ctx.key + key_half, 745 s->blkcipher.deckey, key_half); 746 break; 747 } 748 749 dst = (char *)(crwr + 1) + kctx_len; 750 ccr_write_phys_dsgl(s, dst, dsgl_nsegs); 751 dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; 752 memcpy(dst, iv, iv_len); 753 dst += iv_len; 754 if (imm_len != 0) 755 crypto_copydata(crp, crp->crp_payload_start, 756 crp->crp_payload_length, dst); 757 else 758 ccr_write_ulptx_sgl(s, dst, sgl_nsegs); 759 760 /* XXX: TODO backpressure */ 761 t4_wrq_tx(sc->adapter, wr); 762 763 explicit_bzero(iv, sizeof(iv)); 764 return (0); 765} 766 767static int 768ccr_blkcipher_done(struct ccr_softc *sc, struct ccr_session *s, 769 struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) 770{ 771 772 /* 773 * The updated IV to permit chained requests is at 774 * cpl->data[2], but OCF doesn't permit chained requests. 775 */ 776 return (error); 777} 778 779/* 780 * 'hashsize' is the length of a full digest. 'authsize' is the 781 * requested digest length for this operation which may be less 782 * than 'hashsize'. 783 */ 784static int 785ccr_hmac_ctrl(unsigned int hashsize, unsigned int authsize) 786{ 787 788 if (authsize == 10) 789 return (SCMD_HMAC_CTRL_TRUNC_RFC4366); 790 if (authsize == 12) 791 return (SCMD_HMAC_CTRL_IPSEC_96BIT); 792 if (authsize == hashsize / 2) 793 return (SCMD_HMAC_CTRL_DIV2); 794 return (SCMD_HMAC_CTRL_NO_TRUNC); 795} 796 797static int 798ccr_eta(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) 799{ 800 char iv[CHCR_MAX_CRYPTO_IV_LEN]; 801 struct chcr_wr *crwr; 802 struct wrqe *wr; 803 struct auth_hash *axf; 804 char *dst; 805 u_int kctx_len, key_half, op_type, transhdr_len, wr_len; 806 u_int hash_size_in_response, imm_len, iopad_size, iv_len; 807 u_int aad_start, aad_stop; 808 u_int auth_insert; 809 u_int cipher_start, cipher_stop; 810 u_int hmac_ctrl, input_len; 811 int dsgl_nsegs, dsgl_len; 812 int sgl_nsegs, sgl_len; 813 int error; 814 815 /* 816 * If there is a need in the future, requests with an empty 817 * payload could be supported as HMAC-only requests. 818 */ 819 if (s->blkcipher.key_len == 0 || crp->crp_payload_length == 0) 820 return (EINVAL); 821 if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_CBC && 822 (crp->crp_payload_length % AES_BLOCK_LEN) != 0) 823 return (EINVAL); 824 825 /* For AES-XTS we send a 16-byte IV in the work request. */ 826 if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS) 827 iv_len = AES_BLOCK_LEN; 828 else 829 iv_len = s->blkcipher.iv_len; 830 831 if (crp->crp_aad_length + iv_len > MAX_AAD_LEN) 832 return (EINVAL); 833 834 axf = s->hmac.auth_hash; 835 hash_size_in_response = s->hmac.hash_len; 836 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 837 op_type = CHCR_ENCRYPT_OP; 838 else 839 op_type = CHCR_DECRYPT_OP; 840 841 /* 842 * The output buffer consists of the cipher text followed by 843 * the hash when encrypting. For decryption it only contains 844 * the plain text. 845 * 846 * Due to a firmware bug, the output buffer must include a 847 * dummy output buffer for the IV and AAD prior to the real 848 * output buffer. 849 */ 850 if (op_type == CHCR_ENCRYPT_OP) { 851 if (iv_len + crp->crp_aad_length + crp->crp_payload_length + 852 hash_size_in_response > MAX_REQUEST_SIZE) 853 return (EFBIG); 854 } else { 855 if (iv_len + crp->crp_aad_length + crp->crp_payload_length > 856 MAX_REQUEST_SIZE) 857 return (EFBIG); 858 } 859 sglist_reset(s->sg_dsgl); 860 error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, 861 iv_len + crp->crp_aad_length); 862 if (error) 863 return (error); 864 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) 865 error = sglist_append_sglist(s->sg_dsgl, s->sg_output, 866 crp->crp_payload_output_start, crp->crp_payload_length); 867 else 868 error = sglist_append_sglist(s->sg_dsgl, s->sg_input, 869 crp->crp_payload_start, crp->crp_payload_length); 870 if (error) 871 return (error); 872 if (op_type == CHCR_ENCRYPT_OP) { 873 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) 874 error = sglist_append_sglist(s->sg_dsgl, s->sg_output, 875 crp->crp_digest_start, hash_size_in_response); 876 else 877 error = sglist_append_sglist(s->sg_dsgl, s->sg_input, 878 crp->crp_digest_start, hash_size_in_response); 879 if (error) 880 return (error); 881 } 882 dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN); 883 if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) 884 return (EFBIG); 885 dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); 886 887 /* PADs must be 128-bit aligned. */ 888 iopad_size = roundup2(s->hmac.partial_digest_len, 16); 889 890 /* 891 * The 'key' part of the key context consists of the key followed 892 * by the IPAD and OPAD. 893 */ 894 kctx_len = roundup2(s->blkcipher.key_len, 16) + iopad_size * 2; 895 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len); 896 897 /* 898 * The input buffer consists of the IV, any AAD, and then the 899 * cipher/plain text. For decryption requests the hash is 900 * appended after the cipher text. 901 * 902 * The IV is always stored at the start of the input buffer 903 * even though it may be duplicated in the payload. The 904 * crypto engine doesn't work properly if the IV offset points 905 * inside of the AAD region, so a second copy is always 906 * required. 907 */ 908 input_len = crp->crp_aad_length + crp->crp_payload_length; 909 910 /* 911 * The firmware hangs if sent a request which is a 912 * bit smaller than MAX_REQUEST_SIZE. In particular, the 913 * firmware appears to require 512 - 16 bytes of spare room 914 * along with the size of the hash even if the hash isn't 915 * included in the input buffer. 916 */ 917 if (input_len + roundup2(axf->hashsize, 16) + (512 - 16) > 918 MAX_REQUEST_SIZE) 919 return (EFBIG); 920 if (op_type == CHCR_DECRYPT_OP) 921 input_len += hash_size_in_response; 922 923 if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) { 924 imm_len = input_len; 925 sgl_nsegs = 0; 926 sgl_len = 0; 927 } else { 928 imm_len = 0; 929 sglist_reset(s->sg_ulptx); 930 if (crp->crp_aad_length != 0) { 931 if (crp->crp_aad != NULL) 932 error = sglist_append(s->sg_ulptx, 933 crp->crp_aad, crp->crp_aad_length); 934 else 935 error = sglist_append_sglist(s->sg_ulptx, 936 s->sg_input, crp->crp_aad_start, 937 crp->crp_aad_length); 938 if (error) 939 return (error); 940 } 941 error = sglist_append_sglist(s->sg_ulptx, s->sg_input, 942 crp->crp_payload_start, crp->crp_payload_length); 943 if (error) 944 return (error); 945 if (op_type == CHCR_DECRYPT_OP) { 946 error = sglist_append_sglist(s->sg_ulptx, s->sg_input, 947 crp->crp_digest_start, hash_size_in_response); 948 if (error) 949 return (error); 950 } 951 sgl_nsegs = s->sg_ulptx->sg_nseg; 952 sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); 953 } 954 955 /* Any AAD comes after the IV. */ 956 if (crp->crp_aad_length != 0) { 957 aad_start = iv_len + 1; 958 aad_stop = aad_start + crp->crp_aad_length - 1; 959 } else { 960 aad_start = 0; 961 aad_stop = 0; 962 } 963 cipher_start = iv_len + crp->crp_aad_length + 1; 964 if (op_type == CHCR_DECRYPT_OP) 965 cipher_stop = hash_size_in_response; 966 else 967 cipher_stop = 0; 968 if (op_type == CHCR_DECRYPT_OP) 969 auth_insert = hash_size_in_response; 970 else 971 auth_insert = 0; 972 973 wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) + 974 sgl_len; 975 if (wr_len > SGE_MAX_WR_LEN) 976 return (EFBIG); 977 wr = alloc_wrqe(wr_len, s->port->txq); 978 if (wr == NULL) { 979 counter_u64_add(sc->stats_wr_nomem, 1); 980 return (ENOMEM); 981 } 982 crwr = wrtod(wr); 983 memset(crwr, 0, wr_len); 984 985 crypto_read_iv(crp, iv); 986 987 /* Zero the remainder of the IV for AES-XTS. */ 988 memset(iv + s->blkcipher.iv_len, 0, iv_len - s->blkcipher.iv_len); 989 990 ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len, 991 op_type == CHCR_DECRYPT_OP ? hash_size_in_response : 0, crp); 992 993 crwr->sec_cpl.op_ivinsrtofst = htobe32( 994 V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | 995 V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) | 996 V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | 997 V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | 998 V_CPL_TX_SEC_PDU_IVINSRTOFST(1)); 999 1000 crwr->sec_cpl.pldlen = htobe32(iv_len + input_len); 1001 1002 crwr->sec_cpl.aadstart_cipherstop_hi = htobe32( 1003 V_CPL_TX_SEC_PDU_AADSTART(aad_start) | 1004 V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) | 1005 V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) | 1006 V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(cipher_stop >> 4)); 1007 crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( 1008 V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(cipher_stop & 0xf) | 1009 V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) | 1010 V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) | 1011 V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert)); 1012 1013 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ 1014 hmac_ctrl = ccr_hmac_ctrl(axf->hashsize, hash_size_in_response); 1015 crwr->sec_cpl.seqno_numivs = htobe32( 1016 V_SCMD_SEQ_NO_CTRL(0) | 1017 V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | 1018 V_SCMD_ENC_DEC_CTRL(op_type) | 1019 V_SCMD_CIPH_AUTH_SEQ_CTRL(op_type == CHCR_ENCRYPT_OP ? 1 : 0) | 1020 V_SCMD_CIPH_MODE(s->blkcipher.cipher_mode) | 1021 V_SCMD_AUTH_MODE(s->hmac.auth_mode) | 1022 V_SCMD_HMAC_CTRL(hmac_ctrl) | 1023 V_SCMD_IV_SIZE(iv_len / 2) | 1024 V_SCMD_NUM_IVS(0)); 1025 crwr->sec_cpl.ivgen_hdrlen = htobe32( 1026 V_SCMD_IV_GEN_CTRL(0) | 1027 V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) | 1028 V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len)); 1029 1030 crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr; 1031 switch (s->blkcipher.cipher_mode) { 1032 case SCMD_CIPH_MODE_AES_CBC: 1033 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 1034 memcpy(crwr->key_ctx.key, s->blkcipher.enckey, 1035 s->blkcipher.key_len); 1036 else 1037 memcpy(crwr->key_ctx.key, s->blkcipher.deckey, 1038 s->blkcipher.key_len); 1039 break; 1040 case SCMD_CIPH_MODE_AES_CTR: 1041 memcpy(crwr->key_ctx.key, s->blkcipher.enckey, 1042 s->blkcipher.key_len); 1043 break; 1044 case SCMD_CIPH_MODE_AES_XTS: 1045 key_half = s->blkcipher.key_len / 2; 1046 memcpy(crwr->key_ctx.key, s->blkcipher.enckey + key_half, 1047 key_half); 1048 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 1049 memcpy(crwr->key_ctx.key + key_half, 1050 s->blkcipher.enckey, key_half); 1051 else 1052 memcpy(crwr->key_ctx.key + key_half, 1053 s->blkcipher.deckey, key_half); 1054 break; 1055 } 1056 1057 dst = crwr->key_ctx.key + roundup2(s->blkcipher.key_len, 16); 1058 memcpy(dst, s->hmac.pads, iopad_size * 2); 1059 1060 dst = (char *)(crwr + 1) + kctx_len; 1061 ccr_write_phys_dsgl(s, dst, dsgl_nsegs); 1062 dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; 1063 memcpy(dst, iv, iv_len); 1064 dst += iv_len; 1065 if (imm_len != 0) { 1066 if (crp->crp_aad_length != 0) { 1067 if (crp->crp_aad != NULL) 1068 memcpy(dst, crp->crp_aad, crp->crp_aad_length); 1069 else 1070 crypto_copydata(crp, crp->crp_aad_start, 1071 crp->crp_aad_length, dst); 1072 dst += crp->crp_aad_length; 1073 } 1074 crypto_copydata(crp, crp->crp_payload_start, 1075 crp->crp_payload_length, dst); 1076 dst += crp->crp_payload_length; 1077 if (op_type == CHCR_DECRYPT_OP) 1078 crypto_copydata(crp, crp->crp_digest_start, 1079 hash_size_in_response, dst); 1080 } else 1081 ccr_write_ulptx_sgl(s, dst, sgl_nsegs); 1082 1083 /* XXX: TODO backpressure */ 1084 t4_wrq_tx(sc->adapter, wr); 1085 1086 explicit_bzero(iv, sizeof(iv)); 1087 return (0); 1088} 1089 1090static int 1091ccr_eta_done(struct ccr_softc *sc, struct ccr_session *s, 1092 struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) 1093{ 1094 1095 /* 1096 * The updated IV to permit chained requests is at 1097 * cpl->data[2], but OCF doesn't permit chained requests. 1098 */ 1099 return (error); 1100} 1101 1102static int 1103ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) 1104{ 1105 char iv[CHCR_MAX_CRYPTO_IV_LEN]; 1106 struct chcr_wr *crwr; 1107 struct wrqe *wr; 1108 char *dst; 1109 u_int iv_len, kctx_len, op_type, transhdr_len, wr_len; 1110 u_int hash_size_in_response, imm_len; 1111 u_int aad_start, aad_stop, cipher_start, cipher_stop, auth_insert; 1112 u_int hmac_ctrl, input_len; 1113 int dsgl_nsegs, dsgl_len; 1114 int sgl_nsegs, sgl_len; 1115 int error; 1116 1117 if (s->blkcipher.key_len == 0) 1118 return (EINVAL); 1119 1120 /* 1121 * The crypto engine doesn't handle GCM requests with an empty 1122 * payload, so handle those in software instead. 1123 */ 1124 if (crp->crp_payload_length == 0) 1125 return (EMSGSIZE); 1126 1127 if (crp->crp_aad_length + AES_BLOCK_LEN > MAX_AAD_LEN) 1128 return (EMSGSIZE); 1129 1130 hash_size_in_response = s->gmac.hash_len; 1131 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 1132 op_type = CHCR_ENCRYPT_OP; 1133 else 1134 op_type = CHCR_DECRYPT_OP; 1135 1136 /* 1137 * The IV handling for GCM in OCF is a bit more complicated in 1138 * that IPSec provides a full 16-byte IV (including the 1139 * counter), whereas the /dev/crypto interface sometimes 1140 * provides a full 16-byte IV (if no IV is provided in the 1141 * ioctl) and sometimes a 12-byte IV (if the IV was explicit). 1142 * 1143 * When provided a 12-byte IV, assume the IV is really 16 bytes 1144 * with a counter in the last 4 bytes initialized to 1. 1145 * 1146 * While iv_len is checked below, the value is currently 1147 * always set to 12 when creating a GCM session in this driver 1148 * due to limitations in OCF (there is no way to know what the 1149 * IV length of a given request will be). This means that the 1150 * driver always assumes as 12-byte IV for now. 1151 */ 1152 if (s->blkcipher.iv_len == 12) 1153 iv_len = AES_BLOCK_LEN; 1154 else 1155 iv_len = s->blkcipher.iv_len; 1156 1157 /* 1158 * GCM requests should always provide an explicit IV. 1159 */ 1160 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 1161 return (EINVAL); 1162 1163 /* 1164 * The output buffer consists of the cipher text followed by 1165 * the tag when encrypting. For decryption it only contains 1166 * the plain text. 1167 * 1168 * Due to a firmware bug, the output buffer must include a 1169 * dummy output buffer for the IV and AAD prior to the real 1170 * output buffer. 1171 */ 1172 if (op_type == CHCR_ENCRYPT_OP) { 1173 if (iv_len + crp->crp_aad_length + crp->crp_payload_length + 1174 hash_size_in_response > MAX_REQUEST_SIZE) 1175 return (EFBIG); 1176 } else { 1177 if (iv_len + crp->crp_aad_length + crp->crp_payload_length > 1178 MAX_REQUEST_SIZE) 1179 return (EFBIG); 1180 } 1181 sglist_reset(s->sg_dsgl); 1182 error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, iv_len + 1183 crp->crp_aad_length); 1184 if (error) 1185 return (error); 1186 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) 1187 error = sglist_append_sglist(s->sg_dsgl, s->sg_output, 1188 crp->crp_payload_output_start, crp->crp_payload_length); 1189 else 1190 error = sglist_append_sglist(s->sg_dsgl, s->sg_input, 1191 crp->crp_payload_start, crp->crp_payload_length); 1192 if (error) 1193 return (error); 1194 if (op_type == CHCR_ENCRYPT_OP) { 1195 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) 1196 error = sglist_append_sglist(s->sg_dsgl, s->sg_output, 1197 crp->crp_digest_start, hash_size_in_response); 1198 else 1199 error = sglist_append_sglist(s->sg_dsgl, s->sg_input, 1200 crp->crp_digest_start, hash_size_in_response); 1201 if (error) 1202 return (error); 1203 } 1204 dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN); 1205 if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) 1206 return (EFBIG); 1207 dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); 1208 1209 /* 1210 * The 'key' part of the key context consists of the key followed 1211 * by the Galois hash key. 1212 */ 1213 kctx_len = roundup2(s->blkcipher.key_len, 16) + GMAC_BLOCK_LEN; 1214 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len); 1215 1216 /* 1217 * The input buffer consists of the IV, any AAD, and then the 1218 * cipher/plain text. For decryption requests the hash is 1219 * appended after the cipher text. 1220 * 1221 * The IV is always stored at the start of the input buffer 1222 * even though it may be duplicated in the payload. The 1223 * crypto engine doesn't work properly if the IV offset points 1224 * inside of the AAD region, so a second copy is always 1225 * required. 1226 */ 1227 input_len = crp->crp_aad_length + crp->crp_payload_length; 1228 if (op_type == CHCR_DECRYPT_OP) 1229 input_len += hash_size_in_response; 1230 if (input_len > MAX_REQUEST_SIZE) 1231 return (EFBIG); 1232 if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) { 1233 imm_len = input_len; 1234 sgl_nsegs = 0; 1235 sgl_len = 0; 1236 } else { 1237 imm_len = 0; 1238 sglist_reset(s->sg_ulptx); 1239 if (crp->crp_aad_length != 0) { 1240 if (crp->crp_aad != NULL) 1241 error = sglist_append(s->sg_ulptx, 1242 crp->crp_aad, crp->crp_aad_length); 1243 else 1244 error = sglist_append_sglist(s->sg_ulptx, 1245 s->sg_input, crp->crp_aad_start, 1246 crp->crp_aad_length); 1247 if (error) 1248 return (error); 1249 } 1250 error = sglist_append_sglist(s->sg_ulptx, s->sg_input, 1251 crp->crp_payload_start, crp->crp_payload_length); 1252 if (error) 1253 return (error); 1254 if (op_type == CHCR_DECRYPT_OP) { 1255 error = sglist_append_sglist(s->sg_ulptx, s->sg_input, 1256 crp->crp_digest_start, hash_size_in_response); 1257 if (error) 1258 return (error); 1259 } 1260 sgl_nsegs = s->sg_ulptx->sg_nseg; 1261 sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); 1262 } 1263 1264 if (crp->crp_aad_length != 0) { 1265 aad_start = iv_len + 1; 1266 aad_stop = aad_start + crp->crp_aad_length - 1; 1267 } else { 1268 aad_start = 0; 1269 aad_stop = 0; 1270 } 1271 cipher_start = iv_len + crp->crp_aad_length + 1; 1272 if (op_type == CHCR_DECRYPT_OP) 1273 cipher_stop = hash_size_in_response; 1274 else 1275 cipher_stop = 0; 1276 if (op_type == CHCR_DECRYPT_OP) 1277 auth_insert = hash_size_in_response; 1278 else 1279 auth_insert = 0; 1280 1281 wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) + 1282 sgl_len; 1283 if (wr_len > SGE_MAX_WR_LEN) 1284 return (EFBIG); 1285 wr = alloc_wrqe(wr_len, s->port->txq); 1286 if (wr == NULL) { 1287 counter_u64_add(sc->stats_wr_nomem, 1); 1288 return (ENOMEM); 1289 } 1290 crwr = wrtod(wr); 1291 memset(crwr, 0, wr_len); 1292 1293 memcpy(iv, crp->crp_iv, s->blkcipher.iv_len); 1294 if (s->blkcipher.iv_len == 12) 1295 *(uint32_t *)&iv[12] = htobe32(1); 1296 1297 ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len, 0, 1298 crp); 1299 1300 crwr->sec_cpl.op_ivinsrtofst = htobe32( 1301 V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | 1302 V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) | 1303 V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | 1304 V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | 1305 V_CPL_TX_SEC_PDU_IVINSRTOFST(1)); 1306 1307 crwr->sec_cpl.pldlen = htobe32(iv_len + input_len); 1308 1309 /* 1310 * NB: cipherstop is explicitly set to 0. On encrypt it 1311 * should normally be set to 0 anyway. However, for decrypt 1312 * the cipher ends before the tag in the ETA case (and 1313 * authstop is set to stop before the tag), but for GCM the 1314 * cipher still runs to the end of the buffer. Not sure if 1315 * this is intentional or a firmware quirk, but it is required 1316 * for working tag validation with GCM decryption. 1317 */ 1318 crwr->sec_cpl.aadstart_cipherstop_hi = htobe32( 1319 V_CPL_TX_SEC_PDU_AADSTART(aad_start) | 1320 V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) | 1321 V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) | 1322 V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0)); 1323 crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( 1324 V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0) | 1325 V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) | 1326 V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) | 1327 V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert)); 1328 1329 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ 1330 hmac_ctrl = ccr_hmac_ctrl(AES_GMAC_HASH_LEN, hash_size_in_response); 1331 crwr->sec_cpl.seqno_numivs = htobe32( 1332 V_SCMD_SEQ_NO_CTRL(0) | 1333 V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | 1334 V_SCMD_ENC_DEC_CTRL(op_type) | 1335 V_SCMD_CIPH_AUTH_SEQ_CTRL(op_type == CHCR_ENCRYPT_OP ? 1 : 0) | 1336 V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_AES_GCM) | 1337 V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_GHASH) | 1338 V_SCMD_HMAC_CTRL(hmac_ctrl) | 1339 V_SCMD_IV_SIZE(iv_len / 2) | 1340 V_SCMD_NUM_IVS(0)); 1341 crwr->sec_cpl.ivgen_hdrlen = htobe32( 1342 V_SCMD_IV_GEN_CTRL(0) | 1343 V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) | 1344 V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len)); 1345 1346 crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr; 1347 memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); 1348 dst = crwr->key_ctx.key + roundup2(s->blkcipher.key_len, 16); 1349 memcpy(dst, s->gmac.ghash_h, GMAC_BLOCK_LEN); 1350 1351 dst = (char *)(crwr + 1) + kctx_len; 1352 ccr_write_phys_dsgl(s, dst, dsgl_nsegs); 1353 dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; 1354 memcpy(dst, iv, iv_len); 1355 dst += iv_len; 1356 if (imm_len != 0) { 1357 if (crp->crp_aad_length != 0) { 1358 if (crp->crp_aad != NULL) 1359 memcpy(dst, crp->crp_aad, crp->crp_aad_length); 1360 else 1361 crypto_copydata(crp, crp->crp_aad_start, 1362 crp->crp_aad_length, dst); 1363 dst += crp->crp_aad_length; 1364 } 1365 crypto_copydata(crp, crp->crp_payload_start, 1366 crp->crp_payload_length, dst); 1367 dst += crp->crp_payload_length; 1368 if (op_type == CHCR_DECRYPT_OP) 1369 crypto_copydata(crp, crp->crp_digest_start, 1370 hash_size_in_response, dst); 1371 } else 1372 ccr_write_ulptx_sgl(s, dst, sgl_nsegs); 1373 1374 /* XXX: TODO backpressure */ 1375 t4_wrq_tx(sc->adapter, wr); 1376 1377 explicit_bzero(iv, sizeof(iv)); 1378 return (0); 1379} 1380 1381static int 1382ccr_gcm_done(struct ccr_softc *sc, struct ccr_session *s, 1383 struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) 1384{ 1385 1386 /* 1387 * The updated IV to permit chained requests is at 1388 * cpl->data[2], but OCF doesn't permit chained requests. 1389 * 1390 * Note that the hardware should always verify the GMAC hash. 1391 */ 1392 return (error); 1393} 1394 1395/* 1396 * Handle a GCM request that is not supported by the crypto engine by 1397 * performing the operation in software. Derived from swcr_authenc(). 1398 */ 1399static void 1400ccr_gcm_soft(struct ccr_session *s, struct cryptop *crp) 1401{ 1402 struct auth_hash *axf; 1403 struct enc_xform *exf; 1404 void *auth_ctx, *kschedule; 1405 char block[GMAC_BLOCK_LEN]; 1406 char digest[GMAC_DIGEST_LEN]; 1407 char iv[AES_BLOCK_LEN]; 1408 int error, i, len; 1409 1410 auth_ctx = NULL; 1411 kschedule = NULL; 1412 1413 /* Initialize the MAC. */ 1414 switch (s->blkcipher.key_len) { 1415 case 16: 1416 axf = &auth_hash_nist_gmac_aes_128; 1417 break; 1418 case 24: 1419 axf = &auth_hash_nist_gmac_aes_192; 1420 break; 1421 case 32: 1422 axf = &auth_hash_nist_gmac_aes_256; 1423 break; 1424 default: 1425 error = EINVAL; 1426 goto out; 1427 } 1428 auth_ctx = malloc(axf->ctxsize, M_CCR, M_NOWAIT); 1429 if (auth_ctx == NULL) { 1430 error = ENOMEM; 1431 goto out; 1432 } 1433 axf->Init(auth_ctx); 1434 axf->Setkey(auth_ctx, s->blkcipher.enckey, s->blkcipher.key_len); 1435 1436 /* Initialize the cipher. */ 1437 exf = &enc_xform_aes_nist_gcm; 1438 kschedule = malloc(exf->ctxsize, M_CCR, M_NOWAIT); 1439 if (kschedule == NULL) { 1440 error = ENOMEM; 1441 goto out; 1442 } 1443 error = exf->setkey(kschedule, s->blkcipher.enckey, 1444 s->blkcipher.key_len); 1445 if (error) 1446 goto out; 1447 1448 /* 1449 * This assumes a 12-byte IV from the crp. See longer comment 1450 * above in ccr_gcm() for more details. 1451 */ 1452 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) { 1453 error = EINVAL; 1454 goto out; 1455 } 1456 memcpy(iv, crp->crp_iv, 12); 1457 *(uint32_t *)&iv[12] = htobe32(1); 1458 1459 axf->Reinit(auth_ctx, iv, sizeof(iv)); 1460 1461 /* MAC the AAD. */ 1462 if (crp->crp_aad != NULL) { 1463 len = rounddown(crp->crp_aad_length, sizeof(block)); 1464 if (len != 0) 1465 axf->Update(auth_ctx, crp->crp_aad, len); 1466 if (crp->crp_aad_length != len) { 1467 memset(block, 0, sizeof(block)); 1468 memcpy(block, (char *)crp->crp_aad + len, 1469 crp->crp_aad_length - len); 1470 axf->Update(auth_ctx, block, sizeof(block)); 1471 } 1472 } else { 1473 for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) { 1474 len = imin(crp->crp_aad_length - i, sizeof(block)); 1475 crypto_copydata(crp, crp->crp_aad_start + i, len, 1476 block); 1477 bzero(block + len, sizeof(block) - len); 1478 axf->Update(auth_ctx, block, sizeof(block)); 1479 } 1480 } 1481 1482 exf->reinit(kschedule, iv); 1483 1484 /* Do encryption with MAC */ 1485 for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) { 1486 len = imin(crp->crp_payload_length - i, sizeof(block)); 1487 crypto_copydata(crp, crp->crp_payload_start + i, len, block); 1488 bzero(block + len, sizeof(block) - len); 1489 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1490 exf->encrypt(kschedule, block, block); 1491 axf->Update(auth_ctx, block, len); 1492 crypto_copyback(crp, crp->crp_payload_start + i, len, 1493 block); 1494 } else { 1495 axf->Update(auth_ctx, block, len); 1496 } 1497 } 1498 1499 /* Length block. */ 1500 bzero(block, sizeof(block)); 1501 ((uint32_t *)block)[1] = htobe32(crp->crp_aad_length * 8); 1502 ((uint32_t *)block)[3] = htobe32(crp->crp_payload_length * 8); 1503 axf->Update(auth_ctx, block, sizeof(block)); 1504 1505 /* Finalize MAC. */ 1506 axf->Final(digest, auth_ctx); 1507 1508 /* Inject or validate tag. */ 1509 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1510 crypto_copyback(crp, crp->crp_digest_start, sizeof(digest), 1511 digest); 1512 error = 0; 1513 } else { 1514 char digest2[GMAC_DIGEST_LEN]; 1515 1516 crypto_copydata(crp, crp->crp_digest_start, sizeof(digest2), 1517 digest2); 1518 if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0) { 1519 error = 0; 1520 1521 /* Tag matches, decrypt data. */ 1522 for (i = 0; i < crp->crp_payload_length; 1523 i += sizeof(block)) { 1524 len = imin(crp->crp_payload_length - i, 1525 sizeof(block)); 1526 crypto_copydata(crp, crp->crp_payload_start + i, 1527 len, block); 1528 bzero(block + len, sizeof(block) - len); 1529 exf->decrypt(kschedule, block, block); 1530 crypto_copyback(crp, crp->crp_payload_start + i, 1531 len, block); 1532 } 1533 } else 1534 error = EBADMSG; 1535 explicit_bzero(digest2, sizeof(digest2)); 1536 } 1537 1538out: 1539 zfree(kschedule, M_CCR); 1540 zfree(auth_ctx, M_CCR); 1541 explicit_bzero(block, sizeof(block)); 1542 explicit_bzero(iv, sizeof(iv)); 1543 explicit_bzero(digest, sizeof(digest)); 1544 crp->crp_etype = error; 1545 crypto_done(crp); 1546} 1547 1548static void 1549generate_ccm_b0(struct cryptop *crp, u_int hash_size_in_response, 1550 const char *iv, char *b0) 1551{ 1552 u_int i, payload_len; 1553 1554 /* NB: L is already set in the first byte of the IV. */ 1555 memcpy(b0, iv, CCM_B0_SIZE); 1556 1557 /* Set length of hash in bits 3 - 5. */ 1558 b0[0] |= (((hash_size_in_response - 2) / 2) << 3); 1559 1560 /* Store the payload length as a big-endian value. */ 1561 payload_len = crp->crp_payload_length; 1562 for (i = 0; i < iv[0]; i++) { 1563 b0[CCM_CBC_BLOCK_LEN - 1 - i] = payload_len; 1564 payload_len >>= 8; 1565 } 1566 1567 /* 1568 * If there is AAD in the request, set bit 6 in the flags 1569 * field and store the AAD length as a big-endian value at the 1570 * start of block 1. This only assumes a 16-bit AAD length 1571 * since T6 doesn't support large AAD sizes. 1572 */ 1573 if (crp->crp_aad_length != 0) { 1574 b0[0] |= (1 << 6); 1575 *(uint16_t *)(b0 + CCM_B0_SIZE) = htobe16(crp->crp_aad_length); 1576 } 1577} 1578 1579static int 1580ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) 1581{ 1582 char iv[CHCR_MAX_CRYPTO_IV_LEN]; 1583 struct ulptx_idata *idata; 1584 struct chcr_wr *crwr; 1585 struct wrqe *wr; 1586 char *dst; 1587 u_int iv_len, kctx_len, op_type, transhdr_len, wr_len; 1588 u_int aad_len, b0_len, hash_size_in_response, imm_len; 1589 u_int aad_start, aad_stop, cipher_start, cipher_stop, auth_insert; 1590 u_int hmac_ctrl, input_len; 1591 int dsgl_nsegs, dsgl_len; 1592 int sgl_nsegs, sgl_len; 1593 int error; 1594 1595 if (s->blkcipher.key_len == 0) 1596 return (EINVAL); 1597 1598 /* 1599 * The crypto engine doesn't handle CCM requests with an empty 1600 * payload, so handle those in software instead. 1601 */ 1602 if (crp->crp_payload_length == 0) 1603 return (EMSGSIZE); 1604 1605 /* 1606 * CCM always includes block 0 in the AAD before AAD from the 1607 * request. 1608 */ 1609 b0_len = CCM_B0_SIZE; 1610 if (crp->crp_aad_length != 0) 1611 b0_len += CCM_AAD_FIELD_SIZE; 1612 aad_len = b0_len + crp->crp_aad_length; 1613 1614 /* 1615 * CCM requests should always provide an explicit IV (really 1616 * the nonce). 1617 */ 1618 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 1619 return (EINVAL); 1620 1621 /* 1622 * Always assume a 12 byte input nonce for now since that is 1623 * what OCF always generates. The full IV in the work request 1624 * is 16 bytes. 1625 */ 1626 iv_len = AES_BLOCK_LEN; 1627 1628 if (iv_len + aad_len > MAX_AAD_LEN) 1629 return (EMSGSIZE); 1630 1631 hash_size_in_response = s->ccm_mac.hash_len; 1632 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 1633 op_type = CHCR_ENCRYPT_OP; 1634 else 1635 op_type = CHCR_DECRYPT_OP; 1636 1637 /* 1638 * The output buffer consists of the cipher text followed by 1639 * the tag when encrypting. For decryption it only contains 1640 * the plain text. 1641 * 1642 * Due to a firmware bug, the output buffer must include a 1643 * dummy output buffer for the IV and AAD prior to the real 1644 * output buffer. 1645 */ 1646 if (op_type == CHCR_ENCRYPT_OP) { 1647 if (iv_len + aad_len + crp->crp_payload_length + 1648 hash_size_in_response > MAX_REQUEST_SIZE) 1649 return (EFBIG); 1650 } else { 1651 if (iv_len + aad_len + crp->crp_payload_length > 1652 MAX_REQUEST_SIZE) 1653 return (EFBIG); 1654 } 1655 sglist_reset(s->sg_dsgl); 1656 error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, iv_len + 1657 aad_len); 1658 if (error) 1659 return (error); 1660 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) 1661 error = sglist_append_sglist(s->sg_dsgl, s->sg_output, 1662 crp->crp_payload_output_start, crp->crp_payload_length); 1663 else 1664 error = sglist_append_sglist(s->sg_dsgl, s->sg_input, 1665 crp->crp_payload_start, crp->crp_payload_length); 1666 if (error) 1667 return (error); 1668 if (op_type == CHCR_ENCRYPT_OP) { 1669 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) 1670 error = sglist_append_sglist(s->sg_dsgl, s->sg_output, 1671 crp->crp_digest_start, hash_size_in_response); 1672 else 1673 error = sglist_append_sglist(s->sg_dsgl, s->sg_input, 1674 crp->crp_digest_start, hash_size_in_response); 1675 if (error) 1676 return (error); 1677 } 1678 dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN); 1679 if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) 1680 return (EFBIG); 1681 dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); 1682 1683 /* 1684 * The 'key' part of the key context consists of two copies of 1685 * the AES key. 1686 */ 1687 kctx_len = roundup2(s->blkcipher.key_len, 16) * 2; 1688 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len); 1689 1690 /* 1691 * The input buffer consists of the IV, AAD (including block 1692 * 0), and then the cipher/plain text. For decryption 1693 * requests the hash is appended after the cipher text. 1694 * 1695 * The IV is always stored at the start of the input buffer 1696 * even though it may be duplicated in the payload. The 1697 * crypto engine doesn't work properly if the IV offset points 1698 * inside of the AAD region, so a second copy is always 1699 * required. 1700 */ 1701 input_len = aad_len + crp->crp_payload_length; 1702 if (op_type == CHCR_DECRYPT_OP) 1703 input_len += hash_size_in_response; 1704 if (input_len > MAX_REQUEST_SIZE) 1705 return (EFBIG); 1706 if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) { 1707 imm_len = input_len; 1708 sgl_nsegs = 0; 1709 sgl_len = 0; 1710 } else { 1711 /* Block 0 is passed as immediate data. */ 1712 imm_len = b0_len; 1713 1714 sglist_reset(s->sg_ulptx); 1715 if (crp->crp_aad_length != 0) { 1716 if (crp->crp_aad != NULL) 1717 error = sglist_append(s->sg_ulptx, 1718 crp->crp_aad, crp->crp_aad_length); 1719 else 1720 error = sglist_append_sglist(s->sg_ulptx, 1721 s->sg_input, crp->crp_aad_start, 1722 crp->crp_aad_length); 1723 if (error) 1724 return (error); 1725 } 1726 error = sglist_append_sglist(s->sg_ulptx, s->sg_input, 1727 crp->crp_payload_start, crp->crp_payload_length); 1728 if (error) 1729 return (error); 1730 if (op_type == CHCR_DECRYPT_OP) { 1731 error = sglist_append_sglist(s->sg_ulptx, s->sg_input, 1732 crp->crp_digest_start, hash_size_in_response); 1733 if (error) 1734 return (error); 1735 } 1736 sgl_nsegs = s->sg_ulptx->sg_nseg; 1737 sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); 1738 } 1739 1740 aad_start = iv_len + 1; 1741 aad_stop = aad_start + aad_len - 1; 1742 cipher_start = aad_stop + 1; 1743 if (op_type == CHCR_DECRYPT_OP) 1744 cipher_stop = hash_size_in_response; 1745 else 1746 cipher_stop = 0; 1747 if (op_type == CHCR_DECRYPT_OP) 1748 auth_insert = hash_size_in_response; 1749 else 1750 auth_insert = 0; 1751 1752 wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) + 1753 sgl_len; 1754 if (wr_len > SGE_MAX_WR_LEN) 1755 return (EFBIG); 1756 wr = alloc_wrqe(wr_len, s->port->txq); 1757 if (wr == NULL) { 1758 counter_u64_add(sc->stats_wr_nomem, 1); 1759 return (ENOMEM); 1760 } 1761 crwr = wrtod(wr); 1762 memset(crwr, 0, wr_len); 1763 1764 /* 1765 * Read the nonce from the request. Use the nonce to generate 1766 * the full IV with the counter set to 0. 1767 */ 1768 memset(iv, 0, iv_len); 1769 iv[0] = (15 - AES_CCM_IV_LEN) - 1; 1770 memcpy(iv + 1, crp->crp_iv, AES_CCM_IV_LEN); 1771 1772 ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len, 0, 1773 crp); 1774 1775 crwr->sec_cpl.op_ivinsrtofst = htobe32( 1776 V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | 1777 V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) | 1778 V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | 1779 V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | 1780 V_CPL_TX_SEC_PDU_IVINSRTOFST(1)); 1781 1782 crwr->sec_cpl.pldlen = htobe32(iv_len + input_len); 1783 1784 /* 1785 * NB: cipherstop is explicitly set to 0. See comments above 1786 * in ccr_gcm(). 1787 */ 1788 crwr->sec_cpl.aadstart_cipherstop_hi = htobe32( 1789 V_CPL_TX_SEC_PDU_AADSTART(aad_start) | 1790 V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) | 1791 V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) | 1792 V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0)); 1793 crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( 1794 V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0) | 1795 V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) | 1796 V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) | 1797 V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert)); 1798 1799 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ 1800 hmac_ctrl = ccr_hmac_ctrl(AES_CBC_MAC_HASH_LEN, hash_size_in_response); 1801 crwr->sec_cpl.seqno_numivs = htobe32( 1802 V_SCMD_SEQ_NO_CTRL(0) | 1803 V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | 1804 V_SCMD_ENC_DEC_CTRL(op_type) | 1805 V_SCMD_CIPH_AUTH_SEQ_CTRL(op_type == CHCR_ENCRYPT_OP ? 0 : 1) | 1806 V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_AES_CCM) | 1807 V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_CBCMAC) | 1808 V_SCMD_HMAC_CTRL(hmac_ctrl) | 1809 V_SCMD_IV_SIZE(iv_len / 2) | 1810 V_SCMD_NUM_IVS(0)); 1811 crwr->sec_cpl.ivgen_hdrlen = htobe32( 1812 V_SCMD_IV_GEN_CTRL(0) | 1813 V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) | 1814 V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len)); 1815 1816 crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr; 1817 memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); 1818 memcpy(crwr->key_ctx.key + roundup(s->blkcipher.key_len, 16), 1819 s->blkcipher.enckey, s->blkcipher.key_len); 1820 1821 dst = (char *)(crwr + 1) + kctx_len; 1822 ccr_write_phys_dsgl(s, dst, dsgl_nsegs); 1823 dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; 1824 memcpy(dst, iv, iv_len); 1825 dst += iv_len; 1826 generate_ccm_b0(crp, hash_size_in_response, iv, dst); 1827 if (sgl_nsegs == 0) { 1828 dst += b0_len; 1829 if (crp->crp_aad_length != 0) { 1830 if (crp->crp_aad != NULL) 1831 memcpy(dst, crp->crp_aad, crp->crp_aad_length); 1832 else 1833 crypto_copydata(crp, crp->crp_aad_start, 1834 crp->crp_aad_length, dst); 1835 dst += crp->crp_aad_length; 1836 } 1837 crypto_copydata(crp, crp->crp_payload_start, 1838 crp->crp_payload_length, dst); 1839 dst += crp->crp_payload_length; 1840 if (op_type == CHCR_DECRYPT_OP) 1841 crypto_copydata(crp, crp->crp_digest_start, 1842 hash_size_in_response, dst); 1843 } else { 1844 dst += CCM_B0_SIZE; 1845 if (b0_len > CCM_B0_SIZE) { 1846 /* 1847 * If there is AAD, insert padding including a 1848 * ULP_TX_SC_NOOP so that the ULP_TX_SC_DSGL 1849 * is 16-byte aligned. 1850 */ 1851 KASSERT(b0_len - CCM_B0_SIZE == CCM_AAD_FIELD_SIZE, 1852 ("b0_len mismatch")); 1853 memset(dst + CCM_AAD_FIELD_SIZE, 0, 1854 8 - CCM_AAD_FIELD_SIZE); 1855 idata = (void *)(dst + 8); 1856 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); 1857 idata->len = htobe32(0); 1858 dst = (void *)(idata + 1); 1859 } 1860 ccr_write_ulptx_sgl(s, dst, sgl_nsegs); 1861 } 1862 1863 /* XXX: TODO backpressure */ 1864 t4_wrq_tx(sc->adapter, wr); 1865 1866 explicit_bzero(iv, sizeof(iv)); 1867 return (0); 1868} 1869 1870static int 1871ccr_ccm_done(struct ccr_softc *sc, struct ccr_session *s, 1872 struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) 1873{ 1874 1875 /* 1876 * The updated IV to permit chained requests is at 1877 * cpl->data[2], but OCF doesn't permit chained requests. 1878 * 1879 * Note that the hardware should always verify the CBC MAC 1880 * hash. 1881 */ 1882 return (error); 1883} 1884 1885/* 1886 * Handle a CCM request that is not supported by the crypto engine by 1887 * performing the operation in software. Derived from swcr_authenc(). 1888 */ 1889static void 1890ccr_ccm_soft(struct ccr_session *s, struct cryptop *crp) 1891{ 1892 struct auth_hash *axf; 1893 struct enc_xform *exf; 1894 union authctx *auth_ctx; 1895 void *kschedule; 1896 char block[CCM_CBC_BLOCK_LEN]; 1897 char digest[AES_CBC_MAC_HASH_LEN]; 1898 char iv[AES_CCM_IV_LEN]; 1899 int error, i, len; 1900 1901 auth_ctx = NULL; 1902 kschedule = NULL; 1903 1904 /* Initialize the MAC. */ 1905 switch (s->blkcipher.key_len) { 1906 case 16: 1907 axf = &auth_hash_ccm_cbc_mac_128; 1908 break; 1909 case 24: 1910 axf = &auth_hash_ccm_cbc_mac_192; 1911 break; 1912 case 32: 1913 axf = &auth_hash_ccm_cbc_mac_256; 1914 break; 1915 default: 1916 error = EINVAL; 1917 goto out; 1918 } 1919 auth_ctx = malloc(axf->ctxsize, M_CCR, M_NOWAIT); 1920 if (auth_ctx == NULL) { 1921 error = ENOMEM; 1922 goto out; 1923 } 1924 axf->Init(auth_ctx); 1925 axf->Setkey(auth_ctx, s->blkcipher.enckey, s->blkcipher.key_len); 1926 1927 /* Initialize the cipher. */ 1928 exf = &enc_xform_ccm; 1929 kschedule = malloc(exf->ctxsize, M_CCR, M_NOWAIT); 1930 if (kschedule == NULL) { 1931 error = ENOMEM; 1932 goto out; 1933 } 1934 error = exf->setkey(kschedule, s->blkcipher.enckey, 1935 s->blkcipher.key_len); 1936 if (error) 1937 goto out; 1938 1939 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) { 1940 error = EINVAL; 1941 goto out; 1942 } 1943 memcpy(iv, crp->crp_iv, AES_CCM_IV_LEN); 1944 1945 auth_ctx->aes_cbc_mac_ctx.authDataLength = crp->crp_aad_length; 1946 auth_ctx->aes_cbc_mac_ctx.cryptDataLength = crp->crp_payload_length; 1947 axf->Reinit(auth_ctx, iv, sizeof(iv)); 1948 1949 /* MAC the AAD. */ 1950 if (crp->crp_aad != NULL) 1951 error = axf->Update(auth_ctx, crp->crp_aad, 1952 crp->crp_aad_length); 1953 else 1954 error = crypto_apply(crp, crp->crp_aad_start, 1955 crp->crp_aad_length, axf->Update, auth_ctx); 1956 if (error) 1957 goto out; 1958 1959 exf->reinit(kschedule, iv); 1960 1961 /* Do encryption/decryption with MAC */ 1962 for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) { 1963 len = imin(crp->crp_payload_length - i, sizeof(block)); 1964 crypto_copydata(crp, crp->crp_payload_start + i, len, block); 1965 bzero(block + len, sizeof(block) - len); 1966 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1967 axf->Update(auth_ctx, block, len); 1968 exf->encrypt(kschedule, block, block); 1969 crypto_copyback(crp, crp->crp_payload_start + i, len, 1970 block); 1971 } else { 1972 exf->decrypt(kschedule, block, block); 1973 axf->Update(auth_ctx, block, len); 1974 } 1975 } 1976 1977 /* Finalize MAC. */ 1978 axf->Final(digest, auth_ctx); 1979 1980 /* Inject or validate tag. */ 1981 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1982 crypto_copyback(crp, crp->crp_digest_start, sizeof(digest), 1983 digest); 1984 error = 0; 1985 } else { 1986 char digest2[AES_CBC_MAC_HASH_LEN]; 1987 1988 crypto_copydata(crp, crp->crp_digest_start, sizeof(digest2), 1989 digest2); 1990 if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0) { 1991 error = 0; 1992 1993 /* Tag matches, decrypt data. */ 1994 exf->reinit(kschedule, iv); 1995 for (i = 0; i < crp->crp_payload_length; 1996 i += sizeof(block)) { 1997 len = imin(crp->crp_payload_length - i, 1998 sizeof(block)); 1999 crypto_copydata(crp, crp->crp_payload_start + i, 2000 len, block); 2001 bzero(block + len, sizeof(block) - len); 2002 exf->decrypt(kschedule, block, block); 2003 crypto_copyback(crp, crp->crp_payload_start + i, 2004 len, block); 2005 } 2006 } else 2007 error = EBADMSG; 2008 explicit_bzero(digest2, sizeof(digest2)); 2009 } 2010 2011out: 2012 zfree(kschedule, M_CCR); 2013 zfree(auth_ctx, M_CCR); 2014 explicit_bzero(block, sizeof(block)); 2015 explicit_bzero(iv, sizeof(iv)); 2016 explicit_bzero(digest, sizeof(digest)); 2017 crp->crp_etype = error; 2018 crypto_done(crp); 2019} 2020 2021static void 2022ccr_identify(driver_t *driver, device_t parent) 2023{ 2024 struct adapter *sc; 2025 2026 sc = device_get_softc(parent); 2027 if (sc->cryptocaps & FW_CAPS_CONFIG_CRYPTO_LOOKASIDE && 2028 device_find_child(parent, "ccr", -1) == NULL) 2029 device_add_child(parent, "ccr", -1); 2030} 2031 2032static int 2033ccr_probe(device_t dev) 2034{ 2035 2036 device_set_desc(dev, "Chelsio Crypto Accelerator"); 2037 return (BUS_PROBE_DEFAULT); 2038} 2039 2040static void 2041ccr_sysctls(struct ccr_softc *sc) 2042{ 2043 struct sysctl_ctx_list *ctx; 2044 struct sysctl_oid *oid, *port_oid; 2045 struct sysctl_oid_list *children; 2046 char buf[16]; 2047 int i; 2048 2049 ctx = device_get_sysctl_ctx(sc->dev); 2050 2051 /* 2052 * dev.ccr.X. 2053 */ 2054 oid = device_get_sysctl_tree(sc->dev); 2055 children = SYSCTL_CHILDREN(oid); 2056 2057 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "port_mask", CTLFLAG_RW, 2058 &sc->port_mask, 0, "Mask of enabled ports"); 2059 2060 /* 2061 * dev.ccr.X.stats. 2062 */ 2063 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", 2064 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics"); 2065 children = SYSCTL_CHILDREN(oid); 2066 2067 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "hash", CTLFLAG_RD, 2068 &sc->stats_hash, "Hash requests submitted"); 2069 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "hmac", CTLFLAG_RD, 2070 &sc->stats_hmac, "HMAC requests submitted"); 2071 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "cipher_encrypt", 2072 CTLFLAG_RD, &sc->stats_blkcipher_encrypt, 2073 "Cipher encryption requests submitted"); 2074 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "cipher_decrypt", 2075 CTLFLAG_RD, &sc->stats_blkcipher_decrypt, 2076 "Cipher decryption requests submitted"); 2077 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "eta_encrypt", 2078 CTLFLAG_RD, &sc->stats_eta_encrypt, 2079 "Combined AES+HMAC encryption requests submitted"); 2080 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "eta_decrypt", 2081 CTLFLAG_RD, &sc->stats_eta_decrypt, 2082 "Combined AES+HMAC decryption requests submitted"); 2083 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_encrypt", 2084 CTLFLAG_RD, &sc->stats_gcm_encrypt, 2085 "AES-GCM encryption requests submitted"); 2086 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_decrypt", 2087 CTLFLAG_RD, &sc->stats_gcm_decrypt, 2088 "AES-GCM decryption requests submitted"); 2089 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ccm_encrypt", 2090 CTLFLAG_RD, &sc->stats_ccm_encrypt, 2091 "AES-CCM encryption requests submitted"); 2092 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ccm_decrypt", 2093 CTLFLAG_RD, &sc->stats_ccm_decrypt, 2094 "AES-CCM decryption requests submitted"); 2095 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "wr_nomem", CTLFLAG_RD, 2096 &sc->stats_wr_nomem, "Work request memory allocation failures"); 2097 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "inflight", CTLFLAG_RD, 2098 &sc->stats_inflight, "Requests currently pending"); 2099 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "mac_error", CTLFLAG_RD, 2100 &sc->stats_mac_error, "MAC errors"); 2101 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "pad_error", CTLFLAG_RD, 2102 &sc->stats_pad_error, "Padding errors"); 2103 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sglist_error", 2104 CTLFLAG_RD, &sc->stats_sglist_error, 2105 "Requests for which DMA mapping failed"); 2106 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "process_error", 2107 CTLFLAG_RD, &sc->stats_process_error, 2108 "Requests failed during queueing"); 2109 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sw_fallback", 2110 CTLFLAG_RD, &sc->stats_sw_fallback, 2111 "Requests processed by falling back to software"); 2112 2113 /* 2114 * dev.ccr.X.stats.port 2115 */ 2116 port_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "port", 2117 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Per-port statistics"); 2118 2119 for (i = 0; i < nitems(sc->ports); i++) { 2120 if (sc->ports[i].rxq == NULL) 2121 continue; 2122 2123 /* 2124 * dev.ccr.X.stats.port.Y 2125 */ 2126 snprintf(buf, sizeof(buf), "%d", i); 2127 oid = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(port_oid), OID_AUTO, 2128 buf, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, buf); 2129 children = SYSCTL_CHILDREN(oid); 2130 2131 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "active_sessions", 2132 CTLFLAG_RD, &sc->ports[i].active_sessions, 0, 2133 "Count of active sessions"); 2134 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "queued", 2135 CTLFLAG_RD, &sc->ports[i].stats_queued, "Requests queued"); 2136 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "completed", 2137 CTLFLAG_RD, &sc->ports[i].stats_completed, 2138 "Requests completed"); 2139 } 2140} 2141 2142static void 2143ccr_init_port(struct ccr_softc *sc, int port) 2144{ 2145 struct port_info *pi; 2146 2147 pi = sc->adapter->port[port]; 2148 sc->ports[port].txq = &sc->adapter->sge.ctrlq[port]; 2149 sc->ports[port].rxq = &sc->adapter->sge.rxq[pi->vi->first_rxq]; 2150 sc->ports[port].rx_channel_id = pi->rx_c_chan; 2151 sc->ports[port].tx_channel_id = pi->tx_chan; 2152 sc->ports[port].stats_queued = counter_u64_alloc(M_WAITOK); 2153 sc->ports[port].stats_completed = counter_u64_alloc(M_WAITOK); 2154 _Static_assert(sizeof(sc->port_mask) * NBBY >= MAX_NPORTS - 1, 2155 "Too many ports to fit in port_mask"); 2156 2157 /* 2158 * Completions for crypto requests on port 1 can sometimes 2159 * return a stale cookie value due to a firmware bug. Disable 2160 * requests on port 1 by default on affected firmware. 2161 */ 2162 if (sc->adapter->params.fw_vers >= FW_VERSION32(1, 25, 4, 0) || 2163 port == 0) 2164 sc->port_mask |= 1u << port; 2165} 2166 2167static int 2168ccr_attach(device_t dev) 2169{ 2170 struct ccr_softc *sc; 2171 int32_t cid; 2172 int i; 2173 2174 sc = device_get_softc(dev); 2175 sc->dev = dev; 2176 sc->adapter = device_get_softc(device_get_parent(dev)); 2177 for_each_port(sc->adapter, i) { 2178 ccr_init_port(sc, i); 2179 } 2180 cid = crypto_get_driverid(dev, sizeof(struct ccr_session), 2181 CRYPTOCAP_F_HARDWARE); 2182 if (cid < 0) { 2183 device_printf(dev, "could not get crypto driver id\n"); 2184 return (ENXIO); 2185 } 2186 sc->cid = cid; 2187 sc->adapter->ccr_softc = sc; 2188 2189 /* 2190 * The FID must be the first RXQ for port 0 regardless of 2191 * which port is used to service the request. 2192 */ 2193 sc->first_rxq_id = sc->adapter->sge.rxq[0].iq.abs_id; 2194 2195 mtx_init(&sc->lock, "ccr", NULL, MTX_DEF); 2196 sc->iv_aad_buf = malloc(MAX_AAD_LEN, M_CCR, M_WAITOK); 2197 sc->sg_iv_aad = sglist_build(sc->iv_aad_buf, MAX_AAD_LEN, M_WAITOK); 2198 sc->stats_blkcipher_encrypt = counter_u64_alloc(M_WAITOK); 2199 sc->stats_blkcipher_decrypt = counter_u64_alloc(M_WAITOK); 2200 sc->stats_hash = counter_u64_alloc(M_WAITOK); 2201 sc->stats_hmac = counter_u64_alloc(M_WAITOK); 2202 sc->stats_eta_encrypt = counter_u64_alloc(M_WAITOK); 2203 sc->stats_eta_decrypt = counter_u64_alloc(M_WAITOK); 2204 sc->stats_gcm_encrypt = counter_u64_alloc(M_WAITOK); 2205 sc->stats_gcm_decrypt = counter_u64_alloc(M_WAITOK); 2206 sc->stats_ccm_encrypt = counter_u64_alloc(M_WAITOK); 2207 sc->stats_ccm_decrypt = counter_u64_alloc(M_WAITOK); 2208 sc->stats_wr_nomem = counter_u64_alloc(M_WAITOK); 2209 sc->stats_inflight = counter_u64_alloc(M_WAITOK); 2210 sc->stats_mac_error = counter_u64_alloc(M_WAITOK); 2211 sc->stats_pad_error = counter_u64_alloc(M_WAITOK); 2212 sc->stats_sglist_error = counter_u64_alloc(M_WAITOK); 2213 sc->stats_process_error = counter_u64_alloc(M_WAITOK); 2214 sc->stats_sw_fallback = counter_u64_alloc(M_WAITOK); 2215 ccr_sysctls(sc); 2216 2217 return (0); 2218} 2219 2220static void 2221ccr_free_port(struct ccr_softc *sc, int port) 2222{ 2223 2224 counter_u64_free(sc->ports[port].stats_queued); 2225 counter_u64_free(sc->ports[port].stats_completed); 2226} 2227 2228static int 2229ccr_detach(device_t dev) 2230{ 2231 struct ccr_softc *sc; 2232 int i; 2233 2234 sc = device_get_softc(dev); 2235 2236 mtx_lock(&sc->lock); 2237 sc->detaching = true; 2238 mtx_unlock(&sc->lock); 2239 2240 crypto_unregister_all(sc->cid); 2241 2242 mtx_destroy(&sc->lock); 2243 counter_u64_free(sc->stats_blkcipher_encrypt); 2244 counter_u64_free(sc->stats_blkcipher_decrypt); 2245 counter_u64_free(sc->stats_hash); 2246 counter_u64_free(sc->stats_hmac); 2247 counter_u64_free(sc->stats_eta_encrypt); 2248 counter_u64_free(sc->stats_eta_decrypt); 2249 counter_u64_free(sc->stats_gcm_encrypt); 2250 counter_u64_free(sc->stats_gcm_decrypt); 2251 counter_u64_free(sc->stats_ccm_encrypt); 2252 counter_u64_free(sc->stats_ccm_decrypt); 2253 counter_u64_free(sc->stats_wr_nomem); 2254 counter_u64_free(sc->stats_inflight); 2255 counter_u64_free(sc->stats_mac_error); 2256 counter_u64_free(sc->stats_pad_error); 2257 counter_u64_free(sc->stats_sglist_error); 2258 counter_u64_free(sc->stats_process_error); 2259 counter_u64_free(sc->stats_sw_fallback); 2260 for_each_port(sc->adapter, i) { 2261 ccr_free_port(sc, i); 2262 } 2263 sglist_free(sc->sg_iv_aad); 2264 free(sc->iv_aad_buf, M_CCR); 2265 sc->adapter->ccr_softc = NULL; 2266 return (0); 2267} 2268 2269static void 2270ccr_init_hash_digest(struct ccr_session *s) 2271{ 2272 union authctx auth_ctx; 2273 struct auth_hash *axf; 2274 2275 axf = s->hmac.auth_hash; 2276 axf->Init(&auth_ctx); 2277 t4_copy_partial_hash(axf->type, &auth_ctx, s->hmac.pads); 2278} 2279 2280static bool 2281ccr_aes_check_keylen(int alg, int klen) 2282{ 2283 2284 switch (klen * 8) { 2285 case 128: 2286 case 192: 2287 if (alg == CRYPTO_AES_XTS) 2288 return (false); 2289 break; 2290 case 256: 2291 break; 2292 case 512: 2293 if (alg != CRYPTO_AES_XTS) 2294 return (false); 2295 break; 2296 default: 2297 return (false); 2298 } 2299 return (true); 2300} 2301 2302static void 2303ccr_aes_setkey(struct ccr_session *s, const void *key, int klen) 2304{ 2305 unsigned int ck_size, iopad_size, kctx_flits, kctx_len, kbits, mk_size; 2306 unsigned int opad_present; 2307 2308 if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS) 2309 kbits = (klen / 2) * 8; 2310 else 2311 kbits = klen * 8; 2312 switch (kbits) { 2313 case 128: 2314 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 2315 break; 2316 case 192: 2317 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 2318 break; 2319 case 256: 2320 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 2321 break; 2322 default: 2323 panic("should not get here"); 2324 } 2325 2326 s->blkcipher.key_len = klen; 2327 memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len); 2328 switch (s->blkcipher.cipher_mode) { 2329 case SCMD_CIPH_MODE_AES_CBC: 2330 case SCMD_CIPH_MODE_AES_XTS: 2331 t4_aes_getdeckey(s->blkcipher.deckey, key, kbits); 2332 break; 2333 } 2334 2335 kctx_len = roundup2(s->blkcipher.key_len, 16); 2336 switch (s->mode) { 2337 case ETA: 2338 mk_size = s->hmac.mk_size; 2339 opad_present = 1; 2340 iopad_size = roundup2(s->hmac.partial_digest_len, 16); 2341 kctx_len += iopad_size * 2; 2342 break; 2343 case GCM: 2344 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; 2345 opad_present = 0; 2346 kctx_len += GMAC_BLOCK_LEN; 2347 break; 2348 case CCM: 2349 switch (kbits) { 2350 case 128: 2351 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; 2352 break; 2353 case 192: 2354 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192; 2355 break; 2356 case 256: 2357 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 2358 break; 2359 default: 2360 panic("should not get here"); 2361 } 2362 opad_present = 0; 2363 kctx_len *= 2; 2364 break; 2365 default: 2366 mk_size = CHCR_KEYCTX_NO_KEY; 2367 opad_present = 0; 2368 break; 2369 } 2370 kctx_flits = (sizeof(struct _key_ctx) + kctx_len) / 16; 2371 s->blkcipher.key_ctx_hdr = htobe32(V_KEY_CONTEXT_CTX_LEN(kctx_flits) | 2372 V_KEY_CONTEXT_DUAL_CK(s->blkcipher.cipher_mode == 2373 SCMD_CIPH_MODE_AES_XTS) | 2374 V_KEY_CONTEXT_OPAD_PRESENT(opad_present) | 2375 V_KEY_CONTEXT_SALT_PRESENT(1) | V_KEY_CONTEXT_CK_SIZE(ck_size) | 2376 V_KEY_CONTEXT_MK_SIZE(mk_size) | V_KEY_CONTEXT_VALID(1)); 2377} 2378 2379static bool 2380ccr_auth_supported(const struct crypto_session_params *csp) 2381{ 2382 2383 switch (csp->csp_auth_alg) { 2384 case CRYPTO_SHA1: 2385 case CRYPTO_SHA2_224: 2386 case CRYPTO_SHA2_256: 2387 case CRYPTO_SHA2_384: 2388 case CRYPTO_SHA2_512: 2389 case CRYPTO_SHA1_HMAC: 2390 case CRYPTO_SHA2_224_HMAC: 2391 case CRYPTO_SHA2_256_HMAC: 2392 case CRYPTO_SHA2_384_HMAC: 2393 case CRYPTO_SHA2_512_HMAC: 2394 break; 2395 default: 2396 return (false); 2397 } 2398 return (true); 2399} 2400 2401static bool 2402ccr_cipher_supported(const struct crypto_session_params *csp) 2403{ 2404 2405 switch (csp->csp_cipher_alg) { 2406 case CRYPTO_AES_CBC: 2407 if (csp->csp_ivlen != AES_BLOCK_LEN) 2408 return (false); 2409 break; 2410 case CRYPTO_AES_ICM: 2411 if (csp->csp_ivlen != AES_BLOCK_LEN) 2412 return (false); 2413 break; 2414 case CRYPTO_AES_XTS: 2415 if (csp->csp_ivlen != AES_XTS_IV_LEN) 2416 return (false); 2417 break; 2418 default: 2419 return (false); 2420 } 2421 return (ccr_aes_check_keylen(csp->csp_cipher_alg, 2422 csp->csp_cipher_klen)); 2423} 2424 2425static int 2426ccr_cipher_mode(const struct crypto_session_params *csp) 2427{ 2428 2429 switch (csp->csp_cipher_alg) { 2430 case CRYPTO_AES_CBC: 2431 return (SCMD_CIPH_MODE_AES_CBC); 2432 case CRYPTO_AES_ICM: 2433 return (SCMD_CIPH_MODE_AES_CTR); 2434 case CRYPTO_AES_NIST_GCM_16: 2435 return (SCMD_CIPH_MODE_AES_GCM); 2436 case CRYPTO_AES_XTS: 2437 return (SCMD_CIPH_MODE_AES_XTS); 2438 case CRYPTO_AES_CCM_16: 2439 return (SCMD_CIPH_MODE_AES_CCM); 2440 default: 2441 return (SCMD_CIPH_MODE_NOP); 2442 } 2443} 2444 2445static int 2446ccr_probesession(device_t dev, const struct crypto_session_params *csp) 2447{ 2448 unsigned int cipher_mode; 2449 2450 if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) != 2451 0) 2452 return (EINVAL); 2453 switch (csp->csp_mode) { 2454 case CSP_MODE_DIGEST: 2455 if (!ccr_auth_supported(csp)) 2456 return (EINVAL); 2457 break; 2458 case CSP_MODE_CIPHER: 2459 if (!ccr_cipher_supported(csp)) 2460 return (EINVAL); 2461 break; 2462 case CSP_MODE_AEAD: 2463 switch (csp->csp_cipher_alg) { 2464 case CRYPTO_AES_NIST_GCM_16: 2465 if (csp->csp_ivlen != AES_GCM_IV_LEN) 2466 return (EINVAL); 2467 if (csp->csp_auth_mlen < 0 || 2468 csp->csp_auth_mlen > AES_GMAC_HASH_LEN) 2469 return (EINVAL); 2470 break; 2471 case CRYPTO_AES_CCM_16: 2472 if (csp->csp_ivlen != AES_CCM_IV_LEN) 2473 return (EINVAL); 2474 if (csp->csp_auth_mlen < 0 || 2475 csp->csp_auth_mlen > AES_CBC_MAC_HASH_LEN) 2476 return (EINVAL); 2477 break; 2478 default: 2479 return (EINVAL); 2480 } 2481 break; 2482 case CSP_MODE_ETA: 2483 if (!ccr_auth_supported(csp) || !ccr_cipher_supported(csp)) 2484 return (EINVAL); 2485 break; 2486 default: 2487 return (EINVAL); 2488 } 2489 2490 if (csp->csp_cipher_klen != 0) { 2491 cipher_mode = ccr_cipher_mode(csp); 2492 if (cipher_mode == SCMD_CIPH_MODE_NOP) 2493 return (EINVAL); 2494 } 2495 2496 return (CRYPTODEV_PROBE_HARDWARE); 2497} 2498 2499/* 2500 * Select an available port with the lowest number of active sessions. 2501 */ 2502static struct ccr_port * 2503ccr_choose_port(struct ccr_softc *sc) 2504{ 2505 struct ccr_port *best, *p; 2506 int i; 2507 2508 mtx_assert(&sc->lock, MA_OWNED); 2509 best = NULL; 2510 for (i = 0; i < nitems(sc->ports); i++) { 2511 p = &sc->ports[i]; 2512 2513 /* Ignore non-existent ports. */ 2514 if (p->rxq == NULL) 2515 continue; 2516 2517 /* 2518 * XXX: Ignore ports whose queues aren't initialized. 2519 * This is racy as the rxq can be destroyed by the 2520 * associated VI detaching. Eventually ccr should use 2521 * dedicated queues. 2522 */ 2523 if (p->rxq->iq.adapter == NULL || p->txq->adapter == NULL) 2524 continue; 2525 2526 if ((sc->port_mask & (1u << i)) == 0) 2527 continue; 2528 2529 if (best == NULL || 2530 p->active_sessions < best->active_sessions) 2531 best = p; 2532 } 2533 return (best); 2534} 2535 2536static void 2537ccr_delete_session(struct ccr_session *s) 2538{ 2539 sglist_free(s->sg_input); 2540 sglist_free(s->sg_output); 2541 sglist_free(s->sg_ulptx); 2542 sglist_free(s->sg_dsgl); 2543 mtx_destroy(&s->lock); 2544} 2545 2546static int 2547ccr_newsession(device_t dev, crypto_session_t cses, 2548 const struct crypto_session_params *csp) 2549{ 2550 struct ccr_softc *sc; 2551 struct ccr_session *s; 2552 struct auth_hash *auth_hash; 2553 unsigned int auth_mode, cipher_mode, mk_size; 2554 unsigned int partial_digest_len; 2555 2556 switch (csp->csp_auth_alg) { 2557 case CRYPTO_SHA1: 2558 case CRYPTO_SHA1_HMAC: 2559 auth_hash = &auth_hash_hmac_sha1; 2560 auth_mode = SCMD_AUTH_MODE_SHA1; 2561 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160; 2562 partial_digest_len = SHA1_HASH_LEN; 2563 break; 2564 case CRYPTO_SHA2_224: 2565 case CRYPTO_SHA2_224_HMAC: 2566 auth_hash = &auth_hash_hmac_sha2_224; 2567 auth_mode = SCMD_AUTH_MODE_SHA224; 2568 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 2569 partial_digest_len = SHA2_256_HASH_LEN; 2570 break; 2571 case CRYPTO_SHA2_256: 2572 case CRYPTO_SHA2_256_HMAC: 2573 auth_hash = &auth_hash_hmac_sha2_256; 2574 auth_mode = SCMD_AUTH_MODE_SHA256; 2575 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 2576 partial_digest_len = SHA2_256_HASH_LEN; 2577 break; 2578 case CRYPTO_SHA2_384: 2579 case CRYPTO_SHA2_384_HMAC: 2580 auth_hash = &auth_hash_hmac_sha2_384; 2581 auth_mode = SCMD_AUTH_MODE_SHA512_384; 2582 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; 2583 partial_digest_len = SHA2_512_HASH_LEN; 2584 break; 2585 case CRYPTO_SHA2_512: 2586 case CRYPTO_SHA2_512_HMAC: 2587 auth_hash = &auth_hash_hmac_sha2_512; 2588 auth_mode = SCMD_AUTH_MODE_SHA512_512; 2589 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; 2590 partial_digest_len = SHA2_512_HASH_LEN; 2591 break; 2592 default: 2593 auth_hash = NULL; 2594 auth_mode = SCMD_AUTH_MODE_NOP; 2595 mk_size = 0; 2596 partial_digest_len = 0; 2597 break; 2598 } 2599 2600 cipher_mode = ccr_cipher_mode(csp); 2601 2602#ifdef INVARIANTS 2603 switch (csp->csp_mode) { 2604 case CSP_MODE_CIPHER: 2605 if (cipher_mode == SCMD_CIPH_MODE_NOP || 2606 cipher_mode == SCMD_CIPH_MODE_AES_GCM || 2607 cipher_mode == SCMD_CIPH_MODE_AES_CCM) 2608 panic("invalid cipher algo"); 2609 break; 2610 case CSP_MODE_DIGEST: 2611 if (auth_mode == SCMD_AUTH_MODE_NOP) 2612 panic("invalid auth algo"); 2613 break; 2614 case CSP_MODE_AEAD: 2615 if (cipher_mode != SCMD_CIPH_MODE_AES_GCM && 2616 cipher_mode != SCMD_CIPH_MODE_AES_CCM) 2617 panic("invalid aead cipher algo"); 2618 if (auth_mode != SCMD_AUTH_MODE_NOP) 2619 panic("invalid aead auth aglo"); 2620 break; 2621 case CSP_MODE_ETA: 2622 if (cipher_mode == SCMD_CIPH_MODE_NOP || 2623 cipher_mode == SCMD_CIPH_MODE_AES_GCM || 2624 cipher_mode == SCMD_CIPH_MODE_AES_CCM) 2625 panic("invalid cipher algo"); 2626 if (auth_mode == SCMD_AUTH_MODE_NOP) 2627 panic("invalid auth algo"); 2628 break; 2629 default: 2630 panic("invalid csp mode"); 2631 } 2632#endif 2633 2634 s = crypto_get_driver_session(cses); 2635 mtx_init(&s->lock, "ccr session", NULL, MTX_DEF); 2636 s->sg_input = sglist_alloc(TX_SGL_SEGS, M_NOWAIT); 2637 s->sg_output = sglist_alloc(TX_SGL_SEGS, M_NOWAIT); 2638 s->sg_ulptx = sglist_alloc(TX_SGL_SEGS, M_NOWAIT); 2639 s->sg_dsgl = sglist_alloc(MAX_RX_PHYS_DSGL_SGE, M_NOWAIT); 2640 if (s->sg_input == NULL || s->sg_output == NULL || 2641 s->sg_ulptx == NULL || s->sg_dsgl == NULL) { 2642 ccr_delete_session(s); 2643 return (ENOMEM); 2644 } 2645 2646 sc = device_get_softc(dev); 2647 2648 mtx_lock(&sc->lock); 2649 if (sc->detaching) { 2650 mtx_unlock(&sc->lock); 2651 ccr_delete_session(s); 2652 return (ENXIO); 2653 } 2654 2655 s->port = ccr_choose_port(sc); 2656 if (s->port == NULL) { 2657 mtx_unlock(&sc->lock); 2658 ccr_delete_session(s); 2659 return (ENXIO); 2660 } 2661 2662 switch (csp->csp_mode) { 2663 case CSP_MODE_AEAD: 2664 if (cipher_mode == SCMD_CIPH_MODE_AES_CCM) 2665 s->mode = CCM; 2666 else 2667 s->mode = GCM; 2668 break; 2669 case CSP_MODE_ETA: 2670 s->mode = ETA; 2671 break; 2672 case CSP_MODE_DIGEST: 2673 if (csp->csp_auth_klen != 0) 2674 s->mode = HMAC; 2675 else 2676 s->mode = HASH; 2677 break; 2678 case CSP_MODE_CIPHER: 2679 s->mode = BLKCIPHER; 2680 break; 2681 } 2682 2683 if (s->mode == GCM) { 2684 if (csp->csp_auth_mlen == 0) 2685 s->gmac.hash_len = AES_GMAC_HASH_LEN; 2686 else 2687 s->gmac.hash_len = csp->csp_auth_mlen; 2688 t4_init_gmac_hash(csp->csp_cipher_key, csp->csp_cipher_klen, 2689 s->gmac.ghash_h); 2690 } else if (s->mode == CCM) { 2691 if (csp->csp_auth_mlen == 0) 2692 s->ccm_mac.hash_len = AES_CBC_MAC_HASH_LEN; 2693 else 2694 s->ccm_mac.hash_len = csp->csp_auth_mlen; 2695 } else if (auth_mode != SCMD_AUTH_MODE_NOP) { 2696 s->hmac.auth_hash = auth_hash; 2697 s->hmac.auth_mode = auth_mode; 2698 s->hmac.mk_size = mk_size; 2699 s->hmac.partial_digest_len = partial_digest_len; 2700 if (csp->csp_auth_mlen == 0) 2701 s->hmac.hash_len = auth_hash->hashsize; 2702 else 2703 s->hmac.hash_len = csp->csp_auth_mlen; 2704 if (csp->csp_auth_key != NULL) 2705 t4_init_hmac_digest(auth_hash, partial_digest_len, 2706 csp->csp_auth_key, csp->csp_auth_klen, 2707 s->hmac.pads); 2708 else 2709 ccr_init_hash_digest(s); 2710 } 2711 if (cipher_mode != SCMD_CIPH_MODE_NOP) { 2712 s->blkcipher.cipher_mode = cipher_mode; 2713 s->blkcipher.iv_len = csp->csp_ivlen; 2714 if (csp->csp_cipher_key != NULL) 2715 ccr_aes_setkey(s, csp->csp_cipher_key, 2716 csp->csp_cipher_klen); 2717 } 2718 2719 s->port->active_sessions++; 2720 mtx_unlock(&sc->lock); 2721 return (0); 2722} 2723 2724static void 2725ccr_freesession(device_t dev, crypto_session_t cses) 2726{ 2727 struct ccr_softc *sc; 2728 struct ccr_session *s; 2729 2730 sc = device_get_softc(dev); 2731 s = crypto_get_driver_session(cses); 2732#ifdef INVARIANTS 2733 if (s->pending != 0) 2734 device_printf(dev, 2735 "session %p freed with %d pending requests\n", s, 2736 s->pending); 2737#endif 2738 mtx_lock(&sc->lock); 2739 s->port->active_sessions--; 2740 mtx_unlock(&sc->lock); 2741 ccr_delete_session(s); 2742} 2743 2744static int 2745ccr_process(device_t dev, struct cryptop *crp, int hint) 2746{ 2747 const struct crypto_session_params *csp; 2748 struct ccr_softc *sc; 2749 struct ccr_session *s; 2750 int error; 2751 2752 csp = crypto_get_params(crp->crp_session); 2753 s = crypto_get_driver_session(crp->crp_session); 2754 sc = device_get_softc(dev); 2755 2756 mtx_lock(&s->lock); 2757 error = ccr_populate_sglist(s->sg_input, &crp->crp_buf); 2758 if (error == 0 && CRYPTO_HAS_OUTPUT_BUFFER(crp)) 2759 error = ccr_populate_sglist(s->sg_output, &crp->crp_obuf); 2760 if (error) { 2761 counter_u64_add(sc->stats_sglist_error, 1); 2762 goto out; 2763 } 2764 2765 switch (s->mode) { 2766 case HASH: 2767 error = ccr_hash(sc, s, crp); 2768 if (error == 0) 2769 counter_u64_add(sc->stats_hash, 1); 2770 break; 2771 case HMAC: 2772 if (crp->crp_auth_key != NULL) 2773 t4_init_hmac_digest(s->hmac.auth_hash, 2774 s->hmac.partial_digest_len, crp->crp_auth_key, 2775 csp->csp_auth_klen, s->hmac.pads); 2776 error = ccr_hash(sc, s, crp); 2777 if (error == 0) 2778 counter_u64_add(sc->stats_hmac, 1); 2779 break; 2780 case BLKCIPHER: 2781 if (crp->crp_cipher_key != NULL) 2782 ccr_aes_setkey(s, crp->crp_cipher_key, 2783 csp->csp_cipher_klen); 2784 error = ccr_blkcipher(sc, s, crp); 2785 if (error == 0) { 2786 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 2787 counter_u64_add(sc->stats_blkcipher_encrypt, 1); 2788 else 2789 counter_u64_add(sc->stats_blkcipher_decrypt, 1); 2790 } 2791 break; 2792 case ETA: 2793 if (crp->crp_auth_key != NULL) 2794 t4_init_hmac_digest(s->hmac.auth_hash, 2795 s->hmac.partial_digest_len, crp->crp_auth_key, 2796 csp->csp_auth_klen, s->hmac.pads); 2797 if (crp->crp_cipher_key != NULL) 2798 ccr_aes_setkey(s, crp->crp_cipher_key, 2799 csp->csp_cipher_klen); 2800 error = ccr_eta(sc, s, crp); 2801 if (error == 0) { 2802 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 2803 counter_u64_add(sc->stats_eta_encrypt, 1); 2804 else 2805 counter_u64_add(sc->stats_eta_decrypt, 1); 2806 } 2807 break; 2808 case GCM: 2809 if (crp->crp_cipher_key != NULL) { 2810 t4_init_gmac_hash(crp->crp_cipher_key, 2811 csp->csp_cipher_klen, s->gmac.ghash_h); 2812 ccr_aes_setkey(s, crp->crp_cipher_key, 2813 csp->csp_cipher_klen); 2814 } 2815 if (crp->crp_payload_length == 0) { 2816 mtx_unlock(&s->lock); 2817 ccr_gcm_soft(s, crp); 2818 return (0); 2819 } 2820 error = ccr_gcm(sc, s, crp); 2821 if (error == EMSGSIZE || error == EFBIG) { 2822 counter_u64_add(sc->stats_sw_fallback, 1); 2823 mtx_unlock(&s->lock); 2824 ccr_gcm_soft(s, crp); 2825 return (0); 2826 } 2827 if (error == 0) { 2828 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 2829 counter_u64_add(sc->stats_gcm_encrypt, 1); 2830 else 2831 counter_u64_add(sc->stats_gcm_decrypt, 1); 2832 } 2833 break; 2834 case CCM: 2835 if (crp->crp_cipher_key != NULL) { 2836 ccr_aes_setkey(s, crp->crp_cipher_key, 2837 csp->csp_cipher_klen); 2838 } 2839 error = ccr_ccm(sc, s, crp); 2840 if (error == EMSGSIZE || error == EFBIG) { 2841 counter_u64_add(sc->stats_sw_fallback, 1); 2842 mtx_unlock(&s->lock); 2843 ccr_ccm_soft(s, crp); 2844 return (0); 2845 } 2846 if (error == 0) { 2847 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 2848 counter_u64_add(sc->stats_ccm_encrypt, 1); 2849 else 2850 counter_u64_add(sc->stats_ccm_decrypt, 1); 2851 } 2852 break; 2853 } 2854 2855 if (error == 0) { 2856#ifdef INVARIANTS 2857 s->pending++; 2858#endif 2859 counter_u64_add(sc->stats_inflight, 1); 2860 counter_u64_add(s->port->stats_queued, 1); 2861 } else 2862 counter_u64_add(sc->stats_process_error, 1); 2863 2864out: 2865 mtx_unlock(&s->lock); 2866 2867 if (error) { 2868 crp->crp_etype = error; 2869 crypto_done(crp); 2870 } 2871 2872 return (0); 2873} 2874 2875static int 2876do_cpl6_fw_pld(struct sge_iq *iq, const struct rss_header *rss, 2877 struct mbuf *m) 2878{ 2879 struct ccr_softc *sc = iq->adapter->ccr_softc; 2880 struct ccr_session *s; 2881 const struct cpl_fw6_pld *cpl; 2882 struct cryptop *crp; 2883 uint32_t status; 2884 int error; 2885 2886 if (m != NULL) 2887 cpl = mtod(m, const void *); 2888 else 2889 cpl = (const void *)(rss + 1); 2890 2891 crp = (struct cryptop *)(uintptr_t)be64toh(cpl->data[1]); 2892 s = crypto_get_driver_session(crp->crp_session); 2893 status = be64toh(cpl->data[0]); 2894 if (CHK_MAC_ERR_BIT(status) || CHK_PAD_ERR_BIT(status)) 2895 error = EBADMSG; 2896 else 2897 error = 0; 2898 2899#ifdef INVARIANTS 2900 mtx_lock(&s->lock); 2901 s->pending--; 2902 mtx_unlock(&s->lock); 2903#endif 2904 counter_u64_add(sc->stats_inflight, -1); 2905 counter_u64_add(s->port->stats_completed, 1); 2906 2907 switch (s->mode) { 2908 case HASH: 2909 case HMAC: 2910 error = ccr_hash_done(sc, s, crp, cpl, error); 2911 break; 2912 case BLKCIPHER: 2913 error = ccr_blkcipher_done(sc, s, crp, cpl, error); 2914 break; 2915 case ETA: 2916 error = ccr_eta_done(sc, s, crp, cpl, error); 2917 break; 2918 case GCM: 2919 error = ccr_gcm_done(sc, s, crp, cpl, error); 2920 break; 2921 case CCM: 2922 error = ccr_ccm_done(sc, s, crp, cpl, error); 2923 break; 2924 } 2925 2926 if (error == EBADMSG) { 2927 if (CHK_MAC_ERR_BIT(status)) 2928 counter_u64_add(sc->stats_mac_error, 1); 2929 if (CHK_PAD_ERR_BIT(status)) 2930 counter_u64_add(sc->stats_pad_error, 1); 2931 } 2932 crp->crp_etype = error; 2933 crypto_done(crp); 2934 m_freem(m); 2935 return (0); 2936} 2937 2938static int 2939ccr_modevent(module_t mod, int cmd, void *arg) 2940{ 2941 2942 switch (cmd) { 2943 case MOD_LOAD: 2944 t4_register_cpl_handler(CPL_FW6_PLD, do_cpl6_fw_pld); 2945 return (0); 2946 case MOD_UNLOAD: 2947 t4_register_cpl_handler(CPL_FW6_PLD, NULL); 2948 return (0); 2949 default: 2950 return (EOPNOTSUPP); 2951 } 2952} 2953 2954static device_method_t ccr_methods[] = { 2955 DEVMETHOD(device_identify, ccr_identify), 2956 DEVMETHOD(device_probe, ccr_probe), 2957 DEVMETHOD(device_attach, ccr_attach), 2958 DEVMETHOD(device_detach, ccr_detach), 2959 2960 DEVMETHOD(cryptodev_probesession, ccr_probesession), 2961 DEVMETHOD(cryptodev_newsession, ccr_newsession), 2962 DEVMETHOD(cryptodev_freesession, ccr_freesession), 2963 DEVMETHOD(cryptodev_process, ccr_process), 2964 2965 DEVMETHOD_END 2966}; 2967 2968static driver_t ccr_driver = { 2969 "ccr", 2970 ccr_methods, 2971 sizeof(struct ccr_softc) 2972}; 2973 2974static devclass_t ccr_devclass; 2975 2976DRIVER_MODULE(ccr, t6nex, ccr_driver, ccr_devclass, ccr_modevent, NULL); 2977MODULE_VERSION(ccr, 1); 2978MODULE_DEPEND(ccr, crypto, 1, 1, 1); 2979MODULE_DEPEND(ccr, t6nex, 1, 1, 1); 2980