1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2017-2018 Chelsio Communications, Inc. 5 * All rights reserved. 6 * Written by: John Baldwin <jhb@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30#include "opt_inet.h" 31 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD$"); 34 35#include <sys/param.h> 36#include <sys/sglist.h> 37#include <sys/socket.h> 38#include <sys/socketvar.h> 39#include <sys/systm.h> 40#include <netinet/in.h> 41#include <netinet/in_pcb.h> 42#include <netinet/tcp_var.h> 43#include <netinet/toecore.h> 44 45#ifdef TCP_OFFLOAD 46#include "common/common.h" 47#include "common/t4_tcb.h" 48#include "crypto/t4_crypto.h" 49#include "tom/t4_tom_l2t.h" 50#include "tom/t4_tom.h" 51 52/* 53 * The TCP sequence number of a CPL_TLS_DATA mbuf is saved here while 54 * the mbuf is in the ulp_pdu_reclaimq. 55 */ 56#define tls_tcp_seq PH_loc.thirtytwo[0] 57 58/* 59 * Handshake lock used for the handshake timer. Having a global lock 60 * is perhaps not ideal, but it avoids having to use callout_drain() 61 * in tls_uninit_toep() which can't block. Also, the timer shouldn't 62 * actually fire for most connections. 63 */ 64static struct mtx tls_handshake_lock; 65 66static void 67t4_set_tls_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask, 68 uint64_t val) 69{ 70 struct adapter *sc = td_adapter(toep->td); 71 72 t4_set_tcb_field(sc, toep->ofld_txq, toep, word, mask, val, 0, 0); 73} 74 75/* TLS and DTLS common routines */ 76bool 77can_tls_offload(struct adapter *sc) 78{ 79 80 return (sc->tt.tls && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS); 81} 82 83int 84tls_tx_key(struct toepcb *toep) 85{ 86 struct tls_ofld_info *tls_ofld = &toep->tls; 87 88 return (tls_ofld->tx_key_addr >= 0); 89} 90 91int 92tls_rx_key(struct toepcb *toep) 93{ 94 struct tls_ofld_info *tls_ofld = &toep->tls; 95 96 return (tls_ofld->rx_key_addr >= 0); 97} 98 99static int 100key_size(struct toepcb *toep) 101{ 102 struct tls_ofld_info *tls_ofld = &toep->tls; 103 104 return ((tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) ? 105 tls_ofld->k_ctx.tx_key_info_size : KEY_IN_DDR_SIZE); 106} 107 108/* Set TLS Key-Id in TCB */ 109static void 110t4_set_tls_keyid(struct toepcb *toep, unsigned int key_id) 111{ 112 113 t4_set_tls_tcb_field(toep, W_TCB_RX_TLS_KEY_TAG, 114 V_TCB_RX_TLS_KEY_TAG(M_TCB_RX_TLS_BUF_TAG), 115 V_TCB_RX_TLS_KEY_TAG(key_id)); 116} 117 118/* Clear TF_RX_QUIESCE to re-enable receive. */ 119static void 120t4_clear_rx_quiesce(struct toepcb *toep) 121{ 122 123 t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0); 124} 125 126static void 127tls_clr_ofld_mode(struct toepcb *toep) 128{ 129 130 tls_stop_handshake_timer(toep); 131 132 /* Operate in PDU extraction mode only. */ 133 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, 134 V_TCB_ULP_RAW(M_TCB_ULP_RAW), 135 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1))); 136 t4_clear_rx_quiesce(toep); 137} 138 139static void 140tls_clr_quiesce(struct toepcb *toep) 141{ 142 143 tls_stop_handshake_timer(toep); 144 t4_clear_rx_quiesce(toep); 145} 146 147/* 148 * Calculate the TLS data expansion size 149 */ 150static int 151tls_expansion_size(struct toepcb *toep, int data_len, int full_pdus_only, 152 unsigned short *pdus_per_ulp) 153{ 154 struct tls_ofld_info *tls_ofld = &toep->tls; 155 struct tls_scmd *scmd = &tls_ofld->scmd0; 156 int expn_size = 0, frag_count = 0, pad_per_pdu = 0, 157 pad_last_pdu = 0, last_frag_size = 0, max_frag_size = 0; 158 int exp_per_pdu = 0; 159 int hdr_len = TLS_HEADER_LENGTH; 160 161 do { 162 max_frag_size = tls_ofld->k_ctx.frag_size; 163 if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) == 164 SCMD_CIPH_MODE_AES_GCM) { 165 frag_count = (data_len / max_frag_size); 166 exp_per_pdu = GCM_TAG_SIZE + AEAD_EXPLICIT_DATA_SIZE + 167 hdr_len; 168 expn_size = frag_count * exp_per_pdu; 169 if (full_pdus_only) { 170 *pdus_per_ulp = data_len / (exp_per_pdu + 171 max_frag_size); 172 if (*pdus_per_ulp > 32) 173 *pdus_per_ulp = 32; 174 else if(!*pdus_per_ulp) 175 *pdus_per_ulp = 1; 176 expn_size = (*pdus_per_ulp) * exp_per_pdu; 177 break; 178 } 179 if ((last_frag_size = data_len % max_frag_size) > 0) { 180 frag_count += 1; 181 expn_size += exp_per_pdu; 182 } 183 break; 184 } else if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) != 185 SCMD_CIPH_MODE_NOP) { 186 /* Calculate the number of fragments we can make */ 187 frag_count = (data_len / max_frag_size); 188 if (frag_count > 0) { 189 pad_per_pdu = (((howmany((max_frag_size + 190 tls_ofld->mac_length), 191 CIPHER_BLOCK_SIZE)) * 192 CIPHER_BLOCK_SIZE) - 193 (max_frag_size + 194 tls_ofld->mac_length)); 195 if (!pad_per_pdu) 196 pad_per_pdu = CIPHER_BLOCK_SIZE; 197 exp_per_pdu = pad_per_pdu + 198 tls_ofld->mac_length + 199 hdr_len + CIPHER_BLOCK_SIZE; 200 expn_size = frag_count * exp_per_pdu; 201 } 202 if (full_pdus_only) { 203 *pdus_per_ulp = data_len / (exp_per_pdu + 204 max_frag_size); 205 if (*pdus_per_ulp > 32) 206 *pdus_per_ulp = 32; 207 else if (!*pdus_per_ulp) 208 *pdus_per_ulp = 1; 209 expn_size = (*pdus_per_ulp) * exp_per_pdu; 210 break; 211 } 212 /* Consider the last fragment */ 213 if ((last_frag_size = data_len % max_frag_size) > 0) { 214 pad_last_pdu = (((howmany((last_frag_size + 215 tls_ofld->mac_length), 216 CIPHER_BLOCK_SIZE)) * 217 CIPHER_BLOCK_SIZE) - 218 (last_frag_size + 219 tls_ofld->mac_length)); 220 if (!pad_last_pdu) 221 pad_last_pdu = CIPHER_BLOCK_SIZE; 222 expn_size += (pad_last_pdu + 223 tls_ofld->mac_length + hdr_len + 224 CIPHER_BLOCK_SIZE); 225 } 226 } 227 } while (0); 228 229 return (expn_size); 230} 231 232/* Copy Key to WR */ 233static void 234tls_copy_tx_key(struct toepcb *toep, void *dst) 235{ 236 struct tls_ofld_info *tls_ofld = &toep->tls; 237 struct ulptx_sc_memrd *sc_memrd; 238 struct ulptx_idata *sc; 239 240 if (tls_ofld->k_ctx.tx_key_info_size <= 0) 241 return; 242 243 if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR) { 244 sc = dst; 245 sc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); 246 sc->len = htobe32(0); 247 sc_memrd = (struct ulptx_sc_memrd *)(sc + 1); 248 sc_memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) | 249 V_ULP_TX_SC_MORE(1) | 250 V_ULPTX_LEN16(tls_ofld->k_ctx.tx_key_info_size >> 4)); 251 sc_memrd->addr = htobe32(tls_ofld->tx_key_addr >> 5); 252 } else if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) { 253 memcpy(dst, &tls_ofld->k_ctx.tx, 254 tls_ofld->k_ctx.tx_key_info_size); 255 } 256} 257 258/* TLS/DTLS content type for CPL SFO */ 259static inline unsigned char 260tls_content_type(unsigned char content_type) 261{ 262 /* 263 * XXX: Shouldn't this map CONTENT_TYPE_APP_DATA to DATA and 264 * default to "CUSTOM" for all other types including 265 * heartbeat? 266 */ 267 switch (content_type) { 268 case CONTENT_TYPE_CCS: 269 return CPL_TX_TLS_SFO_TYPE_CCS; 270 case CONTENT_TYPE_ALERT: 271 return CPL_TX_TLS_SFO_TYPE_ALERT; 272 case CONTENT_TYPE_HANDSHAKE: 273 return CPL_TX_TLS_SFO_TYPE_HANDSHAKE; 274 case CONTENT_TYPE_HEARTBEAT: 275 return CPL_TX_TLS_SFO_TYPE_HEARTBEAT; 276 } 277 return CPL_TX_TLS_SFO_TYPE_DATA; 278} 279 280static unsigned char 281get_cipher_key_size(unsigned int ck_size) 282{ 283 switch (ck_size) { 284 case AES_NOP: /* NOP */ 285 return 15; 286 case AES_128: /* AES128 */ 287 return CH_CK_SIZE_128; 288 case AES_192: /* AES192 */ 289 return CH_CK_SIZE_192; 290 case AES_256: /* AES256 */ 291 return CH_CK_SIZE_256; 292 default: 293 return CH_CK_SIZE_256; 294 } 295} 296 297static unsigned char 298get_mac_key_size(unsigned int mk_size) 299{ 300 switch (mk_size) { 301 case SHA_NOP: /* NOP */ 302 return CH_MK_SIZE_128; 303 case SHA_GHASH: /* GHASH */ 304 case SHA_512: /* SHA512 */ 305 return CH_MK_SIZE_512; 306 case SHA_224: /* SHA2-224 */ 307 return CH_MK_SIZE_192; 308 case SHA_256: /* SHA2-256*/ 309 return CH_MK_SIZE_256; 310 case SHA_384: /* SHA384 */ 311 return CH_MK_SIZE_512; 312 case SHA1: /* SHA1 */ 313 default: 314 return CH_MK_SIZE_160; 315 } 316} 317 318static unsigned int 319get_proto_ver(int proto_ver) 320{ 321 switch (proto_ver) { 322 case TLS1_2_VERSION: 323 return TLS_1_2_VERSION; 324 case TLS1_1_VERSION: 325 return TLS_1_1_VERSION; 326 case DTLS1_2_VERSION: 327 return DTLS_1_2_VERSION; 328 default: 329 return TLS_VERSION_MAX; 330 } 331} 332 333static void 334tls_rxkey_flit1(struct tls_keyctx *kwr, struct tls_key_context *kctx) 335{ 336 337 if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) { 338 kwr->u.rxhdr.ivinsert_to_authinsrt = 339 htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | 340 V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | 341 V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | 342 V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(14ULL) | 343 V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(16ULL) | 344 V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(14ULL) | 345 V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | 346 V_TLS_KEYCTX_TX_WR_AUTHINSRT(16ULL)); 347 kwr->u.rxhdr.ivpresent_to_rxmk_size &= 348 ~(V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1)); 349 kwr->u.rxhdr.authmode_to_rxvalid &= 350 ~(V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1)); 351 } else { 352 kwr->u.rxhdr.ivinsert_to_authinsrt = 353 htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | 354 V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | 355 V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | 356 V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(22ULL) | 357 V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(0ULL) | 358 V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(22ULL) | 359 V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | 360 V_TLS_KEYCTX_TX_WR_AUTHINSRT(0ULL)); 361 } 362} 363 364/* Rx key */ 365static void 366prepare_rxkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx) 367{ 368 unsigned int ck_size = kctx->cipher_secret_size; 369 unsigned int mk_size = kctx->mac_secret_size; 370 int proto_ver = kctx->proto_ver; 371 372 kwr->u.rxhdr.flitcnt_hmacctrl = 373 ((kctx->tx_key_info_size >> 4) << 3) | kctx->hmac_ctrl; 374 375 kwr->u.rxhdr.protover_ciphmode = 376 V_TLS_KEYCTX_TX_WR_PROTOVER(get_proto_ver(proto_ver)) | 377 V_TLS_KEYCTX_TX_WR_CIPHMODE(kctx->state.enc_mode); 378 379 kwr->u.rxhdr.authmode_to_rxvalid = 380 V_TLS_KEYCTX_TX_WR_AUTHMODE(kctx->state.auth_mode) | 381 V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1) | 382 V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(3) | 383 V_TLS_KEYCTX_TX_WR_RXVALID(1); 384 385 kwr->u.rxhdr.ivpresent_to_rxmk_size = 386 V_TLS_KEYCTX_TX_WR_IVPRESENT(0) | 387 V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1) | 388 V_TLS_KEYCTX_TX_WR_RXCK_SIZE(get_cipher_key_size(ck_size)) | 389 V_TLS_KEYCTX_TX_WR_RXMK_SIZE(get_mac_key_size(mk_size)); 390 391 tls_rxkey_flit1(kwr, kctx); 392 393 /* No key reversal for GCM */ 394 if (kctx->state.enc_mode != CH_EVP_CIPH_GCM_MODE) { 395 t4_aes_getdeckey(kwr->keys.edkey, kctx->rx.key, 396 (kctx->cipher_secret_size << 3)); 397 memcpy(kwr->keys.edkey + kctx->cipher_secret_size, 398 kctx->rx.key + kctx->cipher_secret_size, 399 (IPAD_SIZE + OPAD_SIZE)); 400 } else { 401 memcpy(kwr->keys.edkey, kctx->rx.key, 402 (kctx->tx_key_info_size - SALT_SIZE)); 403 memcpy(kwr->u.rxhdr.rxsalt, kctx->rx.salt, SALT_SIZE); 404 } 405} 406 407/* Tx key */ 408static void 409prepare_txkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx) 410{ 411 unsigned int ck_size = kctx->cipher_secret_size; 412 unsigned int mk_size = kctx->mac_secret_size; 413 414 kwr->u.txhdr.ctxlen = 415 (kctx->tx_key_info_size >> 4); 416 kwr->u.txhdr.dualck_to_txvalid = 417 V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1) | 418 V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) | 419 V_TLS_KEYCTX_TX_WR_TXCK_SIZE(get_cipher_key_size(ck_size)) | 420 V_TLS_KEYCTX_TX_WR_TXMK_SIZE(get_mac_key_size(mk_size)) | 421 V_TLS_KEYCTX_TX_WR_TXVALID(1); 422 423 memcpy(kwr->keys.edkey, kctx->tx.key, HDR_KCTX_SIZE); 424 if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) { 425 memcpy(kwr->u.txhdr.txsalt, kctx->tx.salt, SALT_SIZE); 426 kwr->u.txhdr.dualck_to_txvalid &= 427 ~(V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1)); 428 } 429 kwr->u.txhdr.dualck_to_txvalid = htons(kwr->u.txhdr.dualck_to_txvalid); 430} 431 432/* TLS Key memory management */ 433static int 434get_new_keyid(struct toepcb *toep, struct tls_key_context *k_ctx) 435{ 436 struct adapter *sc = td_adapter(toep->td); 437 vmem_addr_t addr; 438 439 if (vmem_alloc(sc->key_map, TLS_KEY_CONTEXT_SZ, M_NOWAIT | M_FIRSTFIT, 440 &addr) != 0) 441 return (-1); 442 443 return (addr); 444} 445 446static void 447free_keyid(struct toepcb *toep, int keyid) 448{ 449 struct adapter *sc = td_adapter(toep->td); 450 451 vmem_free(sc->key_map, keyid, TLS_KEY_CONTEXT_SZ); 452} 453 454static void 455clear_tls_keyid(struct toepcb *toep) 456{ 457 struct tls_ofld_info *tls_ofld = &toep->tls; 458 459 if (tls_ofld->rx_key_addr >= 0) { 460 free_keyid(toep, tls_ofld->rx_key_addr); 461 tls_ofld->rx_key_addr = -1; 462 } 463 if (tls_ofld->tx_key_addr >= 0) { 464 free_keyid(toep, tls_ofld->tx_key_addr); 465 tls_ofld->tx_key_addr = -1; 466 } 467} 468 469static int 470get_keyid(struct tls_ofld_info *tls_ofld, unsigned int ops) 471{ 472 return (ops & KEY_WRITE_RX ? tls_ofld->rx_key_addr : 473 ((ops & KEY_WRITE_TX) ? tls_ofld->tx_key_addr : -1)); 474} 475 476static int 477get_tp_plen_max(struct tls_ofld_info *tls_ofld) 478{ 479 int plen = ((min(3*4096, TP_TX_PG_SZ))/1448) * 1448; 480 481 return (tls_ofld->k_ctx.frag_size <= 8192 ? plen : FC_TP_PLEN_MAX); 482} 483 484/* Send request to get the key-id */ 485static int 486tls_program_key_id(struct toepcb *toep, struct tls_key_context *k_ctx) 487{ 488 struct tls_ofld_info *tls_ofld = &toep->tls; 489 struct adapter *sc = td_adapter(toep->td); 490 struct ofld_tx_sdesc *txsd; 491 int kwrlen, kctxlen, keyid, len; 492 struct wrqe *wr; 493 struct tls_key_req *kwr; 494 struct tls_keyctx *kctx; 495 496 kwrlen = sizeof(*kwr); 497 kctxlen = roundup2(sizeof(*kctx), 32); 498 len = roundup2(kwrlen + kctxlen, 16); 499 500 if (toep->txsd_avail == 0) 501 return (EAGAIN); 502 503 /* Dont initialize key for re-neg */ 504 if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) { 505 if ((keyid = get_new_keyid(toep, k_ctx)) < 0) { 506 return (ENOSPC); 507 } 508 } else { 509 keyid = get_keyid(tls_ofld, k_ctx->l_p_key); 510 } 511 512 wr = alloc_wrqe(len, toep->ofld_txq); 513 if (wr == NULL) { 514 free_keyid(toep, keyid); 515 return (ENOMEM); 516 } 517 kwr = wrtod(wr); 518 memset(kwr, 0, kwrlen); 519 520 kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) | F_FW_WR_COMPL | 521 F_FW_WR_ATOMIC); 522 kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(len, 16)) | 523 V_FW_WR_FLOWID(toep->tid)); 524 kwr->protocol = get_proto_ver(k_ctx->proto_ver); 525 kwr->mfs = htons(k_ctx->frag_size); 526 kwr->reneg_to_write_rx = k_ctx->l_p_key; 527 528 /* master command */ 529 kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) | 530 V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1)); 531 kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(kctxlen >> 5)); 532 kwr->len16 = htobe32((toep->tid << 8) | 533 DIV_ROUND_UP(len - sizeof(struct work_request_hdr), 16)); 534 kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5)); 535 536 /* sub command */ 537 kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); 538 kwr->sc_len = htobe32(kctxlen); 539 540 kctx = (struct tls_keyctx *)(kwr + 1); 541 memset(kctx, 0, kctxlen); 542 543 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) { 544 tls_ofld->tx_key_addr = keyid; 545 prepare_txkey_wr(kctx, k_ctx); 546 } else if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { 547 tls_ofld->rx_key_addr = keyid; 548 prepare_rxkey_wr(kctx, k_ctx); 549 } 550 551 txsd = &toep->txsd[toep->txsd_pidx]; 552 txsd->tx_credits = DIV_ROUND_UP(len, 16); 553 txsd->plen = 0; 554 toep->tx_credits -= txsd->tx_credits; 555 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) 556 toep->txsd_pidx = 0; 557 toep->txsd_avail--; 558 559 t4_wrq_tx(sc, wr); 560 561 return (0); 562} 563 564/* Store a key received from SSL in DDR. */ 565static int 566program_key_context(struct tcpcb *tp, struct toepcb *toep, 567 struct tls_key_context *uk_ctx) 568{ 569 struct adapter *sc = td_adapter(toep->td); 570 struct tls_ofld_info *tls_ofld = &toep->tls; 571 struct tls_key_context *k_ctx; 572 int error, key_offset; 573 574 if (tp->t_state != TCPS_ESTABLISHED) { 575 /* 576 * XXX: Matches Linux driver, but not sure this is a 577 * very appropriate error. 578 */ 579 return (ENOENT); 580 } 581 582 /* Stop timer on handshake completion */ 583 tls_stop_handshake_timer(toep); 584 585 toep->flags &= ~TPF_FORCE_CREDITS; 586 587 CTR4(KTR_CXGBE, "%s: tid %d %s proto_ver %#x", __func__, toep->tid, 588 G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX ? "KEY_WRITE_RX" : 589 "KEY_WRITE_TX", uk_ctx->proto_ver); 590 591 if (G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX && 592 ulp_mode(toep) != ULP_MODE_TLS) 593 return (EOPNOTSUPP); 594 595 /* Don't copy the 'tx' and 'rx' fields. */ 596 k_ctx = &tls_ofld->k_ctx; 597 memcpy(&k_ctx->l_p_key, &uk_ctx->l_p_key, 598 sizeof(*k_ctx) - offsetof(struct tls_key_context, l_p_key)); 599 600 /* TLS version != 1.1 and !1.2 OR DTLS != 1.2 */ 601 if (get_proto_ver(k_ctx->proto_ver) > DTLS_1_2_VERSION) { 602 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { 603 tls_ofld->rx_key_addr = -1; 604 t4_clear_rx_quiesce(toep); 605 } else { 606 tls_ofld->tx_key_addr = -1; 607 } 608 return (0); 609 } 610 611 if (k_ctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) { 612 k_ctx->iv_size = 4; 613 k_ctx->mac_first = 0; 614 k_ctx->hmac_ctrl = 0; 615 } else { 616 k_ctx->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */ 617 k_ctx->mac_first = 1; 618 } 619 620 tls_ofld->scmd0.seqno_numivs = 621 (V_SCMD_SEQ_NO_CTRL(3) | 622 V_SCMD_PROTO_VERSION(get_proto_ver(k_ctx->proto_ver)) | 623 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | 624 V_SCMD_CIPH_AUTH_SEQ_CTRL((k_ctx->mac_first == 0)) | 625 V_SCMD_CIPH_MODE(k_ctx->state.enc_mode) | 626 V_SCMD_AUTH_MODE(k_ctx->state.auth_mode) | 627 V_SCMD_HMAC_CTRL(k_ctx->hmac_ctrl) | 628 V_SCMD_IV_SIZE(k_ctx->iv_size)); 629 630 tls_ofld->scmd0.ivgen_hdrlen = 631 (V_SCMD_IV_GEN_CTRL(k_ctx->iv_ctrl) | 632 V_SCMD_KEY_CTX_INLINE(0) | 633 V_SCMD_TLS_FRAG_ENABLE(1)); 634 635 tls_ofld->mac_length = k_ctx->mac_secret_size; 636 637 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { 638 k_ctx->rx = uk_ctx->rx; 639 /* Dont initialize key for re-neg */ 640 if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) 641 tls_ofld->rx_key_addr = -1; 642 } else { 643 k_ctx->tx = uk_ctx->tx; 644 /* Dont initialize key for re-neg */ 645 if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) 646 tls_ofld->tx_key_addr = -1; 647 } 648 649 /* Flush pending data before new Tx key becomes active */ 650 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) { 651 struct sockbuf *sb; 652 653 /* XXX: This might not drain everything. */ 654 t4_push_frames(sc, toep, 0); 655 sb = &toep->inp->inp_socket->so_snd; 656 SOCKBUF_LOCK(sb); 657 658 /* XXX: This asserts that everything has been pushed. */ 659 MPASS(sb->sb_sndptr == NULL || sb->sb_sndptr->m_next == NULL); 660 sb->sb_sndptr = NULL; 661 tls_ofld->sb_off = sbavail(sb); 662 SOCKBUF_UNLOCK(sb); 663 tls_ofld->tx_seq_no = 0; 664 } 665 666 if ((G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) || 667 (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR)) { 668 error = tls_program_key_id(toep, k_ctx); 669 if (error) { 670 /* XXX: Only clear quiesce for KEY_WRITE_RX? */ 671 t4_clear_rx_quiesce(toep); 672 return (error); 673 } 674 } 675 676 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { 677 /* 678 * RX key tags are an index into the key portion of MA 679 * memory stored as an offset from the base address in 680 * units of 64 bytes. 681 */ 682 key_offset = tls_ofld->rx_key_addr - sc->vres.key.start; 683 t4_set_tls_keyid(toep, key_offset / 64); 684 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, 685 V_TCB_ULP_RAW(M_TCB_ULP_RAW), 686 V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) | 687 V_TF_TLS_CONTROL(1) | 688 V_TF_TLS_ACTIVE(1) | 689 V_TF_TLS_ENABLE(1)))); 690 t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ, 691 V_TCB_TLS_SEQ(M_TCB_TLS_SEQ), 692 V_TCB_TLS_SEQ(0)); 693 t4_clear_rx_quiesce(toep); 694 695 toep->flags |= TPF_TLS_RECEIVE; 696 } else { 697 unsigned short pdus_per_ulp; 698 699 if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) 700 tls_ofld->tx_key_addr = 1; 701 702 tls_ofld->fcplenmax = get_tp_plen_max(tls_ofld); 703 tls_ofld->expn_per_ulp = tls_expansion_size(toep, 704 tls_ofld->fcplenmax, 1, &pdus_per_ulp); 705 tls_ofld->pdus_per_ulp = pdus_per_ulp; 706 tls_ofld->adjusted_plen = tls_ofld->pdus_per_ulp * 707 ((tls_ofld->expn_per_ulp/tls_ofld->pdus_per_ulp) + 708 tls_ofld->k_ctx.frag_size); 709 } 710 711 return (0); 712} 713 714/* 715 * In some cases a client connection can hang without sending the 716 * ServerHelloDone message from the NIC to the host. Send a dummy 717 * RX_DATA_ACK with RX_MODULATE to unstick the connection. 718 */ 719static void 720tls_send_handshake_ack(void *arg) 721{ 722 struct toepcb *toep = arg; 723 struct tls_ofld_info *tls_ofld = &toep->tls; 724 struct adapter *sc = td_adapter(toep->td); 725 726 /* 727 * XXX: Does not have the t4_get_tcb() checks to refine the 728 * workaround. 729 */ 730 callout_schedule(&tls_ofld->handshake_timer, TLS_SRV_HELLO_RD_TM * hz); 731 732 CTR2(KTR_CXGBE, "%s: tid %d sending RX_DATA_ACK", __func__, toep->tid); 733 send_rx_modulate(sc, toep); 734} 735 736static void 737tls_start_handshake_timer(struct toepcb *toep) 738{ 739 struct tls_ofld_info *tls_ofld = &toep->tls; 740 741 mtx_lock(&tls_handshake_lock); 742 callout_reset(&tls_ofld->handshake_timer, TLS_SRV_HELLO_BKOFF_TM * hz, 743 tls_send_handshake_ack, toep); 744 mtx_unlock(&tls_handshake_lock); 745} 746 747void 748tls_stop_handshake_timer(struct toepcb *toep) 749{ 750 struct tls_ofld_info *tls_ofld = &toep->tls; 751 752 mtx_lock(&tls_handshake_lock); 753 callout_stop(&tls_ofld->handshake_timer); 754 mtx_unlock(&tls_handshake_lock); 755} 756 757int 758t4_ctloutput_tls(struct socket *so, struct sockopt *sopt) 759{ 760 struct tls_key_context uk_ctx; 761 struct inpcb *inp; 762 struct tcpcb *tp; 763 struct toepcb *toep; 764 int error, optval; 765 766 error = 0; 767 if (sopt->sopt_dir == SOPT_SET && 768 sopt->sopt_name == TCP_TLSOM_SET_TLS_CONTEXT) { 769 error = sooptcopyin(sopt, &uk_ctx, sizeof(uk_ctx), 770 sizeof(uk_ctx)); 771 if (error) 772 return (error); 773 } 774 775 inp = sotoinpcb(so); 776 KASSERT(inp != NULL, ("tcp_ctloutput: inp == NULL")); 777 INP_WLOCK(inp); 778 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { 779 INP_WUNLOCK(inp); 780 return (ECONNRESET); 781 } 782 tp = intotcpcb(inp); 783 toep = tp->t_toe; 784 switch (sopt->sopt_dir) { 785 case SOPT_SET: 786 switch (sopt->sopt_name) { 787 case TCP_TLSOM_SET_TLS_CONTEXT: 788 error = program_key_context(tp, toep, &uk_ctx); 789 INP_WUNLOCK(inp); 790 break; 791 case TCP_TLSOM_CLR_TLS_TOM: 792 if (ulp_mode(toep) == ULP_MODE_TLS) { 793 CTR2(KTR_CXGBE, "%s: tid %d CLR_TLS_TOM", 794 __func__, toep->tid); 795 tls_clr_ofld_mode(toep); 796 } else 797 error = EOPNOTSUPP; 798 INP_WUNLOCK(inp); 799 break; 800 case TCP_TLSOM_CLR_QUIES: 801 if (ulp_mode(toep) == ULP_MODE_TLS) { 802 CTR2(KTR_CXGBE, "%s: tid %d CLR_QUIES", 803 __func__, toep->tid); 804 tls_clr_quiesce(toep); 805 } else 806 error = EOPNOTSUPP; 807 INP_WUNLOCK(inp); 808 break; 809 default: 810 INP_WUNLOCK(inp); 811 error = EOPNOTSUPP; 812 break; 813 } 814 break; 815 case SOPT_GET: 816 switch (sopt->sopt_name) { 817 case TCP_TLSOM_GET_TLS_TOM: 818 /* 819 * TLS TX is permitted on any TOE socket, but 820 * TLS RX requires a TLS ULP mode. 821 */ 822 optval = TLS_TOM_NONE; 823 if (can_tls_offload(td_adapter(toep->td))) { 824 switch (ulp_mode(toep)) { 825 case ULP_MODE_NONE: 826 case ULP_MODE_TCPDDP: 827 optval = TLS_TOM_TXONLY; 828 break; 829 case ULP_MODE_TLS: 830 optval = TLS_TOM_BOTH; 831 break; 832 } 833 } 834 CTR3(KTR_CXGBE, "%s: tid %d GET_TLS_TOM = %d", 835 __func__, toep->tid, optval); 836 INP_WUNLOCK(inp); 837 error = sooptcopyout(sopt, &optval, sizeof(optval)); 838 break; 839 default: 840 INP_WUNLOCK(inp); 841 error = EOPNOTSUPP; 842 break; 843 } 844 break; 845 } 846 return (error); 847} 848 849void 850tls_init_toep(struct toepcb *toep) 851{ 852 struct tls_ofld_info *tls_ofld = &toep->tls; 853 854 tls_ofld->key_location = TLS_SFO_WR_CONTEXTLOC_DDR; 855 tls_ofld->rx_key_addr = -1; 856 tls_ofld->tx_key_addr = -1; 857 if (ulp_mode(toep) == ULP_MODE_TLS) 858 callout_init_mtx(&tls_ofld->handshake_timer, 859 &tls_handshake_lock, 0); 860} 861 862void 863tls_establish(struct toepcb *toep) 864{ 865 866 /* 867 * Enable PDU extraction. 868 * 869 * XXX: Supposedly this should be done by the firmware when 870 * the ULP_MODE FLOWC parameter is set in send_flowc_wr(), but 871 * in practice this seems to be required. 872 */ 873 CTR2(KTR_CXGBE, "%s: tid %d setting TLS_ENABLE", __func__, toep->tid); 874 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW), 875 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1))); 876 877 toep->flags |= TPF_FORCE_CREDITS; 878 879 tls_start_handshake_timer(toep); 880} 881 882void 883tls_uninit_toep(struct toepcb *toep) 884{ 885 886 if (ulp_mode(toep) == ULP_MODE_TLS) 887 tls_stop_handshake_timer(toep); 888 clear_tls_keyid(toep); 889} 890 891#define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16) 892#define MIN_OFLD_TLSTX_CREDITS(toep) \ 893 (howmany(sizeof(struct fw_tlstx_data_wr) + \ 894 sizeof(struct cpl_tx_tls_sfo) + key_size((toep)) + \ 895 CIPHER_BLOCK_SIZE + 1, 16)) 896 897static inline u_int 898max_imm_tls_space(int tx_credits) 899{ 900 const int n = 2; /* Use only up to 2 desc for imm. data WR */ 901 int space; 902 903 KASSERT(tx_credits >= 0 && 904 tx_credits <= MAX_OFLD_TX_CREDITS, 905 ("%s: %d credits", __func__, tx_credits)); 906 907 if (tx_credits >= (n * EQ_ESIZE) / 16) 908 space = (n * EQ_ESIZE); 909 else 910 space = tx_credits * 16; 911 return (space); 912} 913 914static int 915count_mbuf_segs(struct mbuf *m, int skip, int len, int *max_nsegs_1mbufp) 916{ 917 int max_nsegs_1mbuf, n, nsegs; 918 919 while (skip >= m->m_len) { 920 skip -= m->m_len; 921 m = m->m_next; 922 } 923 924 nsegs = 0; 925 max_nsegs_1mbuf = 0; 926 while (len > 0) { 927 n = sglist_count(mtod(m, char *) + skip, m->m_len - skip); 928 if (n > max_nsegs_1mbuf) 929 max_nsegs_1mbuf = n; 930 nsegs += n; 931 len -= m->m_len - skip; 932 skip = 0; 933 m = m->m_next; 934 } 935 *max_nsegs_1mbufp = max_nsegs_1mbuf; 936 return (nsegs); 937} 938 939static void 940write_tlstx_wr(struct fw_tlstx_data_wr *txwr, struct toepcb *toep, 941 unsigned int immdlen, unsigned int plen, unsigned int expn, 942 unsigned int pdus, uint8_t credits, int shove, int imm_ivs) 943{ 944 struct tls_ofld_info *tls_ofld = &toep->tls; 945 unsigned int len = plen + expn; 946 947 txwr->op_to_immdlen = htobe32(V_WR_OP(FW_TLSTX_DATA_WR) | 948 V_FW_TLSTX_DATA_WR_COMPL(1) | 949 V_FW_TLSTX_DATA_WR_IMMDLEN(immdlen)); 950 txwr->flowid_len16 = htobe32(V_FW_TLSTX_DATA_WR_FLOWID(toep->tid) | 951 V_FW_TLSTX_DATA_WR_LEN16(credits)); 952 txwr->plen = htobe32(len); 953 txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ULP_MODE_TLS) | 954 V_TX_URG(0) | /* F_T6_TX_FORCE | */ V_TX_SHOVE(shove)); 955 txwr->ctxloc_to_exp = htobe32(V_FW_TLSTX_DATA_WR_NUMIVS(pdus) | 956 V_FW_TLSTX_DATA_WR_EXP(expn) | 957 V_FW_TLSTX_DATA_WR_CTXLOC(tls_ofld->key_location) | 958 V_FW_TLSTX_DATA_WR_IVDSGL(!imm_ivs) | 959 V_FW_TLSTX_DATA_WR_KEYSIZE(tls_ofld->k_ctx.tx_key_info_size >> 4)); 960 txwr->mfs = htobe16(tls_ofld->k_ctx.frag_size); 961 txwr->adjustedplen_pkd = htobe16( 962 V_FW_TLSTX_DATA_WR_ADJUSTEDPLEN(tls_ofld->adjusted_plen)); 963 txwr->expinplenmax_pkd = htobe16( 964 V_FW_TLSTX_DATA_WR_EXPINPLENMAX(tls_ofld->expn_per_ulp)); 965 txwr->pdusinplenmax_pkd = htobe16( 966 V_FW_TLSTX_DATA_WR_PDUSINPLENMAX(tls_ofld->pdus_per_ulp)); 967} 968 969static void 970write_tlstx_cpl(struct cpl_tx_tls_sfo *cpl, struct toepcb *toep, 971 struct tls_hdr *tls_hdr, unsigned int plen, unsigned int pdus) 972{ 973 struct tls_ofld_info *tls_ofld = &toep->tls; 974 int data_type, seglen; 975 976 if (plen < tls_ofld->k_ctx.frag_size) 977 seglen = plen; 978 else 979 seglen = tls_ofld->k_ctx.frag_size; 980 data_type = tls_content_type(tls_hdr->type); 981 cpl->op_to_seg_len = htobe32(V_CPL_TX_TLS_SFO_OPCODE(CPL_TX_TLS_SFO) | 982 V_CPL_TX_TLS_SFO_DATA_TYPE(data_type) | 983 V_CPL_TX_TLS_SFO_CPL_LEN(2) | V_CPL_TX_TLS_SFO_SEG_LEN(seglen)); 984 cpl->pld_len = htobe32(plen); 985 if (data_type == CPL_TX_TLS_SFO_TYPE_HEARTBEAT) 986 cpl->type_protover = htobe32( 987 V_CPL_TX_TLS_SFO_TYPE(tls_hdr->type)); 988 cpl->seqno_numivs = htobe32(tls_ofld->scmd0.seqno_numivs | 989 V_SCMD_NUM_IVS(pdus)); 990 cpl->ivgen_hdrlen = htobe32(tls_ofld->scmd0.ivgen_hdrlen); 991 cpl->scmd1 = htobe64(tls_ofld->tx_seq_no); 992 tls_ofld->tx_seq_no += pdus; 993} 994 995/* 996 * Similar to write_tx_sgl() except that it accepts an optional 997 * trailer buffer for IVs. 998 */ 999static void 1000write_tlstx_sgl(void *dst, struct mbuf *start, int skip, int plen, 1001 void *iv_buffer, int iv_len, int nsegs, int n) 1002{ 1003 struct mbuf *m; 1004 struct ulptx_sgl *usgl = dst; 1005 int i, j, rc; 1006 struct sglist sg; 1007 struct sglist_seg segs[n]; 1008 1009 KASSERT(nsegs > 0, ("%s: nsegs 0", __func__)); 1010 1011 sglist_init(&sg, n, segs); 1012 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 1013 V_ULPTX_NSGE(nsegs)); 1014 1015 for (m = start; skip >= m->m_len; m = m->m_next) 1016 skip -= m->m_len; 1017 1018 i = -1; 1019 for (m = start; plen > 0; m = m->m_next) { 1020 rc = sglist_append(&sg, mtod(m, char *) + skip, 1021 m->m_len - skip); 1022 if (__predict_false(rc != 0)) 1023 panic("%s: sglist_append %d", __func__, rc); 1024 plen -= m->m_len - skip; 1025 skip = 0; 1026 1027 for (j = 0; j < sg.sg_nseg; i++, j++) { 1028 if (i < 0) { 1029 usgl->len0 = htobe32(segs[j].ss_len); 1030 usgl->addr0 = htobe64(segs[j].ss_paddr); 1031 } else { 1032 usgl->sge[i / 2].len[i & 1] = 1033 htobe32(segs[j].ss_len); 1034 usgl->sge[i / 2].addr[i & 1] = 1035 htobe64(segs[j].ss_paddr); 1036 } 1037#ifdef INVARIANTS 1038 nsegs--; 1039#endif 1040 } 1041 sglist_reset(&sg); 1042 } 1043 if (iv_buffer != NULL) { 1044 rc = sglist_append(&sg, iv_buffer, iv_len); 1045 if (__predict_false(rc != 0)) 1046 panic("%s: sglist_append %d", __func__, rc); 1047 1048 for (j = 0; j < sg.sg_nseg; i++, j++) { 1049 if (i < 0) { 1050 usgl->len0 = htobe32(segs[j].ss_len); 1051 usgl->addr0 = htobe64(segs[j].ss_paddr); 1052 } else { 1053 usgl->sge[i / 2].len[i & 1] = 1054 htobe32(segs[j].ss_len); 1055 usgl->sge[i / 2].addr[i & 1] = 1056 htobe64(segs[j].ss_paddr); 1057 } 1058#ifdef INVARIANTS 1059 nsegs--; 1060#endif 1061 } 1062 } 1063 if (i & 1) 1064 usgl->sge[i / 2].len[1] = htobe32(0); 1065 KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, iv_buffer %p", 1066 __func__, nsegs, start, iv_buffer)); 1067} 1068 1069/* 1070 * Similar to t4_push_frames() but handles TLS sockets when TLS offload 1071 * is enabled. Rather than transmitting bulk data, the socket buffer 1072 * contains TLS records. The work request requires a full TLS record, 1073 * so batch mbufs up until a full TLS record is seen. This requires 1074 * reading the TLS header out of the start of each record to determine 1075 * its length. 1076 */ 1077void 1078t4_push_tls_records(struct adapter *sc, struct toepcb *toep, int drop) 1079{ 1080 struct tls_hdr thdr; 1081 struct mbuf *sndptr; 1082 struct fw_tlstx_data_wr *txwr; 1083 struct cpl_tx_tls_sfo *cpl; 1084 struct wrqe *wr; 1085 u_int plen, nsegs, credits, space, max_nsegs_1mbuf, wr_len; 1086 u_int expn_size, iv_len, pdus, sndptroff; 1087 struct tls_ofld_info *tls_ofld = &toep->tls; 1088 struct inpcb *inp = toep->inp; 1089 struct tcpcb *tp = intotcpcb(inp); 1090 struct socket *so = inp->inp_socket; 1091 struct sockbuf *sb = &so->so_snd; 1092 int tls_size, tx_credits, shove, /* compl,*/ sowwakeup; 1093 struct ofld_tx_sdesc *txsd; 1094 bool imm_ivs, imm_payload; 1095 void *iv_buffer, *iv_dst, *buf; 1096 1097 INP_WLOCK_ASSERT(inp); 1098 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 1099 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); 1100 1101 KASSERT(ulp_mode(toep) == ULP_MODE_NONE || 1102 ulp_mode(toep) == ULP_MODE_TCPDDP || ulp_mode(toep) == ULP_MODE_TLS, 1103 ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep)); 1104 KASSERT(tls_tx_key(toep), 1105 ("%s: TX key not set for toep %p", __func__, toep)); 1106 1107#ifdef VERBOSE_TRACES 1108 CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d", 1109 __func__, toep->tid, toep->flags, tp->t_flags); 1110#endif 1111 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) 1112 return; 1113 1114#ifdef RATELIMIT 1115 if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) && 1116 (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) { 1117 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED; 1118 } 1119#endif 1120 1121 /* 1122 * This function doesn't resume by itself. Someone else must clear the 1123 * flag and call this function. 1124 */ 1125 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { 1126 KASSERT(drop == 0, 1127 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); 1128 return; 1129 } 1130 1131 txsd = &toep->txsd[toep->txsd_pidx]; 1132 for (;;) { 1133 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); 1134 space = max_imm_tls_space(tx_credits); 1135 wr_len = sizeof(struct fw_tlstx_data_wr) + 1136 sizeof(struct cpl_tx_tls_sfo) + key_size(toep); 1137 if (wr_len + CIPHER_BLOCK_SIZE + 1 > space) { 1138#ifdef VERBOSE_TRACES 1139 CTR5(KTR_CXGBE, 1140 "%s: tid %d tx_credits %d min_wr %d space %d", 1141 __func__, toep->tid, tx_credits, wr_len + 1142 CIPHER_BLOCK_SIZE + 1, space); 1143#endif 1144 return; 1145 } 1146 1147 SOCKBUF_LOCK(sb); 1148 sowwakeup = drop; 1149 if (drop) { 1150 sbdrop_locked(sb, drop); 1151 MPASS(tls_ofld->sb_off >= drop); 1152 tls_ofld->sb_off -= drop; 1153 drop = 0; 1154 } 1155 1156 /* 1157 * Send a FIN if requested, but only if there's no 1158 * more data to send. 1159 */ 1160 if (sbavail(sb) == tls_ofld->sb_off && 1161 toep->flags & TPF_SEND_FIN) { 1162 if (sowwakeup) 1163 sowwakeup_locked(so); 1164 else 1165 SOCKBUF_UNLOCK(sb); 1166 SOCKBUF_UNLOCK_ASSERT(sb); 1167 t4_close_conn(sc, toep); 1168 return; 1169 } 1170 1171 if (sbavail(sb) < tls_ofld->sb_off + TLS_HEADER_LENGTH) { 1172 /* 1173 * A full TLS header is not yet queued, stop 1174 * for now until more data is added to the 1175 * socket buffer. However, if the connection 1176 * has been closed, we will never get the rest 1177 * of the header so just discard the partial 1178 * header and close the connection. 1179 */ 1180#ifdef VERBOSE_TRACES 1181 CTR5(KTR_CXGBE, "%s: tid %d sbavail %d sb_off %d%s", 1182 __func__, toep->tid, sbavail(sb), tls_ofld->sb_off, 1183 toep->flags & TPF_SEND_FIN ? "" : " SEND_FIN"); 1184#endif 1185 if (sowwakeup) 1186 sowwakeup_locked(so); 1187 else 1188 SOCKBUF_UNLOCK(sb); 1189 SOCKBUF_UNLOCK_ASSERT(sb); 1190 if (toep->flags & TPF_SEND_FIN) 1191 t4_close_conn(sc, toep); 1192 return; 1193 } 1194 1195 /* Read the header of the next TLS record. */ 1196 sndptr = sbsndmbuf(sb, tls_ofld->sb_off, &sndptroff); 1197 MPASS(!IS_AIOTX_MBUF(sndptr)); 1198 m_copydata(sndptr, sndptroff, sizeof(thdr), (caddr_t)&thdr); 1199 tls_size = htons(thdr.length); 1200 plen = TLS_HEADER_LENGTH + tls_size; 1201 pdus = howmany(tls_size, tls_ofld->k_ctx.frag_size); 1202 iv_len = pdus * CIPHER_BLOCK_SIZE; 1203 1204 if (sbavail(sb) < tls_ofld->sb_off + plen) { 1205 /* 1206 * The full TLS record is not yet queued, stop 1207 * for now until more data is added to the 1208 * socket buffer. However, if the connection 1209 * has been closed, we will never get the rest 1210 * of the record so just discard the partial 1211 * record and close the connection. 1212 */ 1213#ifdef VERBOSE_TRACES 1214 CTR6(KTR_CXGBE, 1215 "%s: tid %d sbavail %d sb_off %d plen %d%s", 1216 __func__, toep->tid, sbavail(sb), tls_ofld->sb_off, 1217 plen, toep->flags & TPF_SEND_FIN ? "" : 1218 " SEND_FIN"); 1219#endif 1220 if (sowwakeup) 1221 sowwakeup_locked(so); 1222 else 1223 SOCKBUF_UNLOCK(sb); 1224 SOCKBUF_UNLOCK_ASSERT(sb); 1225 if (toep->flags & TPF_SEND_FIN) 1226 t4_close_conn(sc, toep); 1227 return; 1228 } 1229 1230 /* Shove if there is no additional data pending. */ 1231 shove = (sbavail(sb) == tls_ofld->sb_off + plen) && 1232 !(tp->t_flags & TF_MORETOCOME); 1233 1234 if (sb->sb_flags & SB_AUTOSIZE && 1235 V_tcp_do_autosndbuf && 1236 sb->sb_hiwat < V_tcp_autosndbuf_max && 1237 sbused(sb) >= sb->sb_hiwat * 7 / 8) { 1238 int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc, 1239 V_tcp_autosndbuf_max); 1240 1241 if (!sbreserve_locked(sb, newsize, so, NULL)) 1242 sb->sb_flags &= ~SB_AUTOSIZE; 1243 else 1244 sowwakeup = 1; /* room available */ 1245 } 1246 if (sowwakeup) 1247 sowwakeup_locked(so); 1248 else 1249 SOCKBUF_UNLOCK(sb); 1250 SOCKBUF_UNLOCK_ASSERT(sb); 1251 1252 if (__predict_false(toep->flags & TPF_FIN_SENT)) 1253 panic("%s: excess tx.", __func__); 1254 1255 /* Determine whether to use immediate vs SGL. */ 1256 imm_payload = false; 1257 imm_ivs = false; 1258 if (wr_len + iv_len <= space) { 1259 imm_ivs = true; 1260 wr_len += iv_len; 1261 if (wr_len + tls_size <= space) { 1262 wr_len += tls_size; 1263 imm_payload = true; 1264 } 1265 } 1266 1267 /* Allocate space for IVs if needed. */ 1268 if (!imm_ivs) { 1269 iv_buffer = malloc(iv_len, M_CXGBE, M_NOWAIT); 1270 if (iv_buffer == NULL) { 1271 /* 1272 * XXX: How to restart this? 1273 */ 1274 if (sowwakeup) 1275 sowwakeup_locked(so); 1276 else 1277 SOCKBUF_UNLOCK(sb); 1278 SOCKBUF_UNLOCK_ASSERT(sb); 1279 CTR3(KTR_CXGBE, 1280 "%s: tid %d failed to alloc IV space len %d", 1281 __func__, toep->tid, iv_len); 1282 return; 1283 } 1284 } else 1285 iv_buffer = NULL; 1286 1287 /* Determine size of SGL. */ 1288 nsegs = 0; 1289 max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ 1290 if (!imm_payload) { 1291 nsegs = count_mbuf_segs(sndptr, sndptroff + 1292 TLS_HEADER_LENGTH, tls_size, &max_nsegs_1mbuf); 1293 if (!imm_ivs) { 1294 int n = sglist_count(iv_buffer, iv_len); 1295 nsegs += n; 1296 if (n > max_nsegs_1mbuf) 1297 max_nsegs_1mbuf = n; 1298 } 1299 1300 /* Account for SGL in work request length. */ 1301 wr_len += sizeof(struct ulptx_sgl) + 1302 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; 1303 } 1304 1305 wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); 1306 if (wr == NULL) { 1307 /* XXX: how will we recover from this? */ 1308 toep->flags |= TPF_TX_SUSPENDED; 1309 return; 1310 } 1311 1312#ifdef VERBOSE_TRACES 1313 CTR5(KTR_CXGBE, "%s: tid %d TLS record %d len %#x pdus %d", 1314 __func__, toep->tid, thdr.type, tls_size, pdus); 1315#endif 1316 txwr = wrtod(wr); 1317 cpl = (struct cpl_tx_tls_sfo *)(txwr + 1); 1318 memset(txwr, 0, roundup2(wr_len, 16)); 1319 credits = howmany(wr_len, 16); 1320 expn_size = tls_expansion_size(toep, tls_size, 0, NULL); 1321 write_tlstx_wr(txwr, toep, imm_payload ? tls_size : 0, 1322 tls_size, expn_size, pdus, credits, shove, imm_ivs ? 1 : 0); 1323 write_tlstx_cpl(cpl, toep, &thdr, tls_size, pdus); 1324 tls_copy_tx_key(toep, cpl + 1); 1325 1326 /* Generate random IVs */ 1327 buf = (char *)(cpl + 1) + key_size(toep); 1328 if (imm_ivs) { 1329 MPASS(iv_buffer == NULL); 1330 iv_dst = buf; 1331 buf = (char *)iv_dst + iv_len; 1332 } else 1333 iv_dst = iv_buffer; 1334 arc4rand(iv_dst, iv_len, 0); 1335 1336 if (imm_payload) { 1337 m_copydata(sndptr, sndptroff + TLS_HEADER_LENGTH, 1338 tls_size, buf); 1339 } else { 1340 write_tlstx_sgl(buf, sndptr, 1341 sndptroff + TLS_HEADER_LENGTH, tls_size, iv_buffer, 1342 iv_len, nsegs, max_nsegs_1mbuf); 1343 } 1344 1345 KASSERT(toep->tx_credits >= credits, 1346 ("%s: not enough credits", __func__)); 1347 1348 toep->tx_credits -= credits; 1349 1350 tp->snd_nxt += plen; 1351 tp->snd_max += plen; 1352 1353 SOCKBUF_LOCK(sb); 1354 sbsndptr_adv(sb, sb->sb_sndptr, plen); 1355 tls_ofld->sb_off += plen; 1356 SOCKBUF_UNLOCK(sb); 1357 1358 toep->flags |= TPF_TX_DATA_SENT; 1359 if (toep->tx_credits < MIN_OFLD_TLSTX_CREDITS(toep)) 1360 toep->flags |= TPF_TX_SUSPENDED; 1361 1362 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); 1363 txsd->plen = plen; 1364 txsd->tx_credits = credits; 1365 txsd->iv_buffer = iv_buffer; 1366 txsd++; 1367 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { 1368 toep->txsd_pidx = 0; 1369 txsd = &toep->txsd[0]; 1370 } 1371 toep->txsd_avail--; 1372 1373 atomic_add_long(&toep->vi->pi->tx_toe_tls_records, 1); 1374 atomic_add_long(&toep->vi->pi->tx_toe_tls_octets, plen); 1375 1376 t4_l2t_send(sc, wr, toep->l2te); 1377 } 1378} 1379 1380/* 1381 * For TLS data we place received mbufs received via CPL_TLS_DATA into 1382 * an mbufq in the TLS offload state. When CPL_RX_TLS_CMP is 1383 * received, the completed PDUs are placed into the socket receive 1384 * buffer. 1385 * 1386 * The TLS code reuses the ulp_pdu_reclaimq to hold the pending mbufs. 1387 */ 1388static int 1389do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1390{ 1391 struct adapter *sc = iq->adapter; 1392 const struct cpl_tls_data *cpl = mtod(m, const void *); 1393 unsigned int tid = GET_TID(cpl); 1394 struct toepcb *toep = lookup_tid(sc, tid); 1395 struct inpcb *inp = toep->inp; 1396 struct tcpcb *tp; 1397 int len; 1398 1399 /* XXX: Should this match do_rx_data instead? */ 1400 KASSERT(!(toep->flags & TPF_SYNQE), 1401 ("%s: toep %p claims to be a synq entry", __func__, toep)); 1402 1403 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); 1404 1405 /* strip off CPL header */ 1406 m_adj(m, sizeof(*cpl)); 1407 len = m->m_pkthdr.len; 1408 1409 atomic_add_long(&toep->vi->pi->rx_toe_tls_octets, len); 1410 1411 KASSERT(len == G_CPL_TLS_DATA_LENGTH(be32toh(cpl->length_pkd)), 1412 ("%s: payload length mismatch", __func__)); 1413 1414 INP_WLOCK(inp); 1415 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { 1416 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 1417 __func__, tid, len, inp->inp_flags); 1418 INP_WUNLOCK(inp); 1419 m_freem(m); 1420 return (0); 1421 } 1422 1423 /* Save TCP sequence number. */ 1424 m->m_pkthdr.tls_tcp_seq = be32toh(cpl->seq); 1425 1426 if (mbufq_enqueue(&toep->ulp_pdu_reclaimq, m)) { 1427#ifdef INVARIANTS 1428 panic("Failed to queue TLS data packet"); 1429#else 1430 printf("%s: Failed to queue TLS data packet\n", __func__); 1431 INP_WUNLOCK(inp); 1432 m_freem(m); 1433 return (0); 1434#endif 1435 } 1436 1437 tp = intotcpcb(inp); 1438 tp->t_rcvtime = ticks; 1439 1440#ifdef VERBOSE_TRACES 1441 CTR4(KTR_CXGBE, "%s: tid %u len %d seq %u", __func__, tid, len, 1442 be32toh(cpl->seq)); 1443#endif 1444 1445 INP_WUNLOCK(inp); 1446 return (0); 1447} 1448 1449static int 1450do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1451{ 1452 struct adapter *sc = iq->adapter; 1453 const struct cpl_rx_tls_cmp *cpl = mtod(m, const void *); 1454 struct tlsrx_hdr_pkt *tls_hdr_pkt; 1455 unsigned int tid = GET_TID(cpl); 1456 struct toepcb *toep = lookup_tid(sc, tid); 1457 struct inpcb *inp = toep->inp; 1458 struct tcpcb *tp; 1459 struct socket *so; 1460 struct sockbuf *sb; 1461 struct mbuf *tls_data; 1462 int len, pdu_length, rx_credits; 1463 1464 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); 1465 KASSERT(!(toep->flags & TPF_SYNQE), 1466 ("%s: toep %p claims to be a synq entry", __func__, toep)); 1467 1468 /* strip off CPL header */ 1469 m_adj(m, sizeof(*cpl)); 1470 len = m->m_pkthdr.len; 1471 1472 atomic_add_long(&toep->vi->pi->rx_toe_tls_records, 1); 1473 1474 KASSERT(len == G_CPL_RX_TLS_CMP_LENGTH(be32toh(cpl->pdulength_length)), 1475 ("%s: payload length mismatch", __func__)); 1476 1477 INP_WLOCK(inp); 1478 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { 1479 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 1480 __func__, tid, len, inp->inp_flags); 1481 INP_WUNLOCK(inp); 1482 m_freem(m); 1483 return (0); 1484 } 1485 1486 pdu_length = G_CPL_RX_TLS_CMP_PDULENGTH(be32toh(cpl->pdulength_length)); 1487 1488 tp = intotcpcb(inp); 1489 1490#ifdef VERBOSE_TRACES 1491 CTR6(KTR_CXGBE, "%s: tid %u PDU len %d len %d seq %u, rcv_nxt %u", 1492 __func__, tid, pdu_length, len, be32toh(cpl->seq), tp->rcv_nxt); 1493#endif 1494 1495 tp->rcv_nxt += pdu_length; 1496 KASSERT(tp->rcv_wnd >= pdu_length, 1497 ("%s: negative window size", __func__)); 1498 tp->rcv_wnd -= pdu_length; 1499 1500 /* XXX: Not sure what to do about urgent data. */ 1501 1502 /* 1503 * The payload of this CPL is the TLS header followed by 1504 * additional fields. 1505 */ 1506 KASSERT(m->m_len >= sizeof(*tls_hdr_pkt), 1507 ("%s: payload too small", __func__)); 1508 tls_hdr_pkt = mtod(m, void *); 1509 1510 /* 1511 * Only the TLS header is sent to OpenSSL, so report errors by 1512 * altering the record type. 1513 */ 1514 if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 0) 1515 tls_hdr_pkt->type = CONTENT_TYPE_ERROR; 1516 1517 /* Trim this CPL's mbuf to only include the TLS header. */ 1518 KASSERT(m->m_len == len && m->m_next == NULL, 1519 ("%s: CPL spans multiple mbufs", __func__)); 1520 m->m_len = TLS_HEADER_LENGTH; 1521 m->m_pkthdr.len = TLS_HEADER_LENGTH; 1522 1523 tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq); 1524 if (tls_data != NULL) { 1525 KASSERT(be32toh(cpl->seq) == tls_data->m_pkthdr.tls_tcp_seq, 1526 ("%s: sequence mismatch", __func__)); 1527 1528 /* 1529 * Update the TLS header length to be the length of 1530 * the payload data. 1531 */ 1532 tls_hdr_pkt->length = htobe16(tls_data->m_pkthdr.len); 1533 1534 m->m_next = tls_data; 1535 m->m_pkthdr.len += tls_data->m_len; 1536 } 1537 1538 so = inp_inpcbtosocket(inp); 1539 sb = &so->so_rcv; 1540 SOCKBUF_LOCK(sb); 1541 1542 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { 1543 struct epoch_tracker et; 1544 1545 CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)", 1546 __func__, tid, pdu_length); 1547 m_freem(m); 1548 SOCKBUF_UNLOCK(sb); 1549 INP_WUNLOCK(inp); 1550 1551 CURVNET_SET(toep->vnet); 1552 INP_INFO_RLOCK_ET(&V_tcbinfo, et); 1553 INP_WLOCK(inp); 1554 tp = tcp_drop(tp, ECONNRESET); 1555 if (tp) 1556 INP_WUNLOCK(inp); 1557 INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); 1558 CURVNET_RESTORE(); 1559 1560 return (0); 1561 } 1562 1563 /* 1564 * Not all of the bytes on the wire are included in the socket buffer 1565 * (e.g. the MAC of the TLS record). However, those bytes are included 1566 * in the TCP sequence space. 1567 */ 1568 1569 /* receive buffer autosize */ 1570 MPASS(toep->vnet == so->so_vnet); 1571 CURVNET_SET(toep->vnet); 1572 if (sb->sb_flags & SB_AUTOSIZE && 1573 V_tcp_do_autorcvbuf && 1574 sb->sb_hiwat < V_tcp_autorcvbuf_max && 1575 m->m_pkthdr.len > (sbspace(sb) / 8 * 7)) { 1576 unsigned int hiwat = sb->sb_hiwat; 1577 unsigned int newsize = min(hiwat + V_tcp_autorcvbuf_inc, 1578 V_tcp_autorcvbuf_max); 1579 1580 if (!sbreserve_locked(sb, newsize, so, NULL)) 1581 sb->sb_flags &= ~SB_AUTOSIZE; 1582 } 1583 1584 sbappendstream_locked(sb, m, 0); 1585 rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0; 1586#ifdef VERBOSE_TRACES 1587 CTR4(KTR_CXGBE, "%s: tid %u rx_credits %u rcv_wnd %u", 1588 __func__, tid, rx_credits, tp->rcv_wnd); 1589#endif 1590 if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) { 1591 rx_credits = send_rx_credits(sc, toep, rx_credits); 1592 tp->rcv_wnd += rx_credits; 1593 tp->rcv_adv += rx_credits; 1594 } 1595 1596 sorwakeup_locked(so); 1597 SOCKBUF_UNLOCK_ASSERT(sb); 1598 1599 INP_WUNLOCK(inp); 1600 CURVNET_RESTORE(); 1601 return (0); 1602} 1603 1604void 1605do_rx_data_tls(const struct cpl_rx_data *cpl, struct toepcb *toep, 1606 struct mbuf *m) 1607{ 1608 struct inpcb *inp = toep->inp; 1609 struct tls_ofld_info *tls_ofld = &toep->tls; 1610 struct tls_hdr *hdr; 1611 struct tcpcb *tp; 1612 struct socket *so; 1613 struct sockbuf *sb; 1614 int error, len, rx_credits; 1615 1616 len = m->m_pkthdr.len; 1617 1618 INP_WLOCK_ASSERT(inp); 1619 1620 so = inp_inpcbtosocket(inp); 1621 tp = intotcpcb(inp); 1622 sb = &so->so_rcv; 1623 SOCKBUF_LOCK(sb); 1624 CURVNET_SET(toep->vnet); 1625 1626 tp->rcv_nxt += len; 1627 KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__)); 1628 tp->rcv_wnd -= len; 1629 1630 /* Do we have a full TLS header? */ 1631 if (len < sizeof(*hdr)) { 1632 CTR3(KTR_CXGBE, "%s: tid %u len %d: too short for a TLS header", 1633 __func__, toep->tid, len); 1634 so->so_error = EMSGSIZE; 1635 goto out; 1636 } 1637 hdr = mtod(m, struct tls_hdr *); 1638 1639 /* Is the header valid? */ 1640 if (be16toh(hdr->version) != tls_ofld->k_ctx.proto_ver) { 1641 CTR3(KTR_CXGBE, "%s: tid %u invalid version %04x", 1642 __func__, toep->tid, be16toh(hdr->version)); 1643 error = EINVAL; 1644 goto report_error; 1645 } 1646 if (be16toh(hdr->length) < sizeof(*hdr)) { 1647 CTR3(KTR_CXGBE, "%s: tid %u invalid length %u", 1648 __func__, toep->tid, be16toh(hdr->length)); 1649 error = EBADMSG; 1650 goto report_error; 1651 } 1652 1653 /* Did we get a truncated record? */ 1654 if (len < be16toh(hdr->length)) { 1655 CTR4(KTR_CXGBE, "%s: tid %u truncated TLS record (%d vs %u)", 1656 __func__, toep->tid, len, be16toh(hdr->length)); 1657 1658 error = EMSGSIZE; 1659 goto report_error; 1660 } 1661 1662 /* Is the header type unknown? */ 1663 switch (hdr->type) { 1664 case CONTENT_TYPE_CCS: 1665 case CONTENT_TYPE_ALERT: 1666 case CONTENT_TYPE_APP_DATA: 1667 case CONTENT_TYPE_HANDSHAKE: 1668 break; 1669 default: 1670 CTR3(KTR_CXGBE, "%s: tid %u invalid TLS record type %u", 1671 __func__, toep->tid, hdr->type); 1672 error = EBADMSG; 1673 goto report_error; 1674 } 1675 1676 /* 1677 * Just punt. Although this could fall back to software 1678 * decryption, this case should never really happen. 1679 */ 1680 CTR4(KTR_CXGBE, "%s: tid %u dropping TLS record type %u, length %u", 1681 __func__, toep->tid, hdr->type, be16toh(hdr->length)); 1682 error = EBADMSG; 1683 1684report_error: 1685#ifdef KERN_TLS 1686 if (toep->tls.mode == TLS_MODE_KTLS) 1687 so->so_error = error; 1688 else 1689#endif 1690 { 1691 /* 1692 * Report errors by sending an empty TLS record 1693 * with an error record type. 1694 */ 1695 hdr->type = CONTENT_TYPE_ERROR; 1696 1697 /* Trim this CPL's mbuf to only include the TLS header. */ 1698 KASSERT(m->m_len == len && m->m_next == NULL, 1699 ("%s: CPL spans multiple mbufs", __func__)); 1700 m->m_len = TLS_HEADER_LENGTH; 1701 m->m_pkthdr.len = TLS_HEADER_LENGTH; 1702 1703 sbappendstream_locked(sb, m, 0); 1704 m = NULL; 1705 } 1706 1707out: 1708 /* 1709 * This connection is going to die anyway, so probably don't 1710 * need to bother with returning credits. 1711 */ 1712 rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0; 1713#ifdef VERBOSE_TRACES 1714 CTR4(KTR_CXGBE, "%s: tid %u rx_credits %u rcv_wnd %u", 1715 __func__, toep->tid, rx_credits, tp->rcv_wnd); 1716#endif 1717 if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) { 1718 rx_credits = send_rx_credits(toep->vi->adapter, toep, 1719 rx_credits); 1720 tp->rcv_wnd += rx_credits; 1721 tp->rcv_adv += rx_credits; 1722 } 1723 1724 sorwakeup_locked(so); 1725 SOCKBUF_UNLOCK_ASSERT(sb); 1726 1727 INP_WUNLOCK(inp); 1728 CURVNET_RESTORE(); 1729 1730 m_freem(m); 1731} 1732 1733void 1734t4_tls_mod_load(void) 1735{ 1736 1737 mtx_init(&tls_handshake_lock, "t4tls handshake", NULL, MTX_DEF); 1738 t4_register_cpl_handler(CPL_TLS_DATA, do_tls_data); 1739 t4_register_cpl_handler(CPL_RX_TLS_CMP, do_rx_tls_cmp); 1740} 1741 1742void 1743t4_tls_mod_unload(void) 1744{ 1745 1746 t4_register_cpl_handler(CPL_TLS_DATA, NULL); 1747 t4_register_cpl_handler(CPL_RX_TLS_CMP, NULL); 1748 mtx_destroy(&tls_handshake_lock); 1749} 1750#endif /* TCP_OFFLOAD */ 1751