t4_tls.c revision 351228
1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2017-2018 Chelsio Communications, Inc. 5 * All rights reserved. 6 * Written by: John Baldwin <jhb@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30#include "opt_inet.h" 31 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD: stable/11/sys/dev/cxgbe/tom/t4_tls.c 351228 2019-08-19 18:50:56Z jhb $"); 34 35#include <sys/param.h> 36#include <sys/sglist.h> 37#include <sys/socket.h> 38#include <sys/socketvar.h> 39#include <sys/systm.h> 40#include <netinet/in.h> 41#include <netinet/in_pcb.h> 42#include <netinet/tcp_var.h> 43#include <netinet/toecore.h> 44 45#ifdef TCP_OFFLOAD 46#include "common/common.h" 47#include "common/t4_tcb.h" 48#include "tom/t4_tom_l2t.h" 49#include "tom/t4_tom.h" 50 51VNET_DECLARE(int, tcp_do_autosndbuf); 52#define V_tcp_do_autosndbuf VNET(tcp_do_autosndbuf) 53VNET_DECLARE(int, tcp_autosndbuf_inc); 54#define V_tcp_autosndbuf_inc VNET(tcp_autosndbuf_inc) 55VNET_DECLARE(int, tcp_autosndbuf_max); 56#define V_tcp_autosndbuf_max VNET(tcp_autosndbuf_max) 57VNET_DECLARE(int, tcp_do_autorcvbuf); 58#define V_tcp_do_autorcvbuf VNET(tcp_do_autorcvbuf) 59VNET_DECLARE(int, tcp_autorcvbuf_inc); 60#define V_tcp_autorcvbuf_inc VNET(tcp_autorcvbuf_inc) 61VNET_DECLARE(int, tcp_autorcvbuf_max); 62#define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max) 63 64/* 65 * The TCP sequence number of a CPL_TLS_DATA mbuf is saved here while 66 * the mbuf is in the ulp_pdu_reclaimq. 67 */ 68#define tls_tcp_seq PH_loc.thirtytwo[0] 69 70/* 71 * Handshake lock used for the handshake timer. Having a global lock 72 * is perhaps not ideal, but it avoids having to use callout_drain() 73 * in tls_uninit_toep() which can't block. Also, the timer shouldn't 74 * actually fire for most connections. 75 */ 76static struct mtx tls_handshake_lock; 77 78static void 79t4_set_tls_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask, 80 uint64_t val) 81{ 82 struct adapter *sc = td_adapter(toep->td); 83 84 t4_set_tcb_field(sc, toep->ofld_txq, toep, word, mask, val, 0, 0); 85} 86 87/* TLS and DTLS common routines */ 88bool 89can_tls_offload(struct adapter *sc) 90{ 91 92 return (sc->tt.tls && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS); 93} 94 95int 96tls_tx_key(struct toepcb *toep) 97{ 98 struct tls_ofld_info *tls_ofld = &toep->tls; 99 100 return (tls_ofld->tx_key_addr >= 0); 101} 102 103int 104tls_rx_key(struct toepcb *toep) 105{ 106 struct tls_ofld_info *tls_ofld = &toep->tls; 107 108 return (tls_ofld->rx_key_addr >= 0); 109} 110 111static int 112key_size(struct toepcb *toep) 113{ 114 struct tls_ofld_info *tls_ofld = &toep->tls; 115 116 return ((tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) ? 117 tls_ofld->k_ctx.tx_key_info_size : KEY_IN_DDR_SIZE); 118} 119 120/* Set TLS Key-Id in TCB */ 121static void 122t4_set_tls_keyid(struct toepcb *toep, unsigned int key_id) 123{ 124 125 t4_set_tls_tcb_field(toep, W_TCB_RX_TLS_KEY_TAG, 126 V_TCB_RX_TLS_KEY_TAG(M_TCB_RX_TLS_BUF_TAG), 127 V_TCB_RX_TLS_KEY_TAG(key_id)); 128} 129 130/* Clear TF_RX_QUIESCE to re-enable receive. */ 131static void 132t4_clear_rx_quiesce(struct toepcb *toep) 133{ 134 135 t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0); 136} 137 138static void 139tls_clr_ofld_mode(struct toepcb *toep) 140{ 141 142 tls_stop_handshake_timer(toep); 143 144 /* Operate in PDU extraction mode only. */ 145 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, 146 V_TCB_ULP_RAW(M_TCB_ULP_RAW), 147 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1))); 148 t4_clear_rx_quiesce(toep); 149} 150 151static void 152tls_clr_quiesce(struct toepcb *toep) 153{ 154 155 tls_stop_handshake_timer(toep); 156 t4_clear_rx_quiesce(toep); 157} 158 159/* 160 * Calculate the TLS data expansion size 161 */ 162static int 163tls_expansion_size(struct toepcb *toep, int data_len, int full_pdus_only, 164 unsigned short *pdus_per_ulp) 165{ 166 struct tls_ofld_info *tls_ofld = &toep->tls; 167 struct tls_scmd *scmd = &tls_ofld->scmd0; 168 int expn_size = 0, frag_count = 0, pad_per_pdu = 0, 169 pad_last_pdu = 0, last_frag_size = 0, max_frag_size = 0; 170 int exp_per_pdu = 0; 171 int hdr_len = TLS_HEADER_LENGTH; 172 173 do { 174 max_frag_size = tls_ofld->k_ctx.frag_size; 175 if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) == 176 SCMD_CIPH_MODE_AES_GCM) { 177 frag_count = (data_len / max_frag_size); 178 exp_per_pdu = GCM_TAG_SIZE + AEAD_EXPLICIT_DATA_SIZE + 179 hdr_len; 180 expn_size = frag_count * exp_per_pdu; 181 if (full_pdus_only) { 182 *pdus_per_ulp = data_len / (exp_per_pdu + 183 max_frag_size); 184 if (*pdus_per_ulp > 32) 185 *pdus_per_ulp = 32; 186 else if(!*pdus_per_ulp) 187 *pdus_per_ulp = 1; 188 expn_size = (*pdus_per_ulp) * exp_per_pdu; 189 break; 190 } 191 if ((last_frag_size = data_len % max_frag_size) > 0) { 192 frag_count += 1; 193 expn_size += exp_per_pdu; 194 } 195 break; 196 } else if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) != 197 SCMD_CIPH_MODE_NOP) { 198 /* Calculate the number of fragments we can make */ 199 frag_count = (data_len / max_frag_size); 200 if (frag_count > 0) { 201 pad_per_pdu = (((howmany((max_frag_size + 202 tls_ofld->mac_length), 203 CIPHER_BLOCK_SIZE)) * 204 CIPHER_BLOCK_SIZE) - 205 (max_frag_size + 206 tls_ofld->mac_length)); 207 if (!pad_per_pdu) 208 pad_per_pdu = CIPHER_BLOCK_SIZE; 209 exp_per_pdu = pad_per_pdu + 210 tls_ofld->mac_length + 211 hdr_len + CIPHER_BLOCK_SIZE; 212 expn_size = frag_count * exp_per_pdu; 213 } 214 if (full_pdus_only) { 215 *pdus_per_ulp = data_len / (exp_per_pdu + 216 max_frag_size); 217 if (*pdus_per_ulp > 32) 218 *pdus_per_ulp = 32; 219 else if (!*pdus_per_ulp) 220 *pdus_per_ulp = 1; 221 expn_size = (*pdus_per_ulp) * exp_per_pdu; 222 break; 223 } 224 /* Consider the last fragment */ 225 if ((last_frag_size = data_len % max_frag_size) > 0) { 226 pad_last_pdu = (((howmany((last_frag_size + 227 tls_ofld->mac_length), 228 CIPHER_BLOCK_SIZE)) * 229 CIPHER_BLOCK_SIZE) - 230 (last_frag_size + 231 tls_ofld->mac_length)); 232 if (!pad_last_pdu) 233 pad_last_pdu = CIPHER_BLOCK_SIZE; 234 expn_size += (pad_last_pdu + 235 tls_ofld->mac_length + hdr_len + 236 CIPHER_BLOCK_SIZE); 237 } 238 } 239 } while (0); 240 241 return (expn_size); 242} 243 244/* Copy Key to WR */ 245static void 246tls_copy_tx_key(struct toepcb *toep, void *dst) 247{ 248 struct tls_ofld_info *tls_ofld = &toep->tls; 249 struct ulptx_sc_memrd *sc_memrd; 250 struct ulptx_idata *sc; 251 252 if (tls_ofld->k_ctx.tx_key_info_size <= 0) 253 return; 254 255 if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR) { 256 sc = dst; 257 sc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); 258 sc->len = htobe32(0); 259 sc_memrd = (struct ulptx_sc_memrd *)(sc + 1); 260 sc_memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) | 261 V_ULP_TX_SC_MORE(1) | 262 V_ULPTX_LEN16(tls_ofld->k_ctx.tx_key_info_size >> 4)); 263 sc_memrd->addr = htobe32(tls_ofld->tx_key_addr >> 5); 264 } else if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) { 265 memcpy(dst, &tls_ofld->k_ctx.tx, 266 tls_ofld->k_ctx.tx_key_info_size); 267 } 268} 269 270/* TLS/DTLS content type for CPL SFO */ 271static inline unsigned char 272tls_content_type(unsigned char content_type) 273{ 274 /* 275 * XXX: Shouldn't this map CONTENT_TYPE_APP_DATA to DATA and 276 * default to "CUSTOM" for all other types including 277 * heartbeat? 278 */ 279 switch (content_type) { 280 case CONTENT_TYPE_CCS: 281 return CPL_TX_TLS_SFO_TYPE_CCS; 282 case CONTENT_TYPE_ALERT: 283 return CPL_TX_TLS_SFO_TYPE_ALERT; 284 case CONTENT_TYPE_HANDSHAKE: 285 return CPL_TX_TLS_SFO_TYPE_HANDSHAKE; 286 case CONTENT_TYPE_HEARTBEAT: 287 return CPL_TX_TLS_SFO_TYPE_HEARTBEAT; 288 } 289 return CPL_TX_TLS_SFO_TYPE_DATA; 290} 291 292static unsigned char 293get_cipher_key_size(unsigned int ck_size) 294{ 295 switch (ck_size) { 296 case AES_NOP: /* NOP */ 297 return 15; 298 case AES_128: /* AES128 */ 299 return CH_CK_SIZE_128; 300 case AES_192: /* AES192 */ 301 return CH_CK_SIZE_192; 302 case AES_256: /* AES256 */ 303 return CH_CK_SIZE_256; 304 default: 305 return CH_CK_SIZE_256; 306 } 307} 308 309static unsigned char 310get_mac_key_size(unsigned int mk_size) 311{ 312 switch (mk_size) { 313 case SHA_NOP: /* NOP */ 314 return CH_MK_SIZE_128; 315 case SHA_GHASH: /* GHASH */ 316 case SHA_512: /* SHA512 */ 317 return CH_MK_SIZE_512; 318 case SHA_224: /* SHA2-224 */ 319 return CH_MK_SIZE_192; 320 case SHA_256: /* SHA2-256*/ 321 return CH_MK_SIZE_256; 322 case SHA_384: /* SHA384 */ 323 return CH_MK_SIZE_512; 324 case SHA1: /* SHA1 */ 325 default: 326 return CH_MK_SIZE_160; 327 } 328} 329 330static unsigned int 331get_proto_ver(int proto_ver) 332{ 333 switch (proto_ver) { 334 case TLS1_2_VERSION: 335 return TLS_1_2_VERSION; 336 case TLS1_1_VERSION: 337 return TLS_1_1_VERSION; 338 case DTLS1_2_VERSION: 339 return DTLS_1_2_VERSION; 340 default: 341 return TLS_VERSION_MAX; 342 } 343} 344 345static void 346tls_rxkey_flit1(struct tls_keyctx *kwr, struct tls_key_context *kctx) 347{ 348 349 if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) { 350 kwr->u.rxhdr.ivinsert_to_authinsrt = 351 htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | 352 V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | 353 V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | 354 V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(14ULL) | 355 V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(16ULL) | 356 V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(14ULL) | 357 V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | 358 V_TLS_KEYCTX_TX_WR_AUTHINSRT(16ULL)); 359 kwr->u.rxhdr.ivpresent_to_rxmk_size &= 360 ~(V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1)); 361 kwr->u.rxhdr.authmode_to_rxvalid &= 362 ~(V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1)); 363 } else { 364 kwr->u.rxhdr.ivinsert_to_authinsrt = 365 htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | 366 V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | 367 V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | 368 V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(22ULL) | 369 V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(0ULL) | 370 V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(22ULL) | 371 V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | 372 V_TLS_KEYCTX_TX_WR_AUTHINSRT(0ULL)); 373 } 374} 375 376/* Rx key */ 377static void 378prepare_rxkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx) 379{ 380 unsigned int ck_size = kctx->cipher_secret_size; 381 unsigned int mk_size = kctx->mac_secret_size; 382 int proto_ver = kctx->proto_ver; 383 384 kwr->u.rxhdr.flitcnt_hmacctrl = 385 ((kctx->tx_key_info_size >> 4) << 3) | kctx->hmac_ctrl; 386 387 kwr->u.rxhdr.protover_ciphmode = 388 V_TLS_KEYCTX_TX_WR_PROTOVER(get_proto_ver(proto_ver)) | 389 V_TLS_KEYCTX_TX_WR_CIPHMODE(kctx->state.enc_mode); 390 391 kwr->u.rxhdr.authmode_to_rxvalid = 392 V_TLS_KEYCTX_TX_WR_AUTHMODE(kctx->state.auth_mode) | 393 V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1) | 394 V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(3) | 395 V_TLS_KEYCTX_TX_WR_RXVALID(1); 396 397 kwr->u.rxhdr.ivpresent_to_rxmk_size = 398 V_TLS_KEYCTX_TX_WR_IVPRESENT(0) | 399 V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1) | 400 V_TLS_KEYCTX_TX_WR_RXCK_SIZE(get_cipher_key_size(ck_size)) | 401 V_TLS_KEYCTX_TX_WR_RXMK_SIZE(get_mac_key_size(mk_size)); 402 403 tls_rxkey_flit1(kwr, kctx); 404 405 /* No key reversal for GCM */ 406 if (kctx->state.enc_mode != CH_EVP_CIPH_GCM_MODE) { 407 t4_aes_getdeckey(kwr->keys.edkey, kctx->rx.key, 408 (kctx->cipher_secret_size << 3)); 409 memcpy(kwr->keys.edkey + kctx->cipher_secret_size, 410 kctx->rx.key + kctx->cipher_secret_size, 411 (IPAD_SIZE + OPAD_SIZE)); 412 } else { 413 memcpy(kwr->keys.edkey, kctx->rx.key, 414 (kctx->tx_key_info_size - SALT_SIZE)); 415 memcpy(kwr->u.rxhdr.rxsalt, kctx->rx.salt, SALT_SIZE); 416 } 417} 418 419/* Tx key */ 420static void 421prepare_txkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx) 422{ 423 unsigned int ck_size = kctx->cipher_secret_size; 424 unsigned int mk_size = kctx->mac_secret_size; 425 426 kwr->u.txhdr.ctxlen = 427 (kctx->tx_key_info_size >> 4); 428 kwr->u.txhdr.dualck_to_txvalid = 429 V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1) | 430 V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) | 431 V_TLS_KEYCTX_TX_WR_TXCK_SIZE(get_cipher_key_size(ck_size)) | 432 V_TLS_KEYCTX_TX_WR_TXMK_SIZE(get_mac_key_size(mk_size)) | 433 V_TLS_KEYCTX_TX_WR_TXVALID(1); 434 435 memcpy(kwr->keys.edkey, kctx->tx.key, HDR_KCTX_SIZE); 436 if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) { 437 memcpy(kwr->u.txhdr.txsalt, kctx->tx.salt, SALT_SIZE); 438 kwr->u.txhdr.dualck_to_txvalid &= 439 ~(V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1)); 440 } 441 kwr->u.txhdr.dualck_to_txvalid = htons(kwr->u.txhdr.dualck_to_txvalid); 442} 443 444/* TLS Key memory management */ 445static int 446get_new_keyid(struct toepcb *toep, struct tls_key_context *k_ctx) 447{ 448 struct adapter *sc = td_adapter(toep->td); 449 vmem_addr_t addr; 450 451 if (vmem_alloc(sc->key_map, TLS_KEY_CONTEXT_SZ, M_NOWAIT | M_FIRSTFIT, 452 &addr) != 0) 453 return (-1); 454 455 return (addr); 456} 457 458static void 459free_keyid(struct toepcb *toep, int keyid) 460{ 461 struct adapter *sc = td_adapter(toep->td); 462 463 vmem_free(sc->key_map, keyid, TLS_KEY_CONTEXT_SZ); 464} 465 466static void 467clear_tls_keyid(struct toepcb *toep) 468{ 469 struct tls_ofld_info *tls_ofld = &toep->tls; 470 471 if (tls_ofld->rx_key_addr >= 0) { 472 free_keyid(toep, tls_ofld->rx_key_addr); 473 tls_ofld->rx_key_addr = -1; 474 } 475 if (tls_ofld->tx_key_addr >= 0) { 476 free_keyid(toep, tls_ofld->tx_key_addr); 477 tls_ofld->tx_key_addr = -1; 478 } 479} 480 481static int 482get_keyid(struct tls_ofld_info *tls_ofld, unsigned int ops) 483{ 484 return (ops & KEY_WRITE_RX ? tls_ofld->rx_key_addr : 485 ((ops & KEY_WRITE_TX) ? tls_ofld->tx_key_addr : -1)); 486} 487 488static int 489get_tp_plen_max(struct tls_ofld_info *tls_ofld) 490{ 491 int plen = ((min(3*4096, TP_TX_PG_SZ))/1448) * 1448; 492 493 return (tls_ofld->k_ctx.frag_size <= 8192 ? plen : FC_TP_PLEN_MAX); 494} 495 496/* Send request to get the key-id */ 497static int 498tls_program_key_id(struct toepcb *toep, struct tls_key_context *k_ctx) 499{ 500 struct tls_ofld_info *tls_ofld = &toep->tls; 501 struct adapter *sc = td_adapter(toep->td); 502 struct ofld_tx_sdesc *txsd; 503 int kwrlen, kctxlen, keyid, len; 504 struct wrqe *wr; 505 struct tls_key_req *kwr; 506 struct tls_keyctx *kctx; 507 508 kwrlen = sizeof(*kwr); 509 kctxlen = roundup2(sizeof(*kctx), 32); 510 len = roundup2(kwrlen + kctxlen, 16); 511 512 if (toep->txsd_avail == 0) 513 return (EAGAIN); 514 515 /* Dont initialize key for re-neg */ 516 if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) { 517 if ((keyid = get_new_keyid(toep, k_ctx)) < 0) { 518 return (ENOSPC); 519 } 520 } else { 521 keyid = get_keyid(tls_ofld, k_ctx->l_p_key); 522 } 523 524 wr = alloc_wrqe(len, toep->ofld_txq); 525 if (wr == NULL) { 526 free_keyid(toep, keyid); 527 return (ENOMEM); 528 } 529 kwr = wrtod(wr); 530 memset(kwr, 0, kwrlen); 531 532 kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) | F_FW_WR_COMPL | 533 F_FW_WR_ATOMIC); 534 kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(len, 16)) | 535 V_FW_WR_FLOWID(toep->tid)); 536 kwr->protocol = get_proto_ver(k_ctx->proto_ver); 537 kwr->mfs = htons(k_ctx->frag_size); 538 kwr->reneg_to_write_rx = k_ctx->l_p_key; 539 540 /* master command */ 541 kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) | 542 V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1)); 543 kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(kctxlen >> 5)); 544 kwr->len16 = htobe32((toep->tid << 8) | 545 DIV_ROUND_UP(len - sizeof(struct work_request_hdr), 16)); 546 kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5)); 547 548 /* sub command */ 549 kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); 550 kwr->sc_len = htobe32(kctxlen); 551 552 kctx = (struct tls_keyctx *)(kwr + 1); 553 memset(kctx, 0, kctxlen); 554 555 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) { 556 tls_ofld->tx_key_addr = keyid; 557 prepare_txkey_wr(kctx, k_ctx); 558 } else if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { 559 tls_ofld->rx_key_addr = keyid; 560 prepare_rxkey_wr(kctx, k_ctx); 561 } 562 563 txsd = &toep->txsd[toep->txsd_pidx]; 564 txsd->tx_credits = DIV_ROUND_UP(len, 16); 565 txsd->plen = 0; 566 toep->tx_credits -= txsd->tx_credits; 567 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) 568 toep->txsd_pidx = 0; 569 toep->txsd_avail--; 570 571 t4_wrq_tx(sc, wr); 572 573 return (0); 574} 575 576/* Store a key received from SSL in DDR. */ 577static int 578program_key_context(struct tcpcb *tp, struct toepcb *toep, 579 struct tls_key_context *uk_ctx) 580{ 581 struct adapter *sc = td_adapter(toep->td); 582 struct tls_ofld_info *tls_ofld = &toep->tls; 583 struct tls_key_context *k_ctx; 584 int error, key_offset; 585 586 if (tp->t_state != TCPS_ESTABLISHED) { 587 /* 588 * XXX: Matches Linux driver, but not sure this is a 589 * very appropriate error. 590 */ 591 return (ENOENT); 592 } 593 594 /* Stop timer on handshake completion */ 595 tls_stop_handshake_timer(toep); 596 597 toep->flags &= ~TPF_FORCE_CREDITS; 598 599 CTR4(KTR_CXGBE, "%s: tid %d %s proto_ver %#x", __func__, toep->tid, 600 G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX ? "KEY_WRITE_RX" : 601 "KEY_WRITE_TX", uk_ctx->proto_ver); 602 603 if (G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX && 604 toep->ulp_mode != ULP_MODE_TLS) 605 return (EOPNOTSUPP); 606 607 /* Don't copy the 'tx' and 'rx' fields. */ 608 k_ctx = &tls_ofld->k_ctx; 609 memcpy(&k_ctx->l_p_key, &uk_ctx->l_p_key, 610 sizeof(*k_ctx) - offsetof(struct tls_key_context, l_p_key)); 611 612 /* TLS version != 1.1 and !1.2 OR DTLS != 1.2 */ 613 if (get_proto_ver(k_ctx->proto_ver) > DTLS_1_2_VERSION) { 614 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { 615 tls_ofld->rx_key_addr = -1; 616 t4_clear_rx_quiesce(toep); 617 } else { 618 tls_ofld->tx_key_addr = -1; 619 } 620 return (0); 621 } 622 623 if (k_ctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) { 624 k_ctx->iv_size = 4; 625 k_ctx->mac_first = 0; 626 k_ctx->hmac_ctrl = 0; 627 } else { 628 k_ctx->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */ 629 k_ctx->mac_first = 1; 630 } 631 632 tls_ofld->scmd0.seqno_numivs = 633 (V_SCMD_SEQ_NO_CTRL(3) | 634 V_SCMD_PROTO_VERSION(get_proto_ver(k_ctx->proto_ver)) | 635 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | 636 V_SCMD_CIPH_AUTH_SEQ_CTRL((k_ctx->mac_first == 0)) | 637 V_SCMD_CIPH_MODE(k_ctx->state.enc_mode) | 638 V_SCMD_AUTH_MODE(k_ctx->state.auth_mode) | 639 V_SCMD_HMAC_CTRL(k_ctx->hmac_ctrl) | 640 V_SCMD_IV_SIZE(k_ctx->iv_size)); 641 642 tls_ofld->scmd0.ivgen_hdrlen = 643 (V_SCMD_IV_GEN_CTRL(k_ctx->iv_ctrl) | 644 V_SCMD_KEY_CTX_INLINE(0) | 645 V_SCMD_TLS_FRAG_ENABLE(1)); 646 647 tls_ofld->mac_length = k_ctx->mac_secret_size; 648 649 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { 650 k_ctx->rx = uk_ctx->rx; 651 /* Dont initialize key for re-neg */ 652 if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) 653 tls_ofld->rx_key_addr = -1; 654 } else { 655 k_ctx->tx = uk_ctx->tx; 656 /* Dont initialize key for re-neg */ 657 if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) 658 tls_ofld->tx_key_addr = -1; 659 } 660 661 /* Flush pending data before new Tx key becomes active */ 662 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) { 663 struct sockbuf *sb; 664 665 /* XXX: This might not drain everything. */ 666 t4_push_frames(sc, toep, 0); 667 sb = &toep->inp->inp_socket->so_snd; 668 SOCKBUF_LOCK(sb); 669 670 /* XXX: This asserts that everything has been pushed. */ 671 MPASS(sb->sb_sndptr == NULL || sb->sb_sndptr->m_next == NULL); 672 sb->sb_sndptr = NULL; 673 tls_ofld->sb_off = sbavail(sb); 674 SOCKBUF_UNLOCK(sb); 675 tls_ofld->tx_seq_no = 0; 676 } 677 678 if ((G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) || 679 (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR)) { 680 error = tls_program_key_id(toep, k_ctx); 681 if (error) { 682 /* XXX: Only clear quiesce for KEY_WRITE_RX? */ 683 t4_clear_rx_quiesce(toep); 684 return (error); 685 } 686 } 687 688 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { 689 /* 690 * RX key tags are an index into the key portion of MA 691 * memory stored as an offset from the base address in 692 * units of 64 bytes. 693 */ 694 key_offset = tls_ofld->rx_key_addr - sc->vres.key.start; 695 t4_set_tls_keyid(toep, key_offset / 64); 696 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, 697 V_TCB_ULP_RAW(M_TCB_ULP_RAW), 698 V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) | 699 V_TF_TLS_CONTROL(1) | 700 V_TF_TLS_ACTIVE(1) | 701 V_TF_TLS_ENABLE(1)))); 702 t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ, 703 V_TCB_TLS_SEQ(M_TCB_TLS_SEQ), 704 V_TCB_TLS_SEQ(0)); 705 t4_clear_rx_quiesce(toep); 706 } else { 707 unsigned short pdus_per_ulp; 708 709 if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) 710 tls_ofld->tx_key_addr = 1; 711 712 tls_ofld->fcplenmax = get_tp_plen_max(tls_ofld); 713 tls_ofld->expn_per_ulp = tls_expansion_size(toep, 714 tls_ofld->fcplenmax, 1, &pdus_per_ulp); 715 tls_ofld->pdus_per_ulp = pdus_per_ulp; 716 tls_ofld->adjusted_plen = tls_ofld->pdus_per_ulp * 717 ((tls_ofld->expn_per_ulp/tls_ofld->pdus_per_ulp) + 718 tls_ofld->k_ctx.frag_size); 719 } 720 721 return (0); 722} 723 724/* 725 * In some cases a client connection can hang without sending the 726 * ServerHelloDone message from the NIC to the host. Send a dummy 727 * RX_DATA_ACK with RX_MODULATE to unstick the connection. 728 */ 729static void 730tls_send_handshake_ack(void *arg) 731{ 732 struct toepcb *toep = arg; 733 struct tls_ofld_info *tls_ofld = &toep->tls; 734 struct adapter *sc = td_adapter(toep->td); 735 736 /* 737 * XXX: Does not have the t4_get_tcb() checks to refine the 738 * workaround. 739 */ 740 callout_schedule(&tls_ofld->handshake_timer, TLS_SRV_HELLO_RD_TM * hz); 741 742 CTR2(KTR_CXGBE, "%s: tid %d sending RX_DATA_ACK", __func__, toep->tid); 743 send_rx_modulate(sc, toep); 744} 745 746static void 747tls_start_handshake_timer(struct toepcb *toep) 748{ 749 struct tls_ofld_info *tls_ofld = &toep->tls; 750 751 mtx_lock(&tls_handshake_lock); 752 callout_reset(&tls_ofld->handshake_timer, TLS_SRV_HELLO_BKOFF_TM * hz, 753 tls_send_handshake_ack, toep); 754 mtx_unlock(&tls_handshake_lock); 755} 756 757void 758tls_stop_handshake_timer(struct toepcb *toep) 759{ 760 struct tls_ofld_info *tls_ofld = &toep->tls; 761 762 mtx_lock(&tls_handshake_lock); 763 callout_stop(&tls_ofld->handshake_timer); 764 mtx_unlock(&tls_handshake_lock); 765} 766 767int 768t4_ctloutput_tls(struct socket *so, struct sockopt *sopt) 769{ 770 struct tls_key_context uk_ctx; 771 struct inpcb *inp; 772 struct tcpcb *tp; 773 struct toepcb *toep; 774 int error, optval; 775 776 error = 0; 777 if (sopt->sopt_dir == SOPT_SET && 778 sopt->sopt_name == TCP_TLSOM_SET_TLS_CONTEXT) { 779 error = sooptcopyin(sopt, &uk_ctx, sizeof(uk_ctx), 780 sizeof(uk_ctx)); 781 if (error) 782 return (error); 783 } 784 785 inp = sotoinpcb(so); 786 KASSERT(inp != NULL, ("tcp_ctloutput: inp == NULL")); 787 INP_WLOCK(inp); 788 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { 789 INP_WUNLOCK(inp); 790 return (ECONNRESET); 791 } 792 tp = intotcpcb(inp); 793 toep = tp->t_toe; 794 switch (sopt->sopt_dir) { 795 case SOPT_SET: 796 switch (sopt->sopt_name) { 797 case TCP_TLSOM_SET_TLS_CONTEXT: 798 error = program_key_context(tp, toep, &uk_ctx); 799 INP_WUNLOCK(inp); 800 break; 801 case TCP_TLSOM_CLR_TLS_TOM: 802 if (toep->ulp_mode == ULP_MODE_TLS) { 803 CTR2(KTR_CXGBE, "%s: tid %d CLR_TLS_TOM", 804 __func__, toep->tid); 805 tls_clr_ofld_mode(toep); 806 } else 807 error = EOPNOTSUPP; 808 INP_WUNLOCK(inp); 809 break; 810 case TCP_TLSOM_CLR_QUIES: 811 if (toep->ulp_mode == ULP_MODE_TLS) { 812 CTR2(KTR_CXGBE, "%s: tid %d CLR_QUIES", 813 __func__, toep->tid); 814 tls_clr_quiesce(toep); 815 } else 816 error = EOPNOTSUPP; 817 INP_WUNLOCK(inp); 818 break; 819 default: 820 INP_WUNLOCK(inp); 821 error = EOPNOTSUPP; 822 break; 823 } 824 break; 825 case SOPT_GET: 826 switch (sopt->sopt_name) { 827 case TCP_TLSOM_GET_TLS_TOM: 828 /* 829 * TLS TX is permitted on any TOE socket, but 830 * TLS RX requires a TLS ULP mode. 831 */ 832 optval = TLS_TOM_NONE; 833 if (can_tls_offload(td_adapter(toep->td))) { 834 switch (toep->ulp_mode) { 835 case ULP_MODE_NONE: 836 case ULP_MODE_TCPDDP: 837 optval = TLS_TOM_TXONLY; 838 break; 839 case ULP_MODE_TLS: 840 optval = TLS_TOM_BOTH; 841 break; 842 } 843 } 844 CTR3(KTR_CXGBE, "%s: tid %d GET_TLS_TOM = %d", 845 __func__, toep->tid, optval); 846 INP_WUNLOCK(inp); 847 error = sooptcopyout(sopt, &optval, sizeof(optval)); 848 break; 849 default: 850 INP_WUNLOCK(inp); 851 error = EOPNOTSUPP; 852 break; 853 } 854 break; 855 } 856 return (error); 857} 858 859void 860tls_init_toep(struct toepcb *toep) 861{ 862 struct tls_ofld_info *tls_ofld = &toep->tls; 863 864 tls_ofld->key_location = TLS_SFO_WR_CONTEXTLOC_DDR; 865 tls_ofld->rx_key_addr = -1; 866 tls_ofld->tx_key_addr = -1; 867 if (toep->ulp_mode == ULP_MODE_TLS) 868 callout_init_mtx(&tls_ofld->handshake_timer, 869 &tls_handshake_lock, 0); 870} 871 872void 873tls_establish(struct toepcb *toep) 874{ 875 876 /* 877 * Enable PDU extraction. 878 * 879 * XXX: Supposedly this should be done by the firmware when 880 * the ULP_MODE FLOWC parameter is set in send_flowc_wr(), but 881 * in practice this seems to be required. 882 */ 883 CTR2(KTR_CXGBE, "%s: tid %d setting TLS_ENABLE", __func__, toep->tid); 884 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW), 885 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1))); 886 887 toep->flags |= TPF_FORCE_CREDITS; 888 889 tls_start_handshake_timer(toep); 890} 891 892void 893tls_uninit_toep(struct toepcb *toep) 894{ 895 896 if (toep->ulp_mode == ULP_MODE_TLS) 897 tls_stop_handshake_timer(toep); 898 clear_tls_keyid(toep); 899} 900 901#define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16) 902#define MIN_OFLD_TLSTX_CREDITS(toep) \ 903 (howmany(sizeof(struct fw_tlstx_data_wr) + \ 904 sizeof(struct cpl_tx_tls_sfo) + key_size((toep)) + \ 905 CIPHER_BLOCK_SIZE + 1, 16)) 906 907static inline u_int 908max_imm_tls_space(int tx_credits) 909{ 910 const int n = 2; /* Use only up to 2 desc for imm. data WR */ 911 int space; 912 913 KASSERT(tx_credits >= 0 && 914 tx_credits <= MAX_OFLD_TX_CREDITS, 915 ("%s: %d credits", __func__, tx_credits)); 916 917 if (tx_credits >= (n * EQ_ESIZE) / 16) 918 space = (n * EQ_ESIZE); 919 else 920 space = tx_credits * 16; 921 return (space); 922} 923 924static int 925count_mbuf_segs(struct mbuf *m, int skip, int len, int *max_nsegs_1mbufp) 926{ 927 int max_nsegs_1mbuf, n, nsegs; 928 929 while (skip >= m->m_len) { 930 skip -= m->m_len; 931 m = m->m_next; 932 } 933 934 nsegs = 0; 935 max_nsegs_1mbuf = 0; 936 while (len > 0) { 937 n = sglist_count(mtod(m, char *) + skip, m->m_len - skip); 938 if (n > max_nsegs_1mbuf) 939 max_nsegs_1mbuf = n; 940 nsegs += n; 941 len -= m->m_len - skip; 942 skip = 0; 943 m = m->m_next; 944 } 945 *max_nsegs_1mbufp = max_nsegs_1mbuf; 946 return (nsegs); 947} 948 949static void 950write_tlstx_wr(struct fw_tlstx_data_wr *txwr, struct toepcb *toep, 951 unsigned int immdlen, unsigned int plen, unsigned int expn, 952 unsigned int pdus, uint8_t credits, int shove, int imm_ivs) 953{ 954 struct tls_ofld_info *tls_ofld = &toep->tls; 955 unsigned int len = plen + expn; 956 957 txwr->op_to_immdlen = htobe32(V_WR_OP(FW_TLSTX_DATA_WR) | 958 V_FW_TLSTX_DATA_WR_COMPL(1) | 959 V_FW_TLSTX_DATA_WR_IMMDLEN(immdlen)); 960 txwr->flowid_len16 = htobe32(V_FW_TLSTX_DATA_WR_FLOWID(toep->tid) | 961 V_FW_TLSTX_DATA_WR_LEN16(credits)); 962 txwr->plen = htobe32(len); 963 txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ULP_MODE_TLS) | 964 V_TX_URG(0) | /* F_T6_TX_FORCE | */ V_TX_SHOVE(shove)); 965 txwr->ctxloc_to_exp = htobe32(V_FW_TLSTX_DATA_WR_NUMIVS(pdus) | 966 V_FW_TLSTX_DATA_WR_EXP(expn) | 967 V_FW_TLSTX_DATA_WR_CTXLOC(tls_ofld->key_location) | 968 V_FW_TLSTX_DATA_WR_IVDSGL(!imm_ivs) | 969 V_FW_TLSTX_DATA_WR_KEYSIZE(tls_ofld->k_ctx.tx_key_info_size >> 4)); 970 txwr->mfs = htobe16(tls_ofld->k_ctx.frag_size); 971 txwr->adjustedplen_pkd = htobe16( 972 V_FW_TLSTX_DATA_WR_ADJUSTEDPLEN(tls_ofld->adjusted_plen)); 973 txwr->expinplenmax_pkd = htobe16( 974 V_FW_TLSTX_DATA_WR_EXPINPLENMAX(tls_ofld->expn_per_ulp)); 975 txwr->pdusinplenmax_pkd = htobe16( 976 V_FW_TLSTX_DATA_WR_PDUSINPLENMAX(tls_ofld->pdus_per_ulp)); 977} 978 979static void 980write_tlstx_cpl(struct cpl_tx_tls_sfo *cpl, struct toepcb *toep, 981 struct tls_hdr *tls_hdr, unsigned int plen, unsigned int pdus) 982{ 983 struct tls_ofld_info *tls_ofld = &toep->tls; 984 int data_type, seglen; 985 986 if (plen < tls_ofld->k_ctx.frag_size) 987 seglen = plen; 988 else 989 seglen = tls_ofld->k_ctx.frag_size; 990 data_type = tls_content_type(tls_hdr->type); 991 cpl->op_to_seg_len = htobe32(V_CPL_TX_TLS_SFO_OPCODE(CPL_TX_TLS_SFO) | 992 V_CPL_TX_TLS_SFO_DATA_TYPE(data_type) | 993 V_CPL_TX_TLS_SFO_CPL_LEN(2) | V_CPL_TX_TLS_SFO_SEG_LEN(seglen)); 994 cpl->pld_len = htobe32(plen); 995 if (data_type == CPL_TX_TLS_SFO_TYPE_HEARTBEAT) 996 cpl->type_protover = htobe32( 997 V_CPL_TX_TLS_SFO_TYPE(tls_hdr->type)); 998 cpl->seqno_numivs = htobe32(tls_ofld->scmd0.seqno_numivs | 999 V_SCMD_NUM_IVS(pdus)); 1000 cpl->ivgen_hdrlen = htobe32(tls_ofld->scmd0.ivgen_hdrlen); 1001 cpl->scmd1 = htobe64(tls_ofld->tx_seq_no); 1002 tls_ofld->tx_seq_no += pdus; 1003} 1004 1005/* 1006 * Similar to write_tx_sgl() except that it accepts an optional 1007 * trailer buffer for IVs. 1008 */ 1009static void 1010write_tlstx_sgl(void *dst, struct mbuf *start, int skip, int plen, 1011 void *iv_buffer, int iv_len, int nsegs, int n) 1012{ 1013 struct mbuf *m; 1014 struct ulptx_sgl *usgl = dst; 1015 int i, j, rc; 1016 struct sglist sg; 1017 struct sglist_seg segs[n]; 1018 1019 KASSERT(nsegs > 0, ("%s: nsegs 0", __func__)); 1020 1021 sglist_init(&sg, n, segs); 1022 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 1023 V_ULPTX_NSGE(nsegs)); 1024 1025 for (m = start; skip >= m->m_len; m = m->m_next) 1026 skip -= m->m_len; 1027 1028 i = -1; 1029 for (m = start; plen > 0; m = m->m_next) { 1030 rc = sglist_append(&sg, mtod(m, char *) + skip, 1031 m->m_len - skip); 1032 if (__predict_false(rc != 0)) 1033 panic("%s: sglist_append %d", __func__, rc); 1034 plen -= m->m_len - skip; 1035 skip = 0; 1036 1037 for (j = 0; j < sg.sg_nseg; i++, j++) { 1038 if (i < 0) { 1039 usgl->len0 = htobe32(segs[j].ss_len); 1040 usgl->addr0 = htobe64(segs[j].ss_paddr); 1041 } else { 1042 usgl->sge[i / 2].len[i & 1] = 1043 htobe32(segs[j].ss_len); 1044 usgl->sge[i / 2].addr[i & 1] = 1045 htobe64(segs[j].ss_paddr); 1046 } 1047#ifdef INVARIANTS 1048 nsegs--; 1049#endif 1050 } 1051 sglist_reset(&sg); 1052 } 1053 if (iv_buffer != NULL) { 1054 rc = sglist_append(&sg, iv_buffer, iv_len); 1055 if (__predict_false(rc != 0)) 1056 panic("%s: sglist_append %d", __func__, rc); 1057 1058 for (j = 0; j < sg.sg_nseg; i++, j++) { 1059 if (i < 0) { 1060 usgl->len0 = htobe32(segs[j].ss_len); 1061 usgl->addr0 = htobe64(segs[j].ss_paddr); 1062 } else { 1063 usgl->sge[i / 2].len[i & 1] = 1064 htobe32(segs[j].ss_len); 1065 usgl->sge[i / 2].addr[i & 1] = 1066 htobe64(segs[j].ss_paddr); 1067 } 1068#ifdef INVARIANTS 1069 nsegs--; 1070#endif 1071 } 1072 } 1073 if (i & 1) 1074 usgl->sge[i / 2].len[1] = htobe32(0); 1075 KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, iv_buffer %p", 1076 __func__, nsegs, start, iv_buffer)); 1077} 1078 1079/* 1080 * Similar to t4_push_frames() but handles TLS sockets when TLS offload 1081 * is enabled. Rather than transmitting bulk data, the socket buffer 1082 * contains TLS records. The work request requires a full TLS record, 1083 * so batch mbufs up until a full TLS record is seen. This requires 1084 * reading the TLS header out of the start of each record to determine 1085 * its length. 1086 */ 1087void 1088t4_push_tls_records(struct adapter *sc, struct toepcb *toep, int drop) 1089{ 1090 struct tls_hdr thdr; 1091 struct mbuf *sndptr; 1092 struct fw_tlstx_data_wr *txwr; 1093 struct cpl_tx_tls_sfo *cpl; 1094 struct wrqe *wr; 1095 u_int plen, nsegs, credits, space, max_nsegs_1mbuf, wr_len; 1096 u_int expn_size, iv_len, pdus, sndptroff; 1097 struct tls_ofld_info *tls_ofld = &toep->tls; 1098 struct inpcb *inp = toep->inp; 1099 struct tcpcb *tp = intotcpcb(inp); 1100 struct socket *so = inp->inp_socket; 1101 struct sockbuf *sb = &so->so_snd; 1102 int tls_size, tx_credits, shove, /* compl,*/ sowwakeup; 1103 struct ofld_tx_sdesc *txsd; 1104 bool imm_ivs, imm_payload; 1105 void *iv_buffer, *iv_dst, *buf; 1106 1107 INP_WLOCK_ASSERT(inp); 1108 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 1109 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); 1110 1111 KASSERT(toep->ulp_mode == ULP_MODE_NONE || 1112 toep->ulp_mode == ULP_MODE_TCPDDP || toep->ulp_mode == ULP_MODE_TLS, 1113 ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); 1114 KASSERT(tls_tx_key(toep), 1115 ("%s: TX key not set for toep %p", __func__, toep)); 1116 1117#ifdef VERBOSE_TRACES 1118 CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d", 1119 __func__, toep->tid, toep->flags, tp->t_flags); 1120#endif 1121 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) 1122 return; 1123 1124#ifdef RATELIMIT 1125 if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) && 1126 (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) { 1127 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED; 1128 } 1129#endif 1130 1131 /* 1132 * This function doesn't resume by itself. Someone else must clear the 1133 * flag and call this function. 1134 */ 1135 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { 1136 KASSERT(drop == 0, 1137 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); 1138 return; 1139 } 1140 1141 txsd = &toep->txsd[toep->txsd_pidx]; 1142 for (;;) { 1143 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); 1144 space = max_imm_tls_space(tx_credits); 1145 wr_len = sizeof(struct fw_tlstx_data_wr) + 1146 sizeof(struct cpl_tx_tls_sfo) + key_size(toep); 1147 if (wr_len + CIPHER_BLOCK_SIZE + 1 > space) { 1148#ifdef VERBOSE_TRACES 1149 CTR5(KTR_CXGBE, 1150 "%s: tid %d tx_credits %d min_wr %d space %d", 1151 __func__, toep->tid, tx_credits, wr_len + 1152 CIPHER_BLOCK_SIZE + 1, space); 1153#endif 1154 return; 1155 } 1156 1157 SOCKBUF_LOCK(sb); 1158 sowwakeup = drop; 1159 if (drop) { 1160 sbdrop_locked(sb, drop); 1161 MPASS(tls_ofld->sb_off >= drop); 1162 tls_ofld->sb_off -= drop; 1163 drop = 0; 1164 } 1165 1166 /* 1167 * Send a FIN if requested, but only if there's no 1168 * more data to send. 1169 */ 1170 if (sbavail(sb) == tls_ofld->sb_off && 1171 toep->flags & TPF_SEND_FIN) { 1172 if (sowwakeup) 1173 sowwakeup_locked(so); 1174 else 1175 SOCKBUF_UNLOCK(sb); 1176 SOCKBUF_UNLOCK_ASSERT(sb); 1177 t4_close_conn(sc, toep); 1178 return; 1179 } 1180 1181 if (sbavail(sb) < tls_ofld->sb_off + TLS_HEADER_LENGTH) { 1182 /* 1183 * A full TLS header is not yet queued, stop 1184 * for now until more data is added to the 1185 * socket buffer. However, if the connection 1186 * has been closed, we will never get the rest 1187 * of the header so just discard the partial 1188 * header and close the connection. 1189 */ 1190#ifdef VERBOSE_TRACES 1191 CTR5(KTR_CXGBE, "%s: tid %d sbavail %d sb_off %d%s", 1192 __func__, toep->tid, sbavail(sb), tls_ofld->sb_off, 1193 toep->flags & TPF_SEND_FIN ? "" : " SEND_FIN"); 1194#endif 1195 if (sowwakeup) 1196 sowwakeup_locked(so); 1197 else 1198 SOCKBUF_UNLOCK(sb); 1199 SOCKBUF_UNLOCK_ASSERT(sb); 1200 if (toep->flags & TPF_SEND_FIN) 1201 t4_close_conn(sc, toep); 1202 return; 1203 } 1204 1205 /* Read the header of the next TLS record. */ 1206 sndptr = sbsndmbuf(sb, tls_ofld->sb_off, &sndptroff); 1207 MPASS(!IS_AIOTX_MBUF(sndptr)); 1208 m_copydata(sndptr, sndptroff, sizeof(thdr), (caddr_t)&thdr); 1209 tls_size = htons(thdr.length); 1210 plen = TLS_HEADER_LENGTH + tls_size; 1211 pdus = howmany(tls_size, tls_ofld->k_ctx.frag_size); 1212 iv_len = pdus * CIPHER_BLOCK_SIZE; 1213 1214 if (sbavail(sb) < tls_ofld->sb_off + plen) { 1215 /* 1216 * The full TLS record is not yet queued, stop 1217 * for now until more data is added to the 1218 * socket buffer. However, if the connection 1219 * has been closed, we will never get the rest 1220 * of the record so just discard the partial 1221 * record and close the connection. 1222 */ 1223#ifdef VERBOSE_TRACES 1224 CTR6(KTR_CXGBE, 1225 "%s: tid %d sbavail %d sb_off %d plen %d%s", 1226 __func__, toep->tid, sbavail(sb), tls_ofld->sb_off, 1227 plen, toep->flags & TPF_SEND_FIN ? "" : 1228 " SEND_FIN"); 1229#endif 1230 if (sowwakeup) 1231 sowwakeup_locked(so); 1232 else 1233 SOCKBUF_UNLOCK(sb); 1234 SOCKBUF_UNLOCK_ASSERT(sb); 1235 if (toep->flags & TPF_SEND_FIN) 1236 t4_close_conn(sc, toep); 1237 return; 1238 } 1239 1240 /* Shove if there is no additional data pending. */ 1241 shove = (sbavail(sb) == tls_ofld->sb_off + plen) && 1242 !(tp->t_flags & TF_MORETOCOME); 1243 1244 if (sb->sb_flags & SB_AUTOSIZE && 1245 V_tcp_do_autosndbuf && 1246 sb->sb_hiwat < V_tcp_autosndbuf_max && 1247 sbused(sb) >= sb->sb_hiwat * 7 / 8) { 1248 int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc, 1249 V_tcp_autosndbuf_max); 1250 1251 if (!sbreserve_locked(sb, newsize, so, NULL)) 1252 sb->sb_flags &= ~SB_AUTOSIZE; 1253 else 1254 sowwakeup = 1; /* room available */ 1255 } 1256 if (sowwakeup) 1257 sowwakeup_locked(so); 1258 else 1259 SOCKBUF_UNLOCK(sb); 1260 SOCKBUF_UNLOCK_ASSERT(sb); 1261 1262 if (__predict_false(toep->flags & TPF_FIN_SENT)) 1263 panic("%s: excess tx.", __func__); 1264 1265 /* Determine whether to use immediate vs SGL. */ 1266 imm_payload = false; 1267 imm_ivs = false; 1268 if (wr_len + iv_len <= space) { 1269 imm_ivs = true; 1270 wr_len += iv_len; 1271 if (wr_len + tls_size <= space) { 1272 wr_len += tls_size; 1273 imm_payload = true; 1274 } 1275 } 1276 1277 /* Allocate space for IVs if needed. */ 1278 if (!imm_ivs) { 1279 iv_buffer = malloc(iv_len, M_CXGBE, M_NOWAIT); 1280 if (iv_buffer == NULL) { 1281 /* 1282 * XXX: How to restart this? 1283 */ 1284 if (sowwakeup) 1285 sowwakeup_locked(so); 1286 else 1287 SOCKBUF_UNLOCK(sb); 1288 SOCKBUF_UNLOCK_ASSERT(sb); 1289 CTR3(KTR_CXGBE, 1290 "%s: tid %d failed to alloc IV space len %d", 1291 __func__, toep->tid, iv_len); 1292 return; 1293 } 1294 } else 1295 iv_buffer = NULL; 1296 1297 /* Determine size of SGL. */ 1298 nsegs = 0; 1299 max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ 1300 if (!imm_payload) { 1301 nsegs = count_mbuf_segs(sndptr, sndptroff + 1302 TLS_HEADER_LENGTH, tls_size, &max_nsegs_1mbuf); 1303 if (!imm_ivs) { 1304 int n = sglist_count(iv_buffer, iv_len); 1305 nsegs += n; 1306 if (n > max_nsegs_1mbuf) 1307 max_nsegs_1mbuf = n; 1308 } 1309 1310 /* Account for SGL in work request length. */ 1311 wr_len += sizeof(struct ulptx_sgl) + 1312 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; 1313 } 1314 1315 wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); 1316 if (wr == NULL) { 1317 /* XXX: how will we recover from this? */ 1318 toep->flags |= TPF_TX_SUSPENDED; 1319 return; 1320 } 1321 1322#ifdef VERBOSE_TRACES 1323 CTR5(KTR_CXGBE, "%s: tid %d TLS record %d len %#x pdus %d", 1324 __func__, toep->tid, thdr.type, tls_size, pdus); 1325#endif 1326 txwr = wrtod(wr); 1327 cpl = (struct cpl_tx_tls_sfo *)(txwr + 1); 1328 memset(txwr, 0, roundup2(wr_len, 16)); 1329 credits = howmany(wr_len, 16); 1330 expn_size = tls_expansion_size(toep, tls_size, 0, NULL); 1331 write_tlstx_wr(txwr, toep, imm_payload ? tls_size : 0, 1332 tls_size, expn_size, pdus, credits, shove, imm_ivs ? 1 : 0); 1333 write_tlstx_cpl(cpl, toep, &thdr, tls_size, pdus); 1334 tls_copy_tx_key(toep, cpl + 1); 1335 1336 /* Generate random IVs */ 1337 buf = (char *)(cpl + 1) + key_size(toep); 1338 if (imm_ivs) { 1339 MPASS(iv_buffer == NULL); 1340 iv_dst = buf; 1341 buf = (char *)iv_dst + iv_len; 1342 } else 1343 iv_dst = iv_buffer; 1344 arc4rand(iv_dst, iv_len, 0); 1345 1346 if (imm_payload) { 1347 m_copydata(sndptr, sndptroff + TLS_HEADER_LENGTH, 1348 tls_size, buf); 1349 } else { 1350 write_tlstx_sgl(buf, sndptr, 1351 sndptroff + TLS_HEADER_LENGTH, tls_size, iv_buffer, 1352 iv_len, nsegs, max_nsegs_1mbuf); 1353 } 1354 1355 KASSERT(toep->tx_credits >= credits, 1356 ("%s: not enough credits", __func__)); 1357 1358 toep->tx_credits -= credits; 1359 1360 tp->snd_nxt += plen; 1361 tp->snd_max += plen; 1362 1363 SOCKBUF_LOCK(sb); 1364 sbsndptr(sb, tls_ofld->sb_off, plen, &sndptroff); 1365 tls_ofld->sb_off += plen; 1366 SOCKBUF_UNLOCK(sb); 1367 1368 toep->flags |= TPF_TX_DATA_SENT; 1369 if (toep->tx_credits < MIN_OFLD_TLSTX_CREDITS(toep)) 1370 toep->flags |= TPF_TX_SUSPENDED; 1371 1372 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); 1373 txsd->plen = plen; 1374 txsd->tx_credits = credits; 1375 txsd->iv_buffer = iv_buffer; 1376 txsd++; 1377 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { 1378 toep->txsd_pidx = 0; 1379 txsd = &toep->txsd[0]; 1380 } 1381 toep->txsd_avail--; 1382 1383 atomic_add_long(&toep->vi->pi->tx_tls_records, 1); 1384 atomic_add_long(&toep->vi->pi->tx_tls_octets, plen); 1385 1386 t4_l2t_send(sc, wr, toep->l2te); 1387 } 1388} 1389 1390/* 1391 * For TLS data we place received mbufs received via CPL_TLS_DATA into 1392 * an mbufq in the TLS offload state. When CPL_RX_TLS_CMP is 1393 * received, the completed PDUs are placed into the socket receive 1394 * buffer. 1395 * 1396 * The TLS code reuses the ulp_pdu_reclaimq to hold the pending mbufs. 1397 */ 1398static int 1399do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1400{ 1401 struct adapter *sc = iq->adapter; 1402 const struct cpl_tls_data *cpl = mtod(m, const void *); 1403 unsigned int tid = GET_TID(cpl); 1404 struct toepcb *toep = lookup_tid(sc, tid); 1405 struct inpcb *inp = toep->inp; 1406 struct tcpcb *tp; 1407 int len; 1408 1409 /* XXX: Should this match do_rx_data instead? */ 1410 KASSERT(!(toep->flags & TPF_SYNQE), 1411 ("%s: toep %p claims to be a synq entry", __func__, toep)); 1412 1413 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); 1414 1415 /* strip off CPL header */ 1416 m_adj(m, sizeof(*cpl)); 1417 len = m->m_pkthdr.len; 1418 1419 atomic_add_long(&toep->vi->pi->rx_tls_octets, len); 1420 1421 KASSERT(len == G_CPL_TLS_DATA_LENGTH(be32toh(cpl->length_pkd)), 1422 ("%s: payload length mismatch", __func__)); 1423 1424 INP_WLOCK(inp); 1425 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { 1426 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 1427 __func__, tid, len, inp->inp_flags); 1428 INP_WUNLOCK(inp); 1429 m_freem(m); 1430 return (0); 1431 } 1432 1433 /* Save TCP sequence number. */ 1434 m->m_pkthdr.tls_tcp_seq = be32toh(cpl->seq); 1435 1436 if (mbufq_enqueue(&toep->ulp_pdu_reclaimq, m)) { 1437#ifdef INVARIANTS 1438 panic("Failed to queue TLS data packet"); 1439#else 1440 printf("%s: Failed to queue TLS data packet\n", __func__); 1441 INP_WUNLOCK(inp); 1442 m_freem(m); 1443 return (0); 1444#endif 1445 } 1446 1447 tp = intotcpcb(inp); 1448 tp->t_rcvtime = ticks; 1449 1450#ifdef VERBOSE_TRACES 1451 CTR4(KTR_CXGBE, "%s: tid %u len %d seq %u", __func__, tid, len, 1452 be32toh(cpl->seq)); 1453#endif 1454 1455 INP_WUNLOCK(inp); 1456 return (0); 1457} 1458 1459static int 1460do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1461{ 1462 struct adapter *sc = iq->adapter; 1463 const struct cpl_rx_tls_cmp *cpl = mtod(m, const void *); 1464 struct tlsrx_hdr_pkt *tls_hdr_pkt; 1465 unsigned int tid = GET_TID(cpl); 1466 struct toepcb *toep = lookup_tid(sc, tid); 1467 struct inpcb *inp = toep->inp; 1468 struct tcpcb *tp; 1469 struct socket *so; 1470 struct sockbuf *sb; 1471 struct mbuf *tls_data; 1472 int len, pdu_length, rx_credits; 1473 1474 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); 1475 KASSERT(!(toep->flags & TPF_SYNQE), 1476 ("%s: toep %p claims to be a synq entry", __func__, toep)); 1477 1478 /* strip off CPL header */ 1479 m_adj(m, sizeof(*cpl)); 1480 len = m->m_pkthdr.len; 1481 1482 atomic_add_long(&toep->vi->pi->rx_tls_records, 1); 1483 1484 KASSERT(len == G_CPL_RX_TLS_CMP_LENGTH(be32toh(cpl->pdulength_length)), 1485 ("%s: payload length mismatch", __func__)); 1486 1487 INP_WLOCK(inp); 1488 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { 1489 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 1490 __func__, tid, len, inp->inp_flags); 1491 INP_WUNLOCK(inp); 1492 m_freem(m); 1493 return (0); 1494 } 1495 1496 pdu_length = G_CPL_RX_TLS_CMP_PDULENGTH(be32toh(cpl->pdulength_length)); 1497 1498 tp = intotcpcb(inp); 1499 1500#ifdef VERBOSE_TRACES 1501 CTR6(KTR_CXGBE, "%s: tid %u PDU len %d len %d seq %u, rcv_nxt %u", 1502 __func__, tid, pdu_length, len, be32toh(cpl->seq), tp->rcv_nxt); 1503#endif 1504 1505 tp->rcv_nxt += pdu_length; 1506 if (tp->rcv_wnd < pdu_length) { 1507 toep->tls.rcv_over += pdu_length - tp->rcv_wnd; 1508 tp->rcv_wnd = 0; 1509 } else 1510 tp->rcv_wnd -= pdu_length; 1511 1512 /* XXX: Not sure what to do about urgent data. */ 1513 1514 /* 1515 * The payload of this CPL is the TLS header followed by 1516 * additional fields. 1517 */ 1518 KASSERT(m->m_len >= sizeof(*tls_hdr_pkt), 1519 ("%s: payload too small", __func__)); 1520 tls_hdr_pkt = mtod(m, void *); 1521 1522 /* 1523 * Only the TLS header is sent to OpenSSL, so report errors by 1524 * altering the record type. 1525 */ 1526 if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 0) 1527 tls_hdr_pkt->type = CONTENT_TYPE_ERROR; 1528 1529 /* Trim this CPL's mbuf to only include the TLS header. */ 1530 KASSERT(m->m_len == len && m->m_next == NULL, 1531 ("%s: CPL spans multiple mbufs", __func__)); 1532 m->m_len = TLS_HEADER_LENGTH; 1533 m->m_pkthdr.len = TLS_HEADER_LENGTH; 1534 1535 tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq); 1536 if (tls_data != NULL) { 1537 KASSERT(be32toh(cpl->seq) == tls_data->m_pkthdr.tls_tcp_seq, 1538 ("%s: sequence mismatch", __func__)); 1539 1540 /* 1541 * Update the TLS header length to be the length of 1542 * the payload data. 1543 */ 1544 tls_hdr_pkt->length = htobe16(tls_data->m_pkthdr.len); 1545 1546 m->m_next = tls_data; 1547 m->m_pkthdr.len += tls_data->m_len; 1548 } 1549 1550 so = inp_inpcbtosocket(inp); 1551 sb = &so->so_rcv; 1552 SOCKBUF_LOCK(sb); 1553 1554 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { 1555 CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)", 1556 __func__, tid, pdu_length); 1557 m_freem(m); 1558 SOCKBUF_UNLOCK(sb); 1559 INP_WUNLOCK(inp); 1560 1561 CURVNET_SET(toep->vnet); 1562 INP_INFO_RLOCK(&V_tcbinfo); 1563 INP_WLOCK(inp); 1564 tp = tcp_drop(tp, ECONNRESET); 1565 if (tp) 1566 INP_WUNLOCK(inp); 1567 INP_INFO_RUNLOCK(&V_tcbinfo); 1568 CURVNET_RESTORE(); 1569 1570 return (0); 1571 } 1572 1573 /* 1574 * Not all of the bytes on the wire are included in the socket buffer 1575 * (e.g. the MAC of the TLS record). However, those bytes are included 1576 * in the TCP sequence space. 1577 */ 1578 1579 /* receive buffer autosize */ 1580 MPASS(toep->vnet == so->so_vnet); 1581 CURVNET_SET(toep->vnet); 1582 if (sb->sb_flags & SB_AUTOSIZE && 1583 V_tcp_do_autorcvbuf && 1584 sb->sb_hiwat < V_tcp_autorcvbuf_max && 1585 m->m_pkthdr.len > (sbspace(sb) / 8 * 7)) { 1586 unsigned int hiwat = sb->sb_hiwat; 1587 unsigned int newsize = min(hiwat + V_tcp_autorcvbuf_inc, 1588 V_tcp_autorcvbuf_max); 1589 1590 if (!sbreserve_locked(sb, newsize, so, NULL)) 1591 sb->sb_flags &= ~SB_AUTOSIZE; 1592 } 1593 1594 sbappendstream_locked(sb, m, 0); 1595 rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0; 1596#ifdef VERBOSE_TRACES 1597 CTR4(KTR_CXGBE, "%s: tid %u rx_credits %u rcv_wnd %u", 1598 __func__, tid, rx_credits, tp->rcv_wnd); 1599#endif 1600 if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) { 1601 rx_credits = send_rx_credits(sc, toep, rx_credits); 1602 tp->rcv_wnd += rx_credits; 1603 tp->rcv_adv += rx_credits; 1604 } 1605 1606 sorwakeup_locked(so); 1607 SOCKBUF_UNLOCK_ASSERT(sb); 1608 1609 INP_WUNLOCK(inp); 1610 CURVNET_RESTORE(); 1611 return (0); 1612} 1613 1614void 1615t4_tls_mod_load(void) 1616{ 1617 1618 mtx_init(&tls_handshake_lock, "t4tls handshake", NULL, MTX_DEF); 1619 t4_register_cpl_handler(CPL_TLS_DATA, do_tls_data); 1620 t4_register_cpl_handler(CPL_RX_TLS_CMP, do_rx_tls_cmp); 1621} 1622 1623void 1624t4_tls_mod_unload(void) 1625{ 1626 1627 t4_register_cpl_handler(CPL_TLS_DATA, NULL); 1628 t4_register_cpl_handler(CPL_RX_TLS_CMP, NULL); 1629 mtx_destroy(&tls_handshake_lock); 1630} 1631#endif /* TCP_OFFLOAD */ 1632