1// SPDX-License-Identifier: GPL-2.0-only 2/* Copyright (C) 2020 Chelsio Communications. All rights reserved. */ 3 4#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 5 6#include <linux/skbuff.h> 7#include <linux/module.h> 8#include <linux/highmem.h> 9#include <linux/ip.h> 10#include <net/ipv6.h> 11#include <linux/netdevice.h> 12#include <crypto/aes.h> 13#include "chcr_ktls.h" 14 15static LIST_HEAD(uld_ctx_list); 16static DEFINE_MUTEX(dev_mutex); 17 18/* chcr_get_nfrags_to_send: get the remaining nfrags after start offset 19 * @skb: skb 20 * @start: start offset. 21 * @len: how much data to send after @start 22 */ 23static int chcr_get_nfrags_to_send(struct sk_buff *skb, u32 start, u32 len) 24{ 25 struct skb_shared_info *si = skb_shinfo(skb); 26 u32 frag_size, skb_linear_data_len = skb_headlen(skb); 27 u8 nfrags = 0, frag_idx = 0; 28 skb_frag_t *frag; 29 30 /* if its a linear skb then return 1 */ 31 if (!skb_is_nonlinear(skb)) 32 return 1; 33 34 if (unlikely(start < skb_linear_data_len)) { 35 frag_size = min(len, skb_linear_data_len - start); 36 } else { 37 start -= skb_linear_data_len; 38 39 frag = &si->frags[frag_idx]; 40 frag_size = skb_frag_size(frag); 41 while (start >= frag_size) { 42 start -= frag_size; 43 frag_idx++; 44 frag = &si->frags[frag_idx]; 45 frag_size = skb_frag_size(frag); 46 } 47 frag_size = min(len, skb_frag_size(frag) - start); 48 } 49 len -= frag_size; 50 nfrags++; 51 52 while (len) { 53 frag_size = min(len, skb_frag_size(&si->frags[frag_idx])); 54 len -= frag_size; 55 nfrags++; 56 frag_idx++; 57 } 58 return nfrags; 59} 60 61static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info); 62static void clear_conn_resources(struct chcr_ktls_info *tx_info); 63/* 64 * chcr_ktls_save_keys: calculate and save crypto keys. 65 * @tx_info - driver specific tls info. 66 * @crypto_info - tls crypto information. 67 * @direction - TX/RX direction. 68 * return - SUCCESS/FAILURE. 69 */ 70static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info, 71 struct tls_crypto_info *crypto_info, 72 enum tls_offload_ctx_dir direction) 73{ 74 int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret; 75 unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE]; 76 struct tls12_crypto_info_aes_gcm_128 *info_128_gcm; 77 struct ktls_key_ctx *kctx = &tx_info->key_ctx; 78 struct crypto_aes_ctx aes_ctx; 79 unsigned char *key, *salt; 80 81 switch (crypto_info->cipher_type) { 82 case TLS_CIPHER_AES_GCM_128: 83 info_128_gcm = 84 (struct tls12_crypto_info_aes_gcm_128 *)crypto_info; 85 keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE; 86 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 87 tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE; 88 mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; 89 tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; 90 tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv); 91 92 ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE; 93 key = info_128_gcm->key; 94 salt = info_128_gcm->salt; 95 tx_info->record_no = *(u64 *)info_128_gcm->rec_seq; 96 97 /* The SCMD fields used when encrypting a full TLS 98 * record. Its a one time calculation till the 99 * connection exists. 100 */ 101 tx_info->scmd0_seqno_numivs = 102 SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT) | 103 SCMD_CIPH_AUTH_SEQ_CTRL_F | 104 SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS) | 105 SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM) | 106 SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH) | 107 SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE >> 1) | 108 SCMD_NUM_IVS_V(1); 109 110 /* keys will be sent inline. */ 111 tx_info->scmd0_ivgen_hdrlen = SCMD_KEY_CTX_INLINE_F; 112 113 /* The SCMD fields used when encrypting a partial TLS 114 * record (no trailer and possibly a truncated payload). 115 */ 116 tx_info->scmd0_short_seqno_numivs = 117 SCMD_CIPH_AUTH_SEQ_CTRL_F | 118 SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) | 119 SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR) | 120 SCMD_IV_SIZE_V(AES_BLOCK_LEN >> 1); 121 122 tx_info->scmd0_short_ivgen_hdrlen = 123 tx_info->scmd0_ivgen_hdrlen | SCMD_AADIVDROP_F; 124 125 break; 126 127 default: 128 pr_err("GCM: cipher type 0x%x not supported\n", 129 crypto_info->cipher_type); 130 ret = -EINVAL; 131 goto out; 132 } 133 134 key_ctx_size = CHCR_KTLS_KEY_CTX_LEN + 135 roundup(keylen, 16) + ghash_size; 136 /* Calculate the H = CIPH(K, 0 repeated 16 times). 137 * It will go in key context 138 */ 139 140 ret = aes_expandkey(&aes_ctx, key, keylen); 141 if (ret) 142 goto out; 143 144 memset(ghash_h, 0, ghash_size); 145 aes_encrypt(&aes_ctx, ghash_h, ghash_h); 146 memzero_explicit(&aes_ctx, sizeof(aes_ctx)); 147 148 /* fill the Key context */ 149 if (direction == TLS_OFFLOAD_CTX_DIR_TX) { 150 kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size, 151 mac_key_size, 152 key_ctx_size >> 4); 153 } else { 154 ret = -EINVAL; 155 goto out; 156 } 157 158 memcpy(kctx->salt, salt, tx_info->salt_size); 159 memcpy(kctx->key, key, keylen); 160 memcpy(kctx->key + keylen, ghash_h, ghash_size); 161 tx_info->key_ctx_len = key_ctx_size; 162 163out: 164 return ret; 165} 166 167/* 168 * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection. 169 * @sk - tcp socket. 170 * @tx_info - driver specific tls info. 171 * @atid - connection active tid. 172 * return - send success/failure. 173 */ 174static int chcr_ktls_act_open_req(struct sock *sk, 175 struct chcr_ktls_info *tx_info, 176 int atid) 177{ 178 struct inet_sock *inet = inet_sk(sk); 179 struct cpl_t6_act_open_req *cpl6; 180 struct cpl_act_open_req *cpl; 181 struct sk_buff *skb; 182 unsigned int len; 183 int qid_atid; 184 u64 options; 185 186 len = sizeof(*cpl6); 187 skb = alloc_skb(len, GFP_KERNEL); 188 if (unlikely(!skb)) 189 return -ENOMEM; 190 /* mark it a control pkt */ 191 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id); 192 193 cpl6 = __skb_put_zero(skb, len); 194 cpl = (struct cpl_act_open_req *)cpl6; 195 INIT_TP_WR(cpl6, 0); 196 qid_atid = TID_QID_V(tx_info->rx_qid) | 197 TID_TID_V(atid); 198 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid)); 199 cpl->local_port = inet->inet_sport; 200 cpl->peer_port = inet->inet_dport; 201 cpl->local_ip = inet->inet_rcv_saddr; 202 cpl->peer_ip = inet->inet_daddr; 203 204 /* fill first 64 bit option field. */ 205 options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F | 206 SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan); 207 cpl->opt0 = cpu_to_be64(options); 208 209 /* next 64 bit option field. */ 210 options = 211 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]); 212 cpl->opt2 = htonl(options); 213 214 return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te); 215} 216 217#if IS_ENABLED(CONFIG_IPV6) 218/* 219 * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection. 220 * @sk - tcp socket. 221 * @tx_info - driver specific tls info. 222 * @atid - connection active tid. 223 * return - send success/failure. 224 */ 225static int chcr_ktls_act_open_req6(struct sock *sk, 226 struct chcr_ktls_info *tx_info, 227 int atid) 228{ 229 struct inet_sock *inet = inet_sk(sk); 230 struct cpl_t6_act_open_req6 *cpl6; 231 struct cpl_act_open_req6 *cpl; 232 struct sk_buff *skb; 233 unsigned int len; 234 int qid_atid; 235 u64 options; 236 237 len = sizeof(*cpl6); 238 skb = alloc_skb(len, GFP_KERNEL); 239 if (unlikely(!skb)) 240 return -ENOMEM; 241 /* mark it a control pkt */ 242 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id); 243 244 cpl6 = __skb_put_zero(skb, len); 245 cpl = (struct cpl_act_open_req6 *)cpl6; 246 INIT_TP_WR(cpl6, 0); 247 qid_atid = TID_QID_V(tx_info->rx_qid) | TID_TID_V(atid); 248 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_atid)); 249 cpl->local_port = inet->inet_sport; 250 cpl->peer_port = inet->inet_dport; 251 cpl->local_ip_hi = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[0]; 252 cpl->local_ip_lo = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[8]; 253 cpl->peer_ip_hi = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[0]; 254 cpl->peer_ip_lo = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[8]; 255 256 /* first 64 bit option field. */ 257 options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F | 258 SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan); 259 cpl->opt0 = cpu_to_be64(options); 260 /* next 64 bit option field. */ 261 options = 262 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]); 263 cpl->opt2 = htonl(options); 264 265 return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te); 266} 267#endif /* #if IS_ENABLED(CONFIG_IPV6) */ 268 269/* 270 * chcr_setup_connection: create a TCB entry so that TP will form tcp packets. 271 * @sk - tcp socket. 272 * @tx_info - driver specific tls info. 273 * return: NET_TX_OK/NET_XMIT_DROP 274 */ 275static int chcr_setup_connection(struct sock *sk, 276 struct chcr_ktls_info *tx_info) 277{ 278 struct tid_info *t = &tx_info->adap->tids; 279 int atid, ret = 0; 280 281 atid = cxgb4_alloc_atid(t, tx_info); 282 if (atid == -1) 283 return -EINVAL; 284 285 tx_info->atid = atid; 286 287 if (tx_info->ip_family == AF_INET) { 288 ret = chcr_ktls_act_open_req(sk, tx_info, atid); 289#if IS_ENABLED(CONFIG_IPV6) 290 } else { 291 ret = cxgb4_clip_get(tx_info->netdev, (const u32 *) 292 &sk->sk_v6_rcv_saddr, 293 1); 294 if (ret) 295 return ret; 296 ret = chcr_ktls_act_open_req6(sk, tx_info, atid); 297#endif 298 } 299 300 /* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret 301 * success, if any other return type clear atid and return that failure. 302 */ 303 if (ret) { 304 if (ret == NET_XMIT_CN) { 305 ret = 0; 306 } else { 307#if IS_ENABLED(CONFIG_IPV6) 308 /* clear clip entry */ 309 if (tx_info->ip_family == AF_INET6) 310 cxgb4_clip_release(tx_info->netdev, 311 (const u32 *) 312 &sk->sk_v6_rcv_saddr, 313 1); 314#endif 315 cxgb4_free_atid(t, atid); 316 } 317 } 318 319 return ret; 320} 321 322/* 323 * chcr_set_tcb_field: update tcb fields. 324 * @tx_info - driver specific tls info. 325 * @word - TCB word. 326 * @mask - TCB word related mask. 327 * @val - TCB word related value. 328 * @no_reply - set 1 if not looking for TP response. 329 */ 330static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word, 331 u64 mask, u64 val, int no_reply) 332{ 333 struct cpl_set_tcb_field *req; 334 struct sk_buff *skb; 335 336 skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC); 337 if (!skb) 338 return -ENOMEM; 339 340 req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req)); 341 INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid); 342 req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) | 343 NO_REPLY_V(no_reply)); 344 req->word_cookie = htons(TCB_WORD_V(word)); 345 req->mask = cpu_to_be64(mask); 346 req->val = cpu_to_be64(val); 347 348 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id); 349 return cxgb4_ofld_send(tx_info->netdev, skb); 350} 351 352/* 353 * chcr_ktls_dev_del: call back for tls_dev_del. 354 * Remove the tid and l2t entry and close the connection. 355 * it per connection basis. 356 * @netdev - net device. 357 * @tls_cts - tls context. 358 * @direction - TX/RX crypto direction 359 */ 360static void chcr_ktls_dev_del(struct net_device *netdev, 361 struct tls_context *tls_ctx, 362 enum tls_offload_ctx_dir direction) 363{ 364 struct chcr_ktls_info *tx_info = chcr_get_ktls_tx_info(tls_ctx); 365 struct ch_ktls_port_stats_debug *port_stats; 366 struct chcr_ktls_uld_ctx *u_ctx; 367 368 if (!tx_info) 369 return; 370 371 u_ctx = tx_info->adap->uld[CXGB4_ULD_KTLS].handle; 372 if (u_ctx && u_ctx->detach) 373 return; 374 /* clear l2t entry */ 375 if (tx_info->l2te) 376 cxgb4_l2t_release(tx_info->l2te); 377 378#if IS_ENABLED(CONFIG_IPV6) 379 /* clear clip entry */ 380 if (tx_info->ip_family == AF_INET6) 381 cxgb4_clip_release(netdev, (const u32 *) 382 &tx_info->sk->sk_v6_rcv_saddr, 383 1); 384#endif 385 386 /* clear tid */ 387 if (tx_info->tid != -1) { 388 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan, 389 tx_info->tid, tx_info->ip_family); 390 391 xa_erase(&u_ctx->tid_list, tx_info->tid); 392 } 393 394 port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id]; 395 atomic64_inc(&port_stats->ktls_tx_connection_close); 396 kvfree(tx_info); 397 chcr_set_ktls_tx_info(tls_ctx, NULL); 398 /* release module refcount */ 399 module_put(THIS_MODULE); 400} 401 402/* 403 * chcr_ktls_dev_add: call back for tls_dev_add. 404 * Create a tcb entry for TP. Also add l2t entry for the connection. And 405 * generate keys & save those keys locally. 406 * @netdev - net device. 407 * @tls_cts - tls context. 408 * @direction - TX/RX crypto direction 409 * return: SUCCESS/FAILURE. 410 */ 411static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, 412 enum tls_offload_ctx_dir direction, 413 struct tls_crypto_info *crypto_info, 414 u32 start_offload_tcp_sn) 415{ 416 struct tls_context *tls_ctx = tls_get_ctx(sk); 417 struct ch_ktls_port_stats_debug *port_stats; 418 struct chcr_ktls_uld_ctx *u_ctx; 419 struct chcr_ktls_info *tx_info; 420 struct dst_entry *dst; 421 struct adapter *adap; 422 struct port_info *pi; 423 struct neighbour *n; 424 u8 daaddr[16]; 425 int ret = -1; 426 427 pi = netdev_priv(netdev); 428 adap = pi->adapter; 429 port_stats = &adap->ch_ktls_stats.ktls_port[pi->port_id]; 430 atomic64_inc(&port_stats->ktls_tx_connection_open); 431 u_ctx = adap->uld[CXGB4_ULD_KTLS].handle; 432 433 if (direction == TLS_OFFLOAD_CTX_DIR_RX) { 434 pr_err("not expecting for RX direction\n"); 435 goto out; 436 } 437 438 if (chcr_get_ktls_tx_info(tls_ctx)) 439 goto out; 440 441 if (u_ctx && u_ctx->detach) 442 goto out; 443 444 tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL); 445 if (!tx_info) 446 goto out; 447 448 tx_info->sk = sk; 449 spin_lock_init(&tx_info->lock); 450 /* initialize tid and atid to -1, 0 is a also a valid id. */ 451 tx_info->tid = -1; 452 tx_info->atid = -1; 453 454 tx_info->adap = adap; 455 tx_info->netdev = netdev; 456 tx_info->first_qset = pi->first_qset; 457 tx_info->tx_chan = pi->tx_chan; 458 tx_info->smt_idx = pi->smt_idx; 459 tx_info->port_id = pi->port_id; 460 tx_info->prev_ack = 0; 461 tx_info->prev_win = 0; 462 463 tx_info->rx_qid = chcr_get_first_rx_qid(adap); 464 if (unlikely(tx_info->rx_qid < 0)) 465 goto free_tx_info; 466 467 tx_info->prev_seq = start_offload_tcp_sn; 468 tx_info->tcp_start_seq_number = start_offload_tcp_sn; 469 470 /* save crypto keys */ 471 ret = chcr_ktls_save_keys(tx_info, crypto_info, direction); 472 if (ret < 0) 473 goto free_tx_info; 474 475 /* get peer ip */ 476 if (sk->sk_family == AF_INET) { 477 memcpy(daaddr, &sk->sk_daddr, 4); 478 tx_info->ip_family = AF_INET; 479#if IS_ENABLED(CONFIG_IPV6) 480 } else { 481 if (!ipv6_only_sock(sk) && 482 ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) { 483 memcpy(daaddr, &sk->sk_daddr, 4); 484 tx_info->ip_family = AF_INET; 485 } else { 486 memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16); 487 tx_info->ip_family = AF_INET6; 488 } 489#endif 490 } 491 492 /* get the l2t index */ 493 dst = sk_dst_get(sk); 494 if (!dst) { 495 pr_err("DST entry not found\n"); 496 goto free_tx_info; 497 } 498 n = dst_neigh_lookup(dst, daaddr); 499 if (!n || !n->dev) { 500 pr_err("neighbour not found\n"); 501 dst_release(dst); 502 goto free_tx_info; 503 } 504 tx_info->l2te = cxgb4_l2t_get(adap->l2t, n, n->dev, 0); 505 506 neigh_release(n); 507 dst_release(dst); 508 509 if (!tx_info->l2te) { 510 pr_err("l2t entry not found\n"); 511 goto free_tx_info; 512 } 513 514 /* Driver shouldn't be removed until any single connection exists */ 515 if (!try_module_get(THIS_MODULE)) 516 goto free_l2t; 517 518 init_completion(&tx_info->completion); 519 /* create a filter and call cxgb4_l2t_send to send the packet out, which 520 * will take care of updating l2t entry in hw if not already done. 521 */ 522 tx_info->open_state = CH_KTLS_OPEN_PENDING; 523 524 if (chcr_setup_connection(sk, tx_info)) 525 goto put_module; 526 527 /* Wait for reply */ 528 wait_for_completion_timeout(&tx_info->completion, 30 * HZ); 529 spin_lock_bh(&tx_info->lock); 530 if (tx_info->open_state) { 531 /* need to wait for hw response, can't free tx_info yet. */ 532 if (tx_info->open_state == CH_KTLS_OPEN_PENDING) 533 tx_info->pending_close = true; 534 else 535 spin_unlock_bh(&tx_info->lock); 536 /* if in pending close, free the lock after the cleanup */ 537 goto put_module; 538 } 539 spin_unlock_bh(&tx_info->lock); 540 541 /* initialize tcb */ 542 reinit_completion(&tx_info->completion); 543 /* mark it pending for hw response */ 544 tx_info->open_state = CH_KTLS_OPEN_PENDING; 545 546 if (chcr_init_tcb_fields(tx_info)) 547 goto free_tid; 548 549 /* Wait for reply */ 550 wait_for_completion_timeout(&tx_info->completion, 30 * HZ); 551 spin_lock_bh(&tx_info->lock); 552 if (tx_info->open_state) { 553 /* need to wait for hw response, can't free tx_info yet. */ 554 tx_info->pending_close = true; 555 /* free the lock after cleanup */ 556 goto free_tid; 557 } 558 spin_unlock_bh(&tx_info->lock); 559 560 if (!cxgb4_check_l2t_valid(tx_info->l2te)) 561 goto free_tid; 562 563 atomic64_inc(&port_stats->ktls_tx_ctx); 564 chcr_set_ktls_tx_info(tls_ctx, tx_info); 565 566 return 0; 567 568free_tid: 569#if IS_ENABLED(CONFIG_IPV6) 570 /* clear clip entry */ 571 if (tx_info->ip_family == AF_INET6) 572 cxgb4_clip_release(netdev, (const u32 *) 573 &sk->sk_v6_rcv_saddr, 574 1); 575#endif 576 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan, 577 tx_info->tid, tx_info->ip_family); 578 579 xa_erase(&u_ctx->tid_list, tx_info->tid); 580 581put_module: 582 /* release module refcount */ 583 module_put(THIS_MODULE); 584free_l2t: 585 cxgb4_l2t_release(tx_info->l2te); 586free_tx_info: 587 if (tx_info->pending_close) 588 spin_unlock_bh(&tx_info->lock); 589 else 590 kvfree(tx_info); 591out: 592 atomic64_inc(&port_stats->ktls_tx_connection_fail); 593 return -1; 594} 595 596/* 597 * chcr_init_tcb_fields: Initialize tcb fields to handle TCP seq number 598 * handling. 599 * @tx_info - driver specific tls info. 600 * return: NET_TX_OK/NET_XMIT_DROP 601 */ 602static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info) 603{ 604 int ret = 0; 605 606 /* set tcb in offload and bypass */ 607 ret = 608 chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W, 609 TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F), 610 TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1); 611 if (ret) 612 return ret; 613 /* reset snd_una and snd_next fields in tcb */ 614 ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W, 615 TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) | 616 TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M), 617 0, 1); 618 if (ret) 619 return ret; 620 621 /* reset send max */ 622 ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W, 623 TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M), 624 0, 1); 625 if (ret) 626 return ret; 627 628 /* update l2t index and request for tp reply to confirm tcb is 629 * initialised to handle tx traffic. 630 */ 631 ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W, 632 TCB_L2T_IX_V(TCB_L2T_IX_M), 633 TCB_L2T_IX_V(tx_info->l2te->idx), 0); 634 return ret; 635} 636 637/* 638 * chcr_ktls_cpl_act_open_rpl: connection reply received from TP. 639 */ 640static int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, 641 unsigned char *input) 642{ 643 const struct cpl_act_open_rpl *p = (void *)input; 644 struct chcr_ktls_info *tx_info = NULL; 645 struct tls_offload_context_tx *tx_ctx; 646 struct chcr_ktls_uld_ctx *u_ctx; 647 unsigned int atid, tid, status; 648 struct tls_context *tls_ctx; 649 struct tid_info *t; 650 int ret = 0; 651 652 tid = GET_TID(p); 653 status = AOPEN_STATUS_G(ntohl(p->atid_status)); 654 atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status))); 655 656 t = &adap->tids; 657 tx_info = lookup_atid(t, atid); 658 659 if (!tx_info || tx_info->atid != atid) { 660 pr_err("%s: incorrect tx_info or atid\n", __func__); 661 return -1; 662 } 663 664 cxgb4_free_atid(t, atid); 665 tx_info->atid = -1; 666 667 spin_lock(&tx_info->lock); 668 /* HW response is very close, finish pending cleanup */ 669 if (tx_info->pending_close) { 670 spin_unlock(&tx_info->lock); 671 if (!status) { 672 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan, 673 tid, tx_info->ip_family); 674 } 675 kvfree(tx_info); 676 return 0; 677 } 678 679 if (!status) { 680 tx_info->tid = tid; 681 cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family); 682 /* Adding tid */ 683 tls_ctx = tls_get_ctx(tx_info->sk); 684 tx_ctx = tls_offload_ctx_tx(tls_ctx); 685 u_ctx = adap->uld[CXGB4_ULD_KTLS].handle; 686 if (u_ctx) { 687 ret = xa_insert_bh(&u_ctx->tid_list, tid, tx_ctx, 688 GFP_NOWAIT); 689 if (ret < 0) { 690 pr_err("%s: Failed to allocate tid XA entry = %d\n", 691 __func__, tx_info->tid); 692 tx_info->open_state = CH_KTLS_OPEN_FAILURE; 693 goto out; 694 } 695 } 696 tx_info->open_state = CH_KTLS_OPEN_SUCCESS; 697 } else { 698 tx_info->open_state = CH_KTLS_OPEN_FAILURE; 699 } 700out: 701 spin_unlock(&tx_info->lock); 702 703 complete(&tx_info->completion); 704 return ret; 705} 706 707/* 708 * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP. 709 */ 710static int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input) 711{ 712 const struct cpl_set_tcb_rpl *p = (void *)input; 713 struct chcr_ktls_info *tx_info = NULL; 714 struct tid_info *t; 715 u32 tid; 716 717 tid = GET_TID(p); 718 719 t = &adap->tids; 720 tx_info = lookup_tid(t, tid); 721 722 if (!tx_info || tx_info->tid != tid) { 723 pr_err("%s: incorrect tx_info or tid\n", __func__); 724 return -1; 725 } 726 727 spin_lock(&tx_info->lock); 728 if (tx_info->pending_close) { 729 spin_unlock(&tx_info->lock); 730 kvfree(tx_info); 731 return 0; 732 } 733 tx_info->open_state = CH_KTLS_OPEN_SUCCESS; 734 spin_unlock(&tx_info->lock); 735 736 complete(&tx_info->completion); 737 return 0; 738} 739 740static void *__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info, 741 u32 tid, void *pos, u16 word, 742 struct sge_eth_txq *q, u64 mask, 743 u64 val, u32 reply) 744{ 745 struct cpl_set_tcb_field_core *cpl; 746 struct ulptx_idata *idata; 747 struct ulp_txpkt *txpkt; 748 749 /* ULP_TXPKT */ 750 txpkt = pos; 751 txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | 752 ULP_TXPKT_CHANNELID_V(tx_info->port_id) | 753 ULP_TXPKT_FID_V(q->q.cntxt_id) | 754 ULP_TXPKT_RO_F); 755 txpkt->len = htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN, 16)); 756 757 /* ULPTX_IDATA sub-command */ 758 idata = (struct ulptx_idata *)(txpkt + 1); 759 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM)); 760 idata->len = htonl(sizeof(*cpl)); 761 pos = idata + 1; 762 763 cpl = pos; 764 /* CPL_SET_TCB_FIELD */ 765 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 766 cpl->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) | 767 NO_REPLY_V(!reply)); 768 cpl->word_cookie = htons(TCB_WORD_V(word)); 769 cpl->mask = cpu_to_be64(mask); 770 cpl->val = cpu_to_be64(val); 771 772 /* ULPTX_NOOP */ 773 idata = (struct ulptx_idata *)(cpl + 1); 774 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP)); 775 idata->len = htonl(0); 776 pos = idata + 1; 777 778 return pos; 779} 780 781 782/* 783 * chcr_write_cpl_set_tcb_ulp: update tcb values. 784 * TCB is responsible to create tcp headers, so all the related values 785 * should be correctly updated. 786 * @tx_info - driver specific tls info. 787 * @q - tx queue on which packet is going out. 788 * @tid - TCB identifier. 789 * @pos - current index where should we start writing. 790 * @word - TCB word. 791 * @mask - TCB word related mask. 792 * @val - TCB word related value. 793 * @reply - set 1 if looking for TP response. 794 * return - next position to write. 795 */ 796static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info, 797 struct sge_eth_txq *q, u32 tid, 798 void *pos, u16 word, u64 mask, 799 u64 val, u32 reply) 800{ 801 int left = (void *)q->q.stat - pos; 802 803 if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) { 804 if (!left) { 805 pos = q->q.desc; 806 } else { 807 u8 buf[48] = {0}; 808 809 __chcr_write_cpl_set_tcb_ulp(tx_info, tid, buf, word, q, 810 mask, val, reply); 811 812 return chcr_copy_to_txd(buf, &q->q, pos, 813 CHCR_SET_TCB_FIELD_LEN); 814 } 815 } 816 817 pos = __chcr_write_cpl_set_tcb_ulp(tx_info, tid, pos, word, q, 818 mask, val, reply); 819 820 /* check again if we are at the end of the queue */ 821 if (left == CHCR_SET_TCB_FIELD_LEN) 822 pos = q->q.desc; 823 824 return pos; 825} 826 827/* 828 * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header 829 * with updated values like tcp seq, ack, window etc. 830 * @tx_info - driver specific tls info. 831 * @q - TX queue. 832 * @tcp_seq 833 * @tcp_ack 834 * @tcp_win 835 * return: NETDEV_TX_BUSY/NET_TX_OK. 836 */ 837static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info, 838 struct sge_eth_txq *q, u64 tcp_seq, 839 u64 tcp_ack, u64 tcp_win, bool offset) 840{ 841 bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0)); 842 struct ch_ktls_port_stats_debug *port_stats; 843 u32 len, cpl = 0, ndesc, wr_len, wr_mid = 0; 844 struct fw_ulptx_wr *wr; 845 int credits; 846 void *pos; 847 848 wr_len = sizeof(*wr); 849 /* there can be max 4 cpls, check if we have enough credits */ 850 len = wr_len + 4 * roundup(CHCR_SET_TCB_FIELD_LEN, 16); 851 ndesc = DIV_ROUND_UP(len, 64); 852 853 credits = chcr_txq_avail(&q->q) - ndesc; 854 if (unlikely(credits < 0)) { 855 chcr_eth_txq_stop(q); 856 return NETDEV_TX_BUSY; 857 } 858 859 if (unlikely(credits < ETHTXQ_STOP_THRES)) { 860 chcr_eth_txq_stop(q); 861 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; 862 } 863 864 pos = &q->q.desc[q->q.pidx]; 865 /* make space for WR, we'll fill it later when we know all the cpls 866 * being sent out and have complete length. 867 */ 868 wr = pos; 869 pos += wr_len; 870 /* update tx_max if its a re-transmit or the first wr */ 871 if (first_wr || tcp_seq != tx_info->prev_seq) { 872 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, 873 TCB_TX_MAX_W, 874 TCB_TX_MAX_V(TCB_TX_MAX_M), 875 TCB_TX_MAX_V(tcp_seq), 0); 876 cpl++; 877 } 878 /* reset snd una if it's a re-transmit pkt */ 879 if (tcp_seq != tx_info->prev_seq || offset) { 880 /* reset snd_una */ 881 port_stats = 882 &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id]; 883 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, 884 TCB_SND_UNA_RAW_W, 885 TCB_SND_UNA_RAW_V 886 (TCB_SND_UNA_RAW_M), 887 TCB_SND_UNA_RAW_V(0), 0); 888 if (tcp_seq != tx_info->prev_seq) 889 atomic64_inc(&port_stats->ktls_tx_ooo); 890 cpl++; 891 } 892 /* update ack */ 893 if (first_wr || tx_info->prev_ack != tcp_ack) { 894 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, 895 TCB_RCV_NXT_W, 896 TCB_RCV_NXT_V(TCB_RCV_NXT_M), 897 TCB_RCV_NXT_V(tcp_ack), 0); 898 tx_info->prev_ack = tcp_ack; 899 cpl++; 900 } 901 /* update receive window */ 902 if (first_wr || tx_info->prev_win != tcp_win) { 903 chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, 904 TCB_RCV_WND_W, 905 TCB_RCV_WND_V(TCB_RCV_WND_M), 906 TCB_RCV_WND_V(tcp_win), 0); 907 tx_info->prev_win = tcp_win; 908 cpl++; 909 } 910 911 if (cpl) { 912 /* get the actual length */ 913 len = wr_len + cpl * roundup(CHCR_SET_TCB_FIELD_LEN, 16); 914 /* ULPTX wr */ 915 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); 916 wr->cookie = 0; 917 /* fill len in wr field */ 918 wr->flowid_len16 = htonl(wr_mid | 919 FW_WR_LEN16_V(DIV_ROUND_UP(len, 16))); 920 921 ndesc = DIV_ROUND_UP(len, 64); 922 chcr_txq_advance(&q->q, ndesc); 923 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); 924 } 925 return 0; 926} 927 928/* 929 * chcr_ktls_get_tx_flits 930 * returns number of flits to be sent out, it includes key context length, WR 931 * size and skb fragments. 932 */ 933static unsigned int 934chcr_ktls_get_tx_flits(u32 nr_frags, unsigned int key_ctx_len) 935{ 936 return chcr_sgl_len(nr_frags) + 937 DIV_ROUND_UP(key_ctx_len + CHCR_KTLS_WR_SIZE, 8); 938} 939 940/* 941 * chcr_ktls_check_tcp_options: To check if there is any TCP option available 942 * other than timestamp. 943 * @skb - skb contains partial record.. 944 * return: 1 / 0 945 */ 946static int 947chcr_ktls_check_tcp_options(struct tcphdr *tcp) 948{ 949 int cnt, opt, optlen; 950 u_char *cp; 951 952 cp = (u_char *)(tcp + 1); 953 cnt = (tcp->doff << 2) - sizeof(struct tcphdr); 954 for (; cnt > 0; cnt -= optlen, cp += optlen) { 955 opt = cp[0]; 956 if (opt == TCPOPT_EOL) 957 break; 958 if (opt == TCPOPT_NOP) { 959 optlen = 1; 960 } else { 961 if (cnt < 2) 962 break; 963 optlen = cp[1]; 964 if (optlen < 2 || optlen > cnt) 965 break; 966 } 967 switch (opt) { 968 case TCPOPT_NOP: 969 break; 970 default: 971 return 1; 972 } 973 } 974 return 0; 975} 976 977/* 978 * chcr_ktls_write_tcp_options : TP can't send out all the options, we need to 979 * send out separately. 980 * @tx_info - driver specific tls info. 981 * @skb - skb contains partial record.. 982 * @q - TX queue. 983 * @tx_chan - channel number. 984 * return: NETDEV_TX_OK/NETDEV_TX_BUSY. 985 */ 986static int 987chcr_ktls_write_tcp_options(struct chcr_ktls_info *tx_info, struct sk_buff *skb, 988 struct sge_eth_txq *q, uint32_t tx_chan) 989{ 990 struct fw_eth_tx_pkt_wr *wr; 991 struct cpl_tx_pkt_core *cpl; 992 u32 ctrl, iplen, maclen; 993 struct ipv6hdr *ip6; 994 unsigned int ndesc; 995 struct tcphdr *tcp; 996 int len16, pktlen; 997 struct iphdr *ip; 998 u32 wr_mid = 0; 999 int credits; 1000 u8 buf[150]; 1001 u64 cntrl1; 1002 void *pos; 1003 1004 iplen = skb_network_header_len(skb); 1005 maclen = skb_mac_header_len(skb); 1006 1007 /* packet length = eth hdr len + ip hdr len + tcp hdr len 1008 * (including options). 1009 */ 1010 pktlen = skb_tcp_all_headers(skb); 1011 1012 ctrl = sizeof(*cpl) + pktlen; 1013 len16 = DIV_ROUND_UP(sizeof(*wr) + ctrl, 16); 1014 /* check how many descriptors needed */ 1015 ndesc = DIV_ROUND_UP(len16, 4); 1016 1017 credits = chcr_txq_avail(&q->q) - ndesc; 1018 if (unlikely(credits < 0)) { 1019 chcr_eth_txq_stop(q); 1020 return NETDEV_TX_BUSY; 1021 } 1022 1023 if (unlikely(credits < ETHTXQ_STOP_THRES)) { 1024 chcr_eth_txq_stop(q); 1025 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; 1026 } 1027 1028 pos = &q->q.desc[q->q.pidx]; 1029 wr = pos; 1030 1031 /* Firmware work request header */ 1032 wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) | 1033 FW_WR_IMMDLEN_V(ctrl)); 1034 1035 wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16)); 1036 wr->r3 = 0; 1037 1038 cpl = (void *)(wr + 1); 1039 1040 /* CPL header */ 1041 cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) | TXPKT_INTF_V(tx_chan) | 1042 TXPKT_PF_V(tx_info->adap->pf)); 1043 cpl->pack = 0; 1044 cpl->len = htons(pktlen); 1045 1046 memcpy(buf, skb->data, pktlen); 1047 if (!IS_ENABLED(CONFIG_IPV6) || tx_info->ip_family == AF_INET) { 1048 /* we need to correct ip header len */ 1049 ip = (struct iphdr *)(buf + maclen); 1050 ip->tot_len = htons(pktlen - maclen); 1051 cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP); 1052 } else { 1053 ip6 = (struct ipv6hdr *)(buf + maclen); 1054 ip6->payload_len = htons(pktlen - maclen - iplen); 1055 cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP6); 1056 } 1057 1058 cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) | 1059 TXPKT_IPHDR_LEN_V(iplen); 1060 /* checksum offload */ 1061 cpl->ctrl1 = cpu_to_be64(cntrl1); 1062 1063 pos = cpl + 1; 1064 1065 /* now take care of the tcp header, if fin is not set then clear push 1066 * bit as well, and if fin is set, it will be sent at the last so we 1067 * need to update the tcp sequence number as per the last packet. 1068 */ 1069 tcp = (struct tcphdr *)(buf + maclen + iplen); 1070 1071 if (!tcp->fin) 1072 tcp->psh = 0; 1073 else 1074 tcp->seq = htonl(tx_info->prev_seq); 1075 1076 chcr_copy_to_txd(buf, &q->q, pos, pktlen); 1077 1078 chcr_txq_advance(&q->q, ndesc); 1079 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); 1080 return 0; 1081} 1082 1083/* 1084 * chcr_ktls_xmit_wr_complete: This sends out the complete record. If an skb 1085 * received has partial end part of the record, send out the complete record, so 1086 * that crypto block will be able to generate TAG/HASH. 1087 * @skb - segment which has complete or partial end part. 1088 * @tx_info - driver specific tls info. 1089 * @q - TX queue. 1090 * @tcp_seq 1091 * @tcp_push - tcp push bit. 1092 * @mss - segment size. 1093 * return: NETDEV_TX_BUSY/NET_TX_OK. 1094 */ 1095static int chcr_ktls_xmit_wr_complete(struct sk_buff *skb, 1096 struct chcr_ktls_info *tx_info, 1097 struct sge_eth_txq *q, u32 tcp_seq, 1098 bool is_last_wr, u32 data_len, 1099 u32 skb_offset, u32 nfrags, 1100 bool tcp_push, u32 mss) 1101{ 1102 u32 len16, wr_mid = 0, flits = 0, ndesc, cipher_start; 1103 struct adapter *adap = tx_info->adap; 1104 int credits, left, last_desc; 1105 struct tx_sw_desc *sgl_sdesc; 1106 struct cpl_tx_data *tx_data; 1107 struct cpl_tx_sec_pdu *cpl; 1108 struct ulptx_idata *idata; 1109 struct ulp_txpkt *ulptx; 1110 struct fw_ulptx_wr *wr; 1111 void *pos; 1112 u64 *end; 1113 1114 /* get the number of flits required */ 1115 flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len); 1116 /* number of descriptors */ 1117 ndesc = chcr_flits_to_desc(flits); 1118 /* check if enough credits available */ 1119 credits = chcr_txq_avail(&q->q) - ndesc; 1120 if (unlikely(credits < 0)) { 1121 chcr_eth_txq_stop(q); 1122 return NETDEV_TX_BUSY; 1123 } 1124 1125 if (unlikely(credits < ETHTXQ_STOP_THRES)) { 1126 /* Credits are below the threshold values, stop the queue after 1127 * injecting the Work Request for this packet. 1128 */ 1129 chcr_eth_txq_stop(q); 1130 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; 1131 } 1132 1133 last_desc = q->q.pidx + ndesc - 1; 1134 if (last_desc >= q->q.size) 1135 last_desc -= q->q.size; 1136 sgl_sdesc = &q->q.sdesc[last_desc]; 1137 1138 if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) { 1139 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); 1140 q->mapping_err++; 1141 return NETDEV_TX_BUSY; 1142 } 1143 1144 if (!is_last_wr) 1145 skb_get(skb); 1146 1147 pos = &q->q.desc[q->q.pidx]; 1148 end = (u64 *)pos + flits; 1149 /* FW_ULPTX_WR */ 1150 wr = pos; 1151 /* WR will need len16 */ 1152 len16 = DIV_ROUND_UP(flits, 2); 1153 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); 1154 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16)); 1155 wr->cookie = 0; 1156 pos += sizeof(*wr); 1157 /* ULP_TXPKT */ 1158 ulptx = pos; 1159 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | 1160 ULP_TXPKT_CHANNELID_V(tx_info->port_id) | 1161 ULP_TXPKT_FID_V(q->q.cntxt_id) | 1162 ULP_TXPKT_RO_F); 1163 ulptx->len = htonl(len16 - 1); 1164 /* ULPTX_IDATA sub-command */ 1165 idata = (struct ulptx_idata *)(ulptx + 1); 1166 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F); 1167 /* idata length will include cpl_tx_sec_pdu + key context size + 1168 * cpl_tx_data header. 1169 */ 1170 idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len + 1171 sizeof(*tx_data)); 1172 /* SEC CPL */ 1173 cpl = (struct cpl_tx_sec_pdu *)(idata + 1); 1174 cpl->op_ivinsrtofst = 1175 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) | 1176 CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) | 1177 CPL_TX_SEC_PDU_PLACEHOLDER_V(1) | 1178 CPL_TX_SEC_PDU_IVINSRTOFST_V(TLS_HEADER_SIZE + 1)); 1179 cpl->pldlen = htonl(data_len); 1180 1181 /* encryption should start after tls header size + iv size */ 1182 cipher_start = TLS_HEADER_SIZE + tx_info->iv_size + 1; 1183 1184 cpl->aadstart_cipherstop_hi = 1185 htonl(CPL_TX_SEC_PDU_AADSTART_V(1) | 1186 CPL_TX_SEC_PDU_AADSTOP_V(TLS_HEADER_SIZE) | 1187 CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start)); 1188 1189 /* authentication will also start after tls header + iv size */ 1190 cpl->cipherstop_lo_authinsert = 1191 htonl(CPL_TX_SEC_PDU_AUTHSTART_V(cipher_start) | 1192 CPL_TX_SEC_PDU_AUTHSTOP_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE) | 1193 CPL_TX_SEC_PDU_AUTHINSERT_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE)); 1194 1195 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ 1196 cpl->seqno_numivs = htonl(tx_info->scmd0_seqno_numivs); 1197 cpl->ivgen_hdrlen = htonl(tx_info->scmd0_ivgen_hdrlen); 1198 cpl->scmd1 = cpu_to_be64(tx_info->record_no); 1199 1200 pos = cpl + 1; 1201 /* check if space left to fill the keys */ 1202 left = (void *)q->q.stat - pos; 1203 if (!left) { 1204 left = (void *)end - (void *)q->q.stat; 1205 pos = q->q.desc; 1206 end = pos + left; 1207 } 1208 1209 pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos, 1210 tx_info->key_ctx_len); 1211 left = (void *)q->q.stat - pos; 1212 1213 if (!left) { 1214 left = (void *)end - (void *)q->q.stat; 1215 pos = q->q.desc; 1216 end = pos + left; 1217 } 1218 /* CPL_TX_DATA */ 1219 tx_data = (void *)pos; 1220 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid)); 1221 tx_data->len = htonl(TX_DATA_MSS_V(mss) | TX_LENGTH_V(data_len)); 1222 1223 tx_data->rsvd = htonl(tcp_seq); 1224 1225 tx_data->flags = htonl(TX_BYPASS_F); 1226 if (tcp_push) 1227 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F); 1228 1229 /* check left again, it might go beyond queue limit */ 1230 pos = tx_data + 1; 1231 left = (void *)q->q.stat - pos; 1232 1233 /* check the position again */ 1234 if (!left) { 1235 left = (void *)end - (void *)q->q.stat; 1236 pos = q->q.desc; 1237 end = pos + left; 1238 } 1239 1240 /* send the complete packet except the header */ 1241 cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr, 1242 skb_offset, data_len); 1243 sgl_sdesc->skb = skb; 1244 1245 chcr_txq_advance(&q->q, ndesc); 1246 cxgb4_ring_tx_db(adap, &q->q, ndesc); 1247 atomic64_inc(&adap->ch_ktls_stats.ktls_tx_send_records); 1248 1249 return 0; 1250} 1251 1252/* 1253 * chcr_ktls_xmit_wr_short: This is to send out partial records. If its 1254 * a middle part of a record, fetch the prior data to make it 16 byte aligned 1255 * and then only send it out. 1256 * 1257 * @skb - skb contains partial record.. 1258 * @tx_info - driver specific tls info. 1259 * @q - TX queue. 1260 * @tcp_seq 1261 * @tcp_push - tcp push bit. 1262 * @mss - segment size. 1263 * @tls_rec_offset - offset from start of the tls record. 1264 * @perior_data - data before the current segment, required to make this record 1265 * 16 byte aligned. 1266 * @prior_data_len - prior_data length (less than 16) 1267 * return: NETDEV_TX_BUSY/NET_TX_OK. 1268 */ 1269static int chcr_ktls_xmit_wr_short(struct sk_buff *skb, 1270 struct chcr_ktls_info *tx_info, 1271 struct sge_eth_txq *q, 1272 u32 tcp_seq, bool tcp_push, u32 mss, 1273 u32 tls_rec_offset, u8 *prior_data, 1274 u32 prior_data_len, u32 data_len, 1275 u32 skb_offset) 1276{ 1277 u32 len16, wr_mid = 0, cipher_start, nfrags; 1278 struct adapter *adap = tx_info->adap; 1279 unsigned int flits = 0, ndesc; 1280 int credits, left, last_desc; 1281 struct tx_sw_desc *sgl_sdesc; 1282 struct cpl_tx_data *tx_data; 1283 struct cpl_tx_sec_pdu *cpl; 1284 struct ulptx_idata *idata; 1285 struct ulp_txpkt *ulptx; 1286 struct fw_ulptx_wr *wr; 1287 __be64 iv_record; 1288 void *pos; 1289 u64 *end; 1290 1291 nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len); 1292 /* get the number of flits required, it's a partial record so 2 flits 1293 * (AES_BLOCK_SIZE) will be added. 1294 */ 1295 flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len) + 2; 1296 /* get the correct 8 byte IV of this record */ 1297 iv_record = cpu_to_be64(tx_info->iv + tx_info->record_no); 1298 /* If it's a middle record and not 16 byte aligned to run AES CTR, need 1299 * to make it 16 byte aligned. So atleadt 2 extra flits of immediate 1300 * data will be added. 1301 */ 1302 if (prior_data_len) 1303 flits += 2; 1304 /* number of descriptors */ 1305 ndesc = chcr_flits_to_desc(flits); 1306 /* check if enough credits available */ 1307 credits = chcr_txq_avail(&q->q) - ndesc; 1308 if (unlikely(credits < 0)) { 1309 chcr_eth_txq_stop(q); 1310 return NETDEV_TX_BUSY; 1311 } 1312 1313 if (unlikely(credits < ETHTXQ_STOP_THRES)) { 1314 chcr_eth_txq_stop(q); 1315 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; 1316 } 1317 1318 last_desc = q->q.pidx + ndesc - 1; 1319 if (last_desc >= q->q.size) 1320 last_desc -= q->q.size; 1321 sgl_sdesc = &q->q.sdesc[last_desc]; 1322 1323 if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) { 1324 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); 1325 q->mapping_err++; 1326 return NETDEV_TX_BUSY; 1327 } 1328 1329 pos = &q->q.desc[q->q.pidx]; 1330 end = (u64 *)pos + flits; 1331 /* FW_ULPTX_WR */ 1332 wr = pos; 1333 /* WR will need len16 */ 1334 len16 = DIV_ROUND_UP(flits, 2); 1335 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); 1336 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16)); 1337 wr->cookie = 0; 1338 pos += sizeof(*wr); 1339 /* ULP_TXPKT */ 1340 ulptx = pos; 1341 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | 1342 ULP_TXPKT_CHANNELID_V(tx_info->port_id) | 1343 ULP_TXPKT_FID_V(q->q.cntxt_id) | 1344 ULP_TXPKT_RO_F); 1345 ulptx->len = htonl(len16 - 1); 1346 /* ULPTX_IDATA sub-command */ 1347 idata = (struct ulptx_idata *)(ulptx + 1); 1348 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F); 1349 /* idata length will include cpl_tx_sec_pdu + key context size + 1350 * cpl_tx_data header. 1351 */ 1352 idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len + 1353 sizeof(*tx_data) + AES_BLOCK_LEN + prior_data_len); 1354 /* SEC CPL */ 1355 cpl = (struct cpl_tx_sec_pdu *)(idata + 1); 1356 /* cipher start will have tls header + iv size extra if its a header 1357 * part of tls record. else only 16 byte IV will be added. 1358 */ 1359 cipher_start = 1360 AES_BLOCK_LEN + 1 + 1361 (!tls_rec_offset ? TLS_HEADER_SIZE + tx_info->iv_size : 0); 1362 1363 cpl->op_ivinsrtofst = 1364 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) | 1365 CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) | 1366 CPL_TX_SEC_PDU_IVINSRTOFST_V(1)); 1367 cpl->pldlen = htonl(data_len + AES_BLOCK_LEN + prior_data_len); 1368 cpl->aadstart_cipherstop_hi = 1369 htonl(CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start)); 1370 cpl->cipherstop_lo_authinsert = 0; 1371 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ 1372 cpl->seqno_numivs = htonl(tx_info->scmd0_short_seqno_numivs); 1373 cpl->ivgen_hdrlen = htonl(tx_info->scmd0_short_ivgen_hdrlen); 1374 cpl->scmd1 = 0; 1375 1376 pos = cpl + 1; 1377 /* check if space left to fill the keys */ 1378 left = (void *)q->q.stat - pos; 1379 if (!left) { 1380 left = (void *)end - (void *)q->q.stat; 1381 pos = q->q.desc; 1382 end = pos + left; 1383 } 1384 1385 pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos, 1386 tx_info->key_ctx_len); 1387 left = (void *)q->q.stat - pos; 1388 1389 if (!left) { 1390 left = (void *)end - (void *)q->q.stat; 1391 pos = q->q.desc; 1392 end = pos + left; 1393 } 1394 /* CPL_TX_DATA */ 1395 tx_data = (void *)pos; 1396 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid)); 1397 tx_data->len = htonl(TX_DATA_MSS_V(mss) | 1398 TX_LENGTH_V(data_len + prior_data_len)); 1399 tx_data->rsvd = htonl(tcp_seq); 1400 tx_data->flags = htonl(TX_BYPASS_F); 1401 if (tcp_push) 1402 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F); 1403 1404 /* check left again, it might go beyond queue limit */ 1405 pos = tx_data + 1; 1406 left = (void *)q->q.stat - pos; 1407 1408 /* check the position again */ 1409 if (!left) { 1410 left = (void *)end - (void *)q->q.stat; 1411 pos = q->q.desc; 1412 end = pos + left; 1413 } 1414 /* copy the 16 byte IV for AES-CTR, which includes 4 bytes of salt, 8 1415 * bytes of actual IV and 4 bytes of 16 byte-sequence. 1416 */ 1417 memcpy(pos, tx_info->key_ctx.salt, tx_info->salt_size); 1418 memcpy(pos + tx_info->salt_size, &iv_record, tx_info->iv_size); 1419 *(__be32 *)(pos + tx_info->salt_size + tx_info->iv_size) = 1420 htonl(2 + (tls_rec_offset ? ((tls_rec_offset - 1421 (TLS_HEADER_SIZE + tx_info->iv_size)) / AES_BLOCK_LEN) : 0)); 1422 1423 pos += 16; 1424 /* Prior_data_len will always be less than 16 bytes, fill the 1425 * prio_data_len after AES_CTRL_BLOCK and clear the remaining length 1426 * to 0. 1427 */ 1428 if (prior_data_len) 1429 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16); 1430 /* send the complete packet except the header */ 1431 cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr, 1432 skb_offset, data_len); 1433 sgl_sdesc->skb = skb; 1434 1435 chcr_txq_advance(&q->q, ndesc); 1436 cxgb4_ring_tx_db(adap, &q->q, ndesc); 1437 1438 return 0; 1439} 1440 1441/* 1442 * chcr_ktls_tx_plaintxt: This handler will take care of the records which has 1443 * only plain text (only tls header and iv) 1444 * @tx_info - driver specific tls info. 1445 * @skb - skb contains partial record.. 1446 * @tcp_seq 1447 * @mss - segment size. 1448 * @tcp_push - tcp push bit. 1449 * @q - TX queue. 1450 * @port_id : port number 1451 * @perior_data - data before the current segment, required to make this record 1452 * 16 byte aligned. 1453 * @prior_data_len - prior_data length (less than 16) 1454 * return: NETDEV_TX_BUSY/NET_TX_OK. 1455 */ 1456static int chcr_ktls_tx_plaintxt(struct chcr_ktls_info *tx_info, 1457 struct sk_buff *skb, u32 tcp_seq, u32 mss, 1458 bool tcp_push, struct sge_eth_txq *q, 1459 u32 port_id, u8 *prior_data, 1460 u32 data_len, u32 skb_offset, 1461 u32 prior_data_len) 1462{ 1463 int credits, left, len16, last_desc; 1464 unsigned int flits = 0, ndesc; 1465 struct tx_sw_desc *sgl_sdesc; 1466 struct cpl_tx_data *tx_data; 1467 struct ulptx_idata *idata; 1468 struct ulp_txpkt *ulptx; 1469 struct fw_ulptx_wr *wr; 1470 u32 wr_mid = 0, nfrags; 1471 void *pos; 1472 u64 *end; 1473 1474 flits = DIV_ROUND_UP(CHCR_PLAIN_TX_DATA_LEN, 8); 1475 nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len); 1476 flits += chcr_sgl_len(nfrags); 1477 if (prior_data_len) 1478 flits += 2; 1479 1480 /* WR will need len16 */ 1481 len16 = DIV_ROUND_UP(flits, 2); 1482 /* check how many descriptors needed */ 1483 ndesc = DIV_ROUND_UP(flits, 8); 1484 1485 credits = chcr_txq_avail(&q->q) - ndesc; 1486 if (unlikely(credits < 0)) { 1487 chcr_eth_txq_stop(q); 1488 return NETDEV_TX_BUSY; 1489 } 1490 1491 if (unlikely(credits < ETHTXQ_STOP_THRES)) { 1492 chcr_eth_txq_stop(q); 1493 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; 1494 } 1495 1496 last_desc = q->q.pidx + ndesc - 1; 1497 if (last_desc >= q->q.size) 1498 last_desc -= q->q.size; 1499 sgl_sdesc = &q->q.sdesc[last_desc]; 1500 1501 if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb, 1502 sgl_sdesc->addr) < 0)) { 1503 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); 1504 q->mapping_err++; 1505 return NETDEV_TX_BUSY; 1506 } 1507 1508 pos = &q->q.desc[q->q.pidx]; 1509 end = (u64 *)pos + flits; 1510 /* FW_ULPTX_WR */ 1511 wr = pos; 1512 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); 1513 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16)); 1514 wr->cookie = 0; 1515 /* ULP_TXPKT */ 1516 ulptx = (struct ulp_txpkt *)(wr + 1); 1517 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | 1518 ULP_TXPKT_DATAMODIFY_V(0) | 1519 ULP_TXPKT_CHANNELID_V(tx_info->port_id) | 1520 ULP_TXPKT_DEST_V(0) | 1521 ULP_TXPKT_FID_V(q->q.cntxt_id) | ULP_TXPKT_RO_V(1)); 1522 ulptx->len = htonl(len16 - 1); 1523 /* ULPTX_IDATA sub-command */ 1524 idata = (struct ulptx_idata *)(ulptx + 1); 1525 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F); 1526 idata->len = htonl(sizeof(*tx_data) + prior_data_len); 1527 /* CPL_TX_DATA */ 1528 tx_data = (struct cpl_tx_data *)(idata + 1); 1529 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid)); 1530 tx_data->len = htonl(TX_DATA_MSS_V(mss) | 1531 TX_LENGTH_V(data_len + prior_data_len)); 1532 /* set tcp seq number */ 1533 tx_data->rsvd = htonl(tcp_seq); 1534 tx_data->flags = htonl(TX_BYPASS_F); 1535 if (tcp_push) 1536 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F); 1537 1538 pos = tx_data + 1; 1539 /* apart from prior_data_len, we should set remaining part of 16 bytes 1540 * to be zero. 1541 */ 1542 if (prior_data_len) 1543 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16); 1544 1545 /* check left again, it might go beyond queue limit */ 1546 left = (void *)q->q.stat - pos; 1547 1548 /* check the position again */ 1549 if (!left) { 1550 left = (void *)end - (void *)q->q.stat; 1551 pos = q->q.desc; 1552 end = pos + left; 1553 } 1554 /* send the complete packet including the header */ 1555 cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr, 1556 skb_offset, data_len); 1557 sgl_sdesc->skb = skb; 1558 1559 chcr_txq_advance(&q->q, ndesc); 1560 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); 1561 return 0; 1562} 1563 1564static int chcr_ktls_tunnel_pkt(struct chcr_ktls_info *tx_info, 1565 struct sk_buff *skb, 1566 struct sge_eth_txq *q) 1567{ 1568 u32 ctrl, iplen, maclen, wr_mid = 0, len16; 1569 struct tx_sw_desc *sgl_sdesc; 1570 struct fw_eth_tx_pkt_wr *wr; 1571 struct cpl_tx_pkt_core *cpl; 1572 unsigned int flits, ndesc; 1573 int credits, last_desc; 1574 u64 cntrl1, *end; 1575 void *pos; 1576 1577 ctrl = sizeof(*cpl); 1578 flits = DIV_ROUND_UP(sizeof(*wr) + ctrl, 8); 1579 1580 flits += chcr_sgl_len(skb_shinfo(skb)->nr_frags + 1); 1581 len16 = DIV_ROUND_UP(flits, 2); 1582 /* check how many descriptors needed */ 1583 ndesc = DIV_ROUND_UP(flits, 8); 1584 1585 credits = chcr_txq_avail(&q->q) - ndesc; 1586 if (unlikely(credits < 0)) { 1587 chcr_eth_txq_stop(q); 1588 return -ENOMEM; 1589 } 1590 1591 if (unlikely(credits < ETHTXQ_STOP_THRES)) { 1592 chcr_eth_txq_stop(q); 1593 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; 1594 } 1595 1596 last_desc = q->q.pidx + ndesc - 1; 1597 if (last_desc >= q->q.size) 1598 last_desc -= q->q.size; 1599 sgl_sdesc = &q->q.sdesc[last_desc]; 1600 1601 if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb, 1602 sgl_sdesc->addr) < 0)) { 1603 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); 1604 q->mapping_err++; 1605 return -ENOMEM; 1606 } 1607 1608 iplen = skb_network_header_len(skb); 1609 maclen = skb_mac_header_len(skb); 1610 1611 pos = &q->q.desc[q->q.pidx]; 1612 end = (u64 *)pos + flits; 1613 wr = pos; 1614 1615 /* Firmware work request header */ 1616 wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) | 1617 FW_WR_IMMDLEN_V(ctrl)); 1618 1619 wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16)); 1620 wr->r3 = 0; 1621 1622 cpl = (void *)(wr + 1); 1623 1624 /* CPL header */ 1625 cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) | 1626 TXPKT_INTF_V(tx_info->tx_chan) | 1627 TXPKT_PF_V(tx_info->adap->pf)); 1628 cpl->pack = 0; 1629 cntrl1 = TXPKT_CSUM_TYPE_V(tx_info->ip_family == AF_INET ? 1630 TX_CSUM_TCPIP : TX_CSUM_TCPIP6); 1631 cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) | 1632 TXPKT_IPHDR_LEN_V(iplen); 1633 /* checksum offload */ 1634 cpl->ctrl1 = cpu_to_be64(cntrl1); 1635 cpl->len = htons(skb->len); 1636 1637 pos = cpl + 1; 1638 1639 cxgb4_write_sgl(skb, &q->q, pos, end, 0, sgl_sdesc->addr); 1640 sgl_sdesc->skb = skb; 1641 chcr_txq_advance(&q->q, ndesc); 1642 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); 1643 return 0; 1644} 1645 1646/* 1647 * chcr_ktls_copy_record_in_skb 1648 * @nskb - new skb where the frags to be added. 1649 * @skb - old skb, to copy socket and destructor details. 1650 * @record - specific record which has complete 16k record in frags. 1651 */ 1652static void chcr_ktls_copy_record_in_skb(struct sk_buff *nskb, 1653 struct sk_buff *skb, 1654 struct tls_record_info *record) 1655{ 1656 int i = 0; 1657 1658 for (i = 0; i < record->num_frags; i++) { 1659 skb_shinfo(nskb)->frags[i] = record->frags[i]; 1660 /* increase the frag ref count */ 1661 __skb_frag_ref(&skb_shinfo(nskb)->frags[i]); 1662 } 1663 1664 skb_shinfo(nskb)->nr_frags = record->num_frags; 1665 nskb->data_len = record->len; 1666 nskb->len += record->len; 1667 nskb->truesize += record->len; 1668 nskb->sk = skb->sk; 1669 nskb->destructor = skb->destructor; 1670 refcount_add(nskb->truesize, &nskb->sk->sk_wmem_alloc); 1671} 1672 1673/* 1674 * chcr_end_part_handler: This handler will handle the record which 1675 * is complete or if record's end part is received. T6 adapter has a issue that 1676 * it can't send out TAG with partial record so if its an end part then we have 1677 * to send TAG as well and for which we need to fetch the complete record and 1678 * send it to crypto module. 1679 * @tx_info - driver specific tls info. 1680 * @skb - skb contains partial record. 1681 * @record - complete record of 16K size. 1682 * @tcp_seq 1683 * @mss - segment size in which TP needs to chop a packet. 1684 * @tcp_push_no_fin - tcp push if fin is not set. 1685 * @q - TX queue. 1686 * @tls_end_offset - offset from end of the record. 1687 * @last wr : check if this is the last part of the skb going out. 1688 * return: NETDEV_TX_OK/NETDEV_TX_BUSY. 1689 */ 1690static int chcr_end_part_handler(struct chcr_ktls_info *tx_info, 1691 struct sk_buff *skb, 1692 struct tls_record_info *record, 1693 u32 tcp_seq, int mss, bool tcp_push_no_fin, 1694 struct sge_eth_txq *q, u32 skb_offset, 1695 u32 tls_end_offset, bool last_wr) 1696{ 1697 bool free_skb_if_tx_fails = false; 1698 struct sk_buff *nskb = NULL; 1699 1700 /* check if it is a complete record */ 1701 if (tls_end_offset == record->len) { 1702 nskb = skb; 1703 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_complete_pkts); 1704 } else { 1705 nskb = alloc_skb(0, GFP_ATOMIC); 1706 if (!nskb) { 1707 dev_kfree_skb_any(skb); 1708 return NETDEV_TX_BUSY; 1709 } 1710 1711 /* copy complete record in skb */ 1712 chcr_ktls_copy_record_in_skb(nskb, skb, record); 1713 /* packet is being sent from the beginning, update the tcp_seq 1714 * accordingly. 1715 */ 1716 tcp_seq = tls_record_start_seq(record); 1717 /* reset skb offset */ 1718 skb_offset = 0; 1719 1720 if (last_wr) 1721 dev_kfree_skb_any(skb); 1722 else 1723 free_skb_if_tx_fails = true; 1724 1725 last_wr = true; 1726 1727 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_end_pkts); 1728 } 1729 1730 if (chcr_ktls_xmit_wr_complete(nskb, tx_info, q, tcp_seq, 1731 last_wr, record->len, skb_offset, 1732 record->num_frags, 1733 (last_wr && tcp_push_no_fin), 1734 mss)) { 1735 if (free_skb_if_tx_fails) 1736 dev_kfree_skb_any(skb); 1737 goto out; 1738 } 1739 tx_info->prev_seq = record->end_seq; 1740 return 0; 1741out: 1742 dev_kfree_skb_any(nskb); 1743 return NETDEV_TX_BUSY; 1744} 1745 1746/* 1747 * chcr_short_record_handler: This handler will take care of the records which 1748 * doesn't have end part (1st part or the middle part(/s) of a record). In such 1749 * cases, AES CTR will be used in place of AES GCM to send out partial packet. 1750 * This partial record might be the first part of the record, or the middle 1751 * part. In case of middle record we should fetch the prior data to make it 16 1752 * byte aligned. If it has a partial tls header or iv then get to the start of 1753 * tls header. And if it has partial TAG, then remove the complete TAG and send 1754 * only the payload. 1755 * There is one more possibility that it gets a partial header, send that 1756 * portion as a plaintext. 1757 * @tx_info - driver specific tls info. 1758 * @skb - skb contains partial record.. 1759 * @record - complete record of 16K size. 1760 * @tcp_seq 1761 * @mss - segment size in which TP needs to chop a packet. 1762 * @tcp_push_no_fin - tcp push if fin is not set. 1763 * @q - TX queue. 1764 * @tls_end_offset - offset from end of the record. 1765 * return: NETDEV_TX_OK/NETDEV_TX_BUSY. 1766 */ 1767static int chcr_short_record_handler(struct chcr_ktls_info *tx_info, 1768 struct sk_buff *skb, 1769 struct tls_record_info *record, 1770 u32 tcp_seq, int mss, bool tcp_push_no_fin, 1771 u32 data_len, u32 skb_offset, 1772 struct sge_eth_txq *q, u32 tls_end_offset) 1773{ 1774 u32 tls_rec_offset = tcp_seq - tls_record_start_seq(record); 1775 u8 prior_data[16] = {0}; 1776 u32 prior_data_len = 0; 1777 1778 /* check if the skb is ending in middle of tag/HASH, its a big 1779 * trouble, send the packet before the HASH. 1780 */ 1781 int remaining_record = tls_end_offset - data_len; 1782 1783 if (remaining_record > 0 && 1784 remaining_record < TLS_CIPHER_AES_GCM_128_TAG_SIZE) { 1785 int trimmed_len = 0; 1786 1787 if (tls_end_offset > TLS_CIPHER_AES_GCM_128_TAG_SIZE) 1788 trimmed_len = data_len - 1789 (TLS_CIPHER_AES_GCM_128_TAG_SIZE - 1790 remaining_record); 1791 if (!trimmed_len) 1792 return FALLBACK; 1793 1794 WARN_ON(trimmed_len > data_len); 1795 1796 data_len = trimmed_len; 1797 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_trimmed_pkts); 1798 } 1799 1800 /* check if it is only the header part. */ 1801 if (tls_rec_offset + data_len <= (TLS_HEADER_SIZE + tx_info->iv_size)) { 1802 if (chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss, 1803 tcp_push_no_fin, q, 1804 tx_info->port_id, prior_data, 1805 data_len, skb_offset, prior_data_len)) 1806 goto out; 1807 1808 tx_info->prev_seq = tcp_seq + data_len; 1809 return 0; 1810 } 1811 1812 /* check if the middle record's start point is 16 byte aligned. CTR 1813 * needs 16 byte aligned start point to start encryption. 1814 */ 1815 if (tls_rec_offset) { 1816 /* there is an offset from start, means its a middle record */ 1817 int remaining = 0; 1818 1819 if (tls_rec_offset < (TLS_HEADER_SIZE + tx_info->iv_size)) { 1820 prior_data_len = tls_rec_offset; 1821 tls_rec_offset = 0; 1822 remaining = 0; 1823 } else { 1824 prior_data_len = 1825 (tls_rec_offset - 1826 (TLS_HEADER_SIZE + tx_info->iv_size)) 1827 % AES_BLOCK_LEN; 1828 remaining = tls_rec_offset - prior_data_len; 1829 } 1830 1831 /* if prior_data_len is not zero, means we need to fetch prior 1832 * data to make this record 16 byte aligned, or we need to reach 1833 * to start offset. 1834 */ 1835 if (prior_data_len) { 1836 int i = 0; 1837 skb_frag_t *f; 1838 int frag_size = 0, frag_delta = 0; 1839 1840 while (remaining > 0) { 1841 frag_size = skb_frag_size(&record->frags[i]); 1842 if (remaining < frag_size) 1843 break; 1844 1845 remaining -= frag_size; 1846 i++; 1847 } 1848 f = &record->frags[i]; 1849 frag_delta = skb_frag_size(f) - remaining; 1850 1851 if (frag_delta >= prior_data_len) { 1852 memcpy_from_page(prior_data, skb_frag_page(f), 1853 skb_frag_off(f) + remaining, 1854 prior_data_len); 1855 } else { 1856 memcpy_from_page(prior_data, skb_frag_page(f), 1857 skb_frag_off(f) + remaining, 1858 frag_delta); 1859 1860 /* get the next page */ 1861 f = &record->frags[i + 1]; 1862 1863 memcpy_from_page(prior_data + frag_delta, 1864 skb_frag_page(f), 1865 skb_frag_off(f), 1866 prior_data_len - frag_delta); 1867 } 1868 /* reset tcp_seq as per the prior_data_required len */ 1869 tcp_seq -= prior_data_len; 1870 } 1871 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_middle_pkts); 1872 } else { 1873 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_start_pkts); 1874 } 1875 1876 if (chcr_ktls_xmit_wr_short(skb, tx_info, q, tcp_seq, tcp_push_no_fin, 1877 mss, tls_rec_offset, prior_data, 1878 prior_data_len, data_len, skb_offset)) { 1879 goto out; 1880 } 1881 1882 tx_info->prev_seq = tcp_seq + data_len + prior_data_len; 1883 return 0; 1884out: 1885 dev_kfree_skb_any(skb); 1886 return NETDEV_TX_BUSY; 1887} 1888 1889static int chcr_ktls_sw_fallback(struct sk_buff *skb, 1890 struct chcr_ktls_info *tx_info, 1891 struct sge_eth_txq *q) 1892{ 1893 u32 data_len, skb_offset; 1894 struct sk_buff *nskb; 1895 struct tcphdr *th; 1896 1897 nskb = tls_encrypt_skb(skb); 1898 1899 if (!nskb) 1900 return 0; 1901 1902 th = tcp_hdr(nskb); 1903 skb_offset = skb_tcp_all_headers(nskb); 1904 data_len = nskb->len - skb_offset; 1905 skb_tx_timestamp(nskb); 1906 1907 if (chcr_ktls_tunnel_pkt(tx_info, nskb, q)) 1908 goto out; 1909 1910 tx_info->prev_seq = ntohl(th->seq) + data_len; 1911 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_fallback); 1912 return 0; 1913out: 1914 dev_kfree_skb_any(nskb); 1915 return 0; 1916} 1917/* nic tls TX handler */ 1918static int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev) 1919{ 1920 u32 tls_end_offset, tcp_seq, skb_data_len, skb_offset; 1921 struct ch_ktls_port_stats_debug *port_stats; 1922 struct tls_offload_context_tx *tx_ctx; 1923 struct ch_ktls_stats_debug *stats; 1924 struct tcphdr *th = tcp_hdr(skb); 1925 int data_len, qidx, ret = 0, mss; 1926 struct tls_record_info *record; 1927 struct chcr_ktls_info *tx_info; 1928 struct net_device *tls_netdev; 1929 struct tls_context *tls_ctx; 1930 struct sge_eth_txq *q; 1931 struct adapter *adap; 1932 unsigned long flags; 1933 1934 tcp_seq = ntohl(th->seq); 1935 skb_offset = skb_tcp_all_headers(skb); 1936 skb_data_len = skb->len - skb_offset; 1937 data_len = skb_data_len; 1938 1939 mss = skb_is_gso(skb) ? skb_shinfo(skb)->gso_size : data_len; 1940 1941 tls_ctx = tls_get_ctx(skb->sk); 1942 tx_ctx = tls_offload_ctx_tx(tls_ctx); 1943 tls_netdev = rcu_dereference_bh(tls_ctx->netdev); 1944 /* Don't quit on NULL: if tls_device_down is running in parallel, 1945 * netdev might become NULL, even if tls_is_skb_tx_device_offloaded was 1946 * true. Rather continue processing this packet. 1947 */ 1948 if (unlikely(tls_netdev && tls_netdev != dev)) 1949 goto out; 1950 1951 tx_info = chcr_get_ktls_tx_info(tls_ctx); 1952 1953 if (unlikely(!tx_info)) 1954 goto out; 1955 1956 adap = tx_info->adap; 1957 stats = &adap->ch_ktls_stats; 1958 port_stats = &stats->ktls_port[tx_info->port_id]; 1959 1960 qidx = skb->queue_mapping; 1961 q = &adap->sge.ethtxq[qidx + tx_info->first_qset]; 1962 cxgb4_reclaim_completed_tx(adap, &q->q, true); 1963 /* if tcp options are set but finish is not send the options first */ 1964 if (!th->fin && chcr_ktls_check_tcp_options(th)) { 1965 ret = chcr_ktls_write_tcp_options(tx_info, skb, q, 1966 tx_info->tx_chan); 1967 if (ret) 1968 return NETDEV_TX_BUSY; 1969 } 1970 1971 /* TCP segments can be in received either complete or partial. 1972 * chcr_end_part_handler will handle cases if complete record or end 1973 * part of the record is received. In case of partial end part of record, 1974 * we will send the complete record again. 1975 */ 1976 1977 spin_lock_irqsave(&tx_ctx->lock, flags); 1978 1979 do { 1980 1981 cxgb4_reclaim_completed_tx(adap, &q->q, true); 1982 /* fetch the tls record */ 1983 record = tls_get_record(tx_ctx, tcp_seq, 1984 &tx_info->record_no); 1985 /* By the time packet reached to us, ACK is received, and record 1986 * won't be found in that case, handle it gracefully. 1987 */ 1988 if (unlikely(!record)) { 1989 spin_unlock_irqrestore(&tx_ctx->lock, flags); 1990 atomic64_inc(&port_stats->ktls_tx_drop_no_sync_data); 1991 goto out; 1992 } 1993 1994 tls_end_offset = record->end_seq - tcp_seq; 1995 1996 pr_debug("seq 0x%x, end_seq 0x%x prev_seq 0x%x, datalen 0x%x\n", 1997 tcp_seq, record->end_seq, tx_info->prev_seq, data_len); 1998 /* update tcb for the skb */ 1999 if (skb_data_len == data_len) { 2000 u32 tx_max = tcp_seq; 2001 2002 if (!tls_record_is_start_marker(record) && 2003 tls_end_offset < TLS_CIPHER_AES_GCM_128_TAG_SIZE) 2004 tx_max = record->end_seq - 2005 TLS_CIPHER_AES_GCM_128_TAG_SIZE; 2006 2007 ret = chcr_ktls_xmit_tcb_cpls(tx_info, q, tx_max, 2008 ntohl(th->ack_seq), 2009 ntohs(th->window), 2010 tls_end_offset != 2011 record->len); 2012 if (ret) { 2013 spin_unlock_irqrestore(&tx_ctx->lock, 2014 flags); 2015 goto out; 2016 } 2017 2018 if (th->fin) 2019 skb_get(skb); 2020 } 2021 2022 if (unlikely(tls_record_is_start_marker(record))) { 2023 atomic64_inc(&port_stats->ktls_tx_skip_no_sync_data); 2024 /* If tls_end_offset < data_len, means there is some 2025 * data after start marker, which needs encryption, send 2026 * plaintext first and take skb refcount. else send out 2027 * complete pkt as plaintext. 2028 */ 2029 if (tls_end_offset < data_len) 2030 skb_get(skb); 2031 else 2032 tls_end_offset = data_len; 2033 2034 ret = chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss, 2035 (!th->fin && th->psh), q, 2036 tx_info->port_id, NULL, 2037 tls_end_offset, skb_offset, 2038 0); 2039 2040 if (ret) { 2041 /* free the refcount taken earlier */ 2042 if (tls_end_offset < data_len) 2043 dev_kfree_skb_any(skb); 2044 spin_unlock_irqrestore(&tx_ctx->lock, flags); 2045 goto out; 2046 } 2047 2048 data_len -= tls_end_offset; 2049 tcp_seq = record->end_seq; 2050 skb_offset += tls_end_offset; 2051 continue; 2052 } 2053 2054 /* if a tls record is finishing in this SKB */ 2055 if (tls_end_offset <= data_len) { 2056 ret = chcr_end_part_handler(tx_info, skb, record, 2057 tcp_seq, mss, 2058 (!th->fin && th->psh), q, 2059 skb_offset, 2060 tls_end_offset, 2061 skb_offset + 2062 tls_end_offset == skb->len); 2063 2064 data_len -= tls_end_offset; 2065 /* tcp_seq increment is required to handle next record. 2066 */ 2067 tcp_seq += tls_end_offset; 2068 skb_offset += tls_end_offset; 2069 } else { 2070 ret = chcr_short_record_handler(tx_info, skb, 2071 record, tcp_seq, mss, 2072 (!th->fin && th->psh), 2073 data_len, skb_offset, 2074 q, tls_end_offset); 2075 data_len = 0; 2076 } 2077 2078 /* if any failure, come out from the loop. */ 2079 if (ret) { 2080 spin_unlock_irqrestore(&tx_ctx->lock, flags); 2081 if (th->fin) 2082 dev_kfree_skb_any(skb); 2083 2084 if (ret == FALLBACK) 2085 return chcr_ktls_sw_fallback(skb, tx_info, q); 2086 2087 return NETDEV_TX_OK; 2088 } 2089 2090 /* length should never be less than 0 */ 2091 WARN_ON(data_len < 0); 2092 2093 } while (data_len > 0); 2094 2095 spin_unlock_irqrestore(&tx_ctx->lock, flags); 2096 atomic64_inc(&port_stats->ktls_tx_encrypted_packets); 2097 atomic64_add(skb_data_len, &port_stats->ktls_tx_encrypted_bytes); 2098 2099 /* tcp finish is set, send a separate tcp msg including all the options 2100 * as well. 2101 */ 2102 if (th->fin) { 2103 chcr_ktls_write_tcp_options(tx_info, skb, q, tx_info->tx_chan); 2104 dev_kfree_skb_any(skb); 2105 } 2106 2107 return NETDEV_TX_OK; 2108out: 2109 dev_kfree_skb_any(skb); 2110 return NETDEV_TX_OK; 2111} 2112 2113static void *chcr_ktls_uld_add(const struct cxgb4_lld_info *lldi) 2114{ 2115 struct chcr_ktls_uld_ctx *u_ctx; 2116 2117 pr_info_once("%s - version %s\n", CHCR_KTLS_DRV_DESC, 2118 CHCR_KTLS_DRV_VERSION); 2119 u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL); 2120 if (!u_ctx) { 2121 u_ctx = ERR_PTR(-ENOMEM); 2122 goto out; 2123 } 2124 u_ctx->lldi = *lldi; 2125 u_ctx->detach = false; 2126 xa_init_flags(&u_ctx->tid_list, XA_FLAGS_LOCK_BH); 2127out: 2128 return u_ctx; 2129} 2130 2131static const struct tlsdev_ops chcr_ktls_ops = { 2132 .tls_dev_add = chcr_ktls_dev_add, 2133 .tls_dev_del = chcr_ktls_dev_del, 2134}; 2135 2136static chcr_handler_func work_handlers[NUM_CPL_CMDS] = { 2137 [CPL_ACT_OPEN_RPL] = chcr_ktls_cpl_act_open_rpl, 2138 [CPL_SET_TCB_RPL] = chcr_ktls_cpl_set_tcb_rpl, 2139}; 2140 2141static int chcr_ktls_uld_rx_handler(void *handle, const __be64 *rsp, 2142 const struct pkt_gl *pgl) 2143{ 2144 const struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)rsp; 2145 struct chcr_ktls_uld_ctx *u_ctx = handle; 2146 u8 opcode = rpl->ot.opcode; 2147 struct adapter *adap; 2148 2149 adap = pci_get_drvdata(u_ctx->lldi.pdev); 2150 2151 if (!work_handlers[opcode]) { 2152 pr_err("Unsupported opcode %d received\n", opcode); 2153 return 0; 2154 } 2155 2156 work_handlers[opcode](adap, (unsigned char *)&rsp[1]); 2157 return 0; 2158} 2159 2160static void clear_conn_resources(struct chcr_ktls_info *tx_info) 2161{ 2162 /* clear l2t entry */ 2163 if (tx_info->l2te) 2164 cxgb4_l2t_release(tx_info->l2te); 2165 2166#if IS_ENABLED(CONFIG_IPV6) 2167 /* clear clip entry */ 2168 if (tx_info->ip_family == AF_INET6) 2169 cxgb4_clip_release(tx_info->netdev, (const u32 *) 2170 &tx_info->sk->sk_v6_rcv_saddr, 2171 1); 2172#endif 2173 2174 /* clear tid */ 2175 if (tx_info->tid != -1) 2176 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan, 2177 tx_info->tid, tx_info->ip_family); 2178} 2179 2180static void ch_ktls_reset_all_conn(struct chcr_ktls_uld_ctx *u_ctx) 2181{ 2182 struct ch_ktls_port_stats_debug *port_stats; 2183 struct tls_offload_context_tx *tx_ctx; 2184 struct chcr_ktls_info *tx_info; 2185 unsigned long index; 2186 2187 xa_for_each(&u_ctx->tid_list, index, tx_ctx) { 2188 tx_info = __chcr_get_ktls_tx_info(tx_ctx); 2189 clear_conn_resources(tx_info); 2190 port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id]; 2191 atomic64_inc(&port_stats->ktls_tx_connection_close); 2192 kvfree(tx_info); 2193 memset(tx_ctx->driver_state, 0, TLS_DRIVER_STATE_SIZE_TX); 2194 /* release module refcount */ 2195 module_put(THIS_MODULE); 2196 } 2197} 2198 2199static int chcr_ktls_uld_state_change(void *handle, enum cxgb4_state new_state) 2200{ 2201 struct chcr_ktls_uld_ctx *u_ctx = handle; 2202 2203 switch (new_state) { 2204 case CXGB4_STATE_UP: 2205 pr_info("%s: Up\n", pci_name(u_ctx->lldi.pdev)); 2206 mutex_lock(&dev_mutex); 2207 list_add_tail(&u_ctx->entry, &uld_ctx_list); 2208 mutex_unlock(&dev_mutex); 2209 break; 2210 case CXGB4_STATE_START_RECOVERY: 2211 case CXGB4_STATE_DOWN: 2212 case CXGB4_STATE_DETACH: 2213 pr_info("%s: Down\n", pci_name(u_ctx->lldi.pdev)); 2214 mutex_lock(&dev_mutex); 2215 u_ctx->detach = true; 2216 list_del(&u_ctx->entry); 2217 ch_ktls_reset_all_conn(u_ctx); 2218 xa_destroy(&u_ctx->tid_list); 2219 mutex_unlock(&dev_mutex); 2220 break; 2221 default: 2222 break; 2223 } 2224 2225 return 0; 2226} 2227 2228static struct cxgb4_uld_info chcr_ktls_uld_info = { 2229 .name = CHCR_KTLS_DRV_MODULE_NAME, 2230 .nrxq = 1, 2231 .rxq_size = 1024, 2232 .add = chcr_ktls_uld_add, 2233 .tx_handler = chcr_ktls_xmit, 2234 .rx_handler = chcr_ktls_uld_rx_handler, 2235 .state_change = chcr_ktls_uld_state_change, 2236 .tlsdev_ops = &chcr_ktls_ops, 2237}; 2238 2239static int __init chcr_ktls_init(void) 2240{ 2241 cxgb4_register_uld(CXGB4_ULD_KTLS, &chcr_ktls_uld_info); 2242 return 0; 2243} 2244 2245static void __exit chcr_ktls_exit(void) 2246{ 2247 struct chcr_ktls_uld_ctx *u_ctx, *tmp; 2248 struct adapter *adap; 2249 2250 pr_info("driver unloaded\n"); 2251 2252 mutex_lock(&dev_mutex); 2253 list_for_each_entry_safe(u_ctx, tmp, &uld_ctx_list, entry) { 2254 adap = pci_get_drvdata(u_ctx->lldi.pdev); 2255 memset(&adap->ch_ktls_stats, 0, sizeof(adap->ch_ktls_stats)); 2256 list_del(&u_ctx->entry); 2257 xa_destroy(&u_ctx->tid_list); 2258 kfree(u_ctx); 2259 } 2260 mutex_unlock(&dev_mutex); 2261 cxgb4_unregister_uld(CXGB4_ULD_KTLS); 2262} 2263 2264module_init(chcr_ktls_init); 2265module_exit(chcr_ktls_exit); 2266 2267MODULE_DESCRIPTION("Chelsio NIC TLS ULD driver"); 2268MODULE_LICENSE("GPL"); 2269MODULE_AUTHOR("Chelsio Communications"); 2270MODULE_VERSION(CHCR_KTLS_DRV_VERSION); 2271