1/* 2 * Copyright (c) 2017 Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 34#include <crypto/internal/geniv.h> 35#include <crypto/aead.h> 36#include <linux/inetdevice.h> 37#include <linux/netdevice.h> 38#include <net/netevent.h> 39 40#include "en.h" 41#include "eswitch.h" 42#include "ipsec.h" 43#include "ipsec_rxtx.h" 44#include "en_rep.h" 45 46#define MLX5_IPSEC_RESCHED msecs_to_jiffies(1000) 47#define MLX5E_IPSEC_TUNNEL_SA XA_MARK_1 48 49static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(struct xfrm_state *x) 50{ 51 return (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle; 52} 53 54static struct mlx5e_ipsec_pol_entry *to_ipsec_pol_entry(struct xfrm_policy *x) 55{ 56 return (struct mlx5e_ipsec_pol_entry *)x->xdo.offload_handle; 57} 58 59static void mlx5e_ipsec_handle_sw_limits(struct work_struct *_work) 60{ 61 struct mlx5e_ipsec_dwork *dwork = 62 container_of(_work, struct mlx5e_ipsec_dwork, dwork.work); 63 struct mlx5e_ipsec_sa_entry *sa_entry = dwork->sa_entry; 64 struct xfrm_state *x = sa_entry->x; 65 66 if (sa_entry->attrs.drop) 67 return; 68 69 spin_lock_bh(&x->lock); 70 xfrm_state_check_expire(x); 71 if (x->km.state == XFRM_STATE_EXPIRED) { 72 sa_entry->attrs.drop = true; 73 spin_unlock_bh(&x->lock); 74 75 mlx5e_accel_ipsec_fs_modify(sa_entry); 76 return; 77 } 78 spin_unlock_bh(&x->lock); 79 80 queue_delayed_work(sa_entry->ipsec->wq, &dwork->dwork, 81 MLX5_IPSEC_RESCHED); 82} 83 84static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) 85{ 86 struct xfrm_state *x = sa_entry->x; 87 u32 seq_bottom = 0; 88 u32 esn, esn_msb; 89 u8 overlap; 90 91 switch (x->xso.type) { 92 case XFRM_DEV_OFFLOAD_PACKET: 93 switch (x->xso.dir) { 94 case XFRM_DEV_OFFLOAD_IN: 95 esn = x->replay_esn->seq; 96 esn_msb = x->replay_esn->seq_hi; 97 break; 98 case XFRM_DEV_OFFLOAD_OUT: 99 esn = x->replay_esn->oseq; 100 esn_msb = x->replay_esn->oseq_hi; 101 break; 102 default: 103 WARN_ON(true); 104 return false; 105 } 106 break; 107 case XFRM_DEV_OFFLOAD_CRYPTO: 108 /* Already parsed by XFRM core */ 109 esn = x->replay_esn->seq; 110 break; 111 default: 112 WARN_ON(true); 113 return false; 114 } 115 116 overlap = sa_entry->esn_state.overlap; 117 118 if (esn >= x->replay_esn->replay_window) 119 seq_bottom = esn - x->replay_esn->replay_window + 1; 120 121 if (x->xso.type == XFRM_DEV_OFFLOAD_CRYPTO) 122 esn_msb = xfrm_replay_seqhi(x, htonl(seq_bottom)); 123 124 if (sa_entry->esn_state.esn_msb) 125 sa_entry->esn_state.esn = esn; 126 else 127 /* According to RFC4303, section "3.3.3. Sequence Number Generation", 128 * the first packet sent using a given SA will contain a sequence 129 * number of 1. 130 */ 131 sa_entry->esn_state.esn = max_t(u32, esn, 1); 132 sa_entry->esn_state.esn_msb = esn_msb; 133 134 if (unlikely(overlap && seq_bottom < MLX5E_IPSEC_ESN_SCOPE_MID)) { 135 sa_entry->esn_state.overlap = 0; 136 return true; 137 } else if (unlikely(!overlap && 138 (seq_bottom >= MLX5E_IPSEC_ESN_SCOPE_MID))) { 139 sa_entry->esn_state.overlap = 1; 140 return true; 141 } 142 143 return false; 144} 145 146static void mlx5e_ipsec_init_limits(struct mlx5e_ipsec_sa_entry *sa_entry, 147 struct mlx5_accel_esp_xfrm_attrs *attrs) 148{ 149 struct xfrm_state *x = sa_entry->x; 150 s64 start_value, n; 151 152 attrs->lft.hard_packet_limit = x->lft.hard_packet_limit; 153 attrs->lft.soft_packet_limit = x->lft.soft_packet_limit; 154 if (x->lft.soft_packet_limit == XFRM_INF) 155 return; 156 157 /* Compute hard limit initial value and number of rounds. 158 * 159 * The counting pattern of hardware counter goes: 160 * value -> 2^31-1 161 * 2^31 | (2^31-1) -> 2^31-1 162 * 2^31 | (2^31-1) -> 2^31-1 163 * [..] 164 * 2^31 | (2^31-1) -> 0 165 * 166 * The pattern is created by using an ASO operation to atomically set 167 * bit 31 after the down counter clears bit 31. This is effectively an 168 * atomic addition of 2**31 to the counter. 169 * 170 * We wish to configure the counter, within the above pattern, so that 171 * when it reaches 0, it has hit the hard limit. This is defined by this 172 * system of equations: 173 * 174 * hard_limit == start_value + n * 2^31 175 * n >= 0 176 * start_value < 2^32, start_value >= 0 177 * 178 * These equations are not single-solution, there are often two choices: 179 * hard_limit == start_value + n * 2^31 180 * hard_limit == (start_value+2^31) + (n-1) * 2^31 181 * 182 * The algorithm selects the solution that keeps the counter value 183 * above 2^31 until the final iteration. 184 */ 185 186 /* Start by estimating n and compute start_value */ 187 n = attrs->lft.hard_packet_limit / BIT_ULL(31); 188 start_value = attrs->lft.hard_packet_limit - n * BIT_ULL(31); 189 190 /* Choose the best of the two solutions: */ 191 if (n >= 1) 192 n -= 1; 193 194 /* Computed values solve the system of equations: */ 195 start_value = attrs->lft.hard_packet_limit - n * BIT_ULL(31); 196 197 /* The best solution means: when there are multiple iterations we must 198 * start above 2^31 and count down to 2**31 to get the interrupt. 199 */ 200 attrs->lft.hard_packet_limit = lower_32_bits(start_value); 201 attrs->lft.numb_rounds_hard = (u64)n; 202 203 /* Compute soft limit initial value and number of rounds. 204 * 205 * The soft_limit is achieved by adjusting the counter's 206 * interrupt_value. This is embedded in the counting pattern created by 207 * hard packet calculations above. 208 * 209 * We wish to compute the interrupt_value for the soft_limit. This is 210 * defined by this system of equations: 211 * 212 * soft_limit == start_value - soft_value + n * 2^31 213 * n >= 0 214 * soft_value < 2^32, soft_value >= 0 215 * for n == 0 start_value > soft_value 216 * 217 * As with compute_hard_n_value() the equations are not single-solution. 218 * The algorithm selects the solution that has: 219 * 2^30 <= soft_limit < 2^31 + 2^30 220 * for the interior iterations, which guarantees a large guard band 221 * around the counter hard limit and next interrupt. 222 */ 223 224 /* Start by estimating n and compute soft_value */ 225 n = (x->lft.soft_packet_limit - attrs->lft.hard_packet_limit) / BIT_ULL(31); 226 start_value = attrs->lft.hard_packet_limit + n * BIT_ULL(31) - 227 x->lft.soft_packet_limit; 228 229 /* Compare against constraints and adjust n */ 230 if (n < 0) 231 n = 0; 232 else if (start_value >= BIT_ULL(32)) 233 n -= 1; 234 else if (start_value < 0) 235 n += 1; 236 237 /* Choose the best of the two solutions: */ 238 start_value = attrs->lft.hard_packet_limit + n * BIT_ULL(31) - start_value; 239 if (n != attrs->lft.numb_rounds_hard && start_value < BIT_ULL(30)) 240 n += 1; 241 242 /* Note that the upper limit of soft_value happens naturally because we 243 * always select the lowest soft_value. 244 */ 245 246 /* Computed values solve the system of equations: */ 247 start_value = attrs->lft.hard_packet_limit + n * BIT_ULL(31) - start_value; 248 249 /* The best solution means: when there are multiple iterations we must 250 * not fall below 2^30 as that would get too close to the false 251 * hard_limit and when we reach an interior iteration for soft_limit it 252 * has to be far away from 2**32-1 which is the counter reset point 253 * after the +2^31 to accommodate latency. 254 */ 255 attrs->lft.soft_packet_limit = lower_32_bits(start_value); 256 attrs->lft.numb_rounds_soft = (u64)n; 257} 258 259static void mlx5e_ipsec_init_macs(struct mlx5e_ipsec_sa_entry *sa_entry, 260 struct mlx5_accel_esp_xfrm_attrs *attrs) 261{ 262 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 263 struct xfrm_state *x = sa_entry->x; 264 struct net_device *netdev; 265 struct neighbour *n; 266 u8 addr[ETH_ALEN]; 267 const void *pkey; 268 u8 *dst, *src; 269 270 if (attrs->mode != XFRM_MODE_TUNNEL || 271 attrs->type != XFRM_DEV_OFFLOAD_PACKET) 272 return; 273 274 netdev = x->xso.real_dev; 275 276 mlx5_query_mac_address(mdev, addr); 277 switch (attrs->dir) { 278 case XFRM_DEV_OFFLOAD_IN: 279 src = attrs->dmac; 280 dst = attrs->smac; 281 pkey = &attrs->saddr.a4; 282 break; 283 case XFRM_DEV_OFFLOAD_OUT: 284 src = attrs->smac; 285 dst = attrs->dmac; 286 pkey = &attrs->daddr.a4; 287 break; 288 default: 289 return; 290 } 291 292 ether_addr_copy(src, addr); 293 n = neigh_lookup(&arp_tbl, pkey, netdev); 294 if (!n) { 295 n = neigh_create(&arp_tbl, pkey, netdev); 296 if (IS_ERR(n)) 297 return; 298 neigh_event_send(n, NULL); 299 attrs->drop = true; 300 } else { 301 neigh_ha_snapshot(addr, n, netdev); 302 ether_addr_copy(dst, addr); 303 } 304 neigh_release(n); 305} 306 307void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, 308 struct mlx5_accel_esp_xfrm_attrs *attrs) 309{ 310 struct xfrm_state *x = sa_entry->x; 311 struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm; 312 struct aead_geniv_ctx *geniv_ctx; 313 struct crypto_aead *aead; 314 unsigned int crypto_data_len, key_len; 315 int ivsize; 316 317 memset(attrs, 0, sizeof(*attrs)); 318 319 /* key */ 320 crypto_data_len = (x->aead->alg_key_len + 7) / 8; 321 key_len = crypto_data_len - 4; /* 4 bytes salt at end */ 322 323 memcpy(aes_gcm->aes_key, x->aead->alg_key, key_len); 324 aes_gcm->key_len = key_len * 8; 325 326 /* salt and seq_iv */ 327 aead = x->data; 328 geniv_ctx = crypto_aead_ctx(aead); 329 ivsize = crypto_aead_ivsize(aead); 330 memcpy(&aes_gcm->seq_iv, &geniv_ctx->salt, ivsize); 331 memcpy(&aes_gcm->salt, x->aead->alg_key + key_len, 332 sizeof(aes_gcm->salt)); 333 334 attrs->authsize = crypto_aead_authsize(aead) / 4; /* in dwords */ 335 336 /* iv len */ 337 aes_gcm->icv_len = x->aead->alg_icv_len; 338 339 attrs->dir = x->xso.dir; 340 341 /* esn */ 342 if (x->props.flags & XFRM_STATE_ESN) { 343 attrs->replay_esn.trigger = true; 344 attrs->replay_esn.esn = sa_entry->esn_state.esn; 345 attrs->replay_esn.esn_msb = sa_entry->esn_state.esn_msb; 346 attrs->replay_esn.overlap = sa_entry->esn_state.overlap; 347 if (attrs->dir == XFRM_DEV_OFFLOAD_OUT) 348 goto skip_replay_window; 349 350 switch (x->replay_esn->replay_window) { 351 case 32: 352 attrs->replay_esn.replay_window = 353 MLX5_IPSEC_ASO_REPLAY_WIN_32BIT; 354 break; 355 case 64: 356 attrs->replay_esn.replay_window = 357 MLX5_IPSEC_ASO_REPLAY_WIN_64BIT; 358 break; 359 case 128: 360 attrs->replay_esn.replay_window = 361 MLX5_IPSEC_ASO_REPLAY_WIN_128BIT; 362 break; 363 case 256: 364 attrs->replay_esn.replay_window = 365 MLX5_IPSEC_ASO_REPLAY_WIN_256BIT; 366 break; 367 default: 368 WARN_ON(true); 369 return; 370 } 371 } 372 373skip_replay_window: 374 /* spi */ 375 attrs->spi = be32_to_cpu(x->id.spi); 376 377 /* source , destination ips */ 378 memcpy(&attrs->saddr, x->props.saddr.a6, sizeof(attrs->saddr)); 379 memcpy(&attrs->daddr, x->id.daddr.a6, sizeof(attrs->daddr)); 380 attrs->family = x->props.family; 381 attrs->type = x->xso.type; 382 attrs->reqid = x->props.reqid; 383 attrs->upspec.dport = ntohs(x->sel.dport); 384 attrs->upspec.dport_mask = ntohs(x->sel.dport_mask); 385 attrs->upspec.sport = ntohs(x->sel.sport); 386 attrs->upspec.sport_mask = ntohs(x->sel.sport_mask); 387 attrs->upspec.proto = x->sel.proto; 388 attrs->mode = x->props.mode; 389 390 mlx5e_ipsec_init_limits(sa_entry, attrs); 391 mlx5e_ipsec_init_macs(sa_entry, attrs); 392 393 if (x->encap) { 394 attrs->encap = true; 395 attrs->sport = x->encap->encap_sport; 396 attrs->dport = x->encap->encap_dport; 397 } 398} 399 400static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev, 401 struct xfrm_state *x, 402 struct netlink_ext_ack *extack) 403{ 404 if (x->props.aalgo != SADB_AALG_NONE) { 405 NL_SET_ERR_MSG_MOD(extack, "Cannot offload authenticated xfrm states"); 406 return -EINVAL; 407 } 408 if (x->props.ealgo != SADB_X_EALG_AES_GCM_ICV16) { 409 NL_SET_ERR_MSG_MOD(extack, "Only AES-GCM-ICV16 xfrm state may be offloaded"); 410 return -EINVAL; 411 } 412 if (x->props.calgo != SADB_X_CALG_NONE) { 413 NL_SET_ERR_MSG_MOD(extack, "Cannot offload compressed xfrm states"); 414 return -EINVAL; 415 } 416 if (x->props.flags & XFRM_STATE_ESN && 417 !(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ESN)) { 418 NL_SET_ERR_MSG_MOD(extack, "Cannot offload ESN xfrm states"); 419 return -EINVAL; 420 } 421 if (x->props.family != AF_INET && 422 x->props.family != AF_INET6) { 423 NL_SET_ERR_MSG_MOD(extack, "Only IPv4/6 xfrm states may be offloaded"); 424 return -EINVAL; 425 } 426 if (x->id.proto != IPPROTO_ESP) { 427 NL_SET_ERR_MSG_MOD(extack, "Only ESP xfrm state may be offloaded"); 428 return -EINVAL; 429 } 430 if (x->encap) { 431 if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ESPINUDP)) { 432 NL_SET_ERR_MSG_MOD(extack, "Encapsulation is not supported"); 433 return -EINVAL; 434 } 435 436 if (x->encap->encap_type != UDP_ENCAP_ESPINUDP) { 437 NL_SET_ERR_MSG_MOD(extack, "Encapsulation other than UDP is not supported"); 438 return -EINVAL; 439 } 440 441 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) { 442 NL_SET_ERR_MSG_MOD(extack, "Encapsulation is supported in packet offload mode only"); 443 return -EINVAL; 444 } 445 446 if (x->props.mode != XFRM_MODE_TRANSPORT) { 447 NL_SET_ERR_MSG_MOD(extack, "Encapsulation is supported in transport mode only"); 448 return -EINVAL; 449 } 450 } 451 if (!x->aead) { 452 NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states without aead"); 453 return -EINVAL; 454 } 455 if (x->aead->alg_icv_len != 128) { 456 NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states with AEAD ICV length other than 128bit"); 457 return -EINVAL; 458 } 459 if ((x->aead->alg_key_len != 128 + 32) && 460 (x->aead->alg_key_len != 256 + 32)) { 461 NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states with AEAD key length other than 128/256 bit"); 462 return -EINVAL; 463 } 464 if (x->tfcpad) { 465 NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states with tfc padding"); 466 return -EINVAL; 467 } 468 if (!x->geniv) { 469 NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states without geniv"); 470 return -EINVAL; 471 } 472 if (strcmp(x->geniv, "seqiv")) { 473 NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states with geniv other than seqiv"); 474 return -EINVAL; 475 } 476 477 if (x->sel.proto != IPPROTO_IP && x->sel.proto != IPPROTO_UDP && 478 x->sel.proto != IPPROTO_TCP) { 479 NL_SET_ERR_MSG_MOD(extack, "Device does not support upper protocol other than TCP/UDP"); 480 return -EINVAL; 481 } 482 483 if (x->props.mode != XFRM_MODE_TRANSPORT && x->props.mode != XFRM_MODE_TUNNEL) { 484 NL_SET_ERR_MSG_MOD(extack, "Only transport and tunnel xfrm states may be offloaded"); 485 return -EINVAL; 486 } 487 488 switch (x->xso.type) { 489 case XFRM_DEV_OFFLOAD_CRYPTO: 490 if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_CRYPTO)) { 491 NL_SET_ERR_MSG_MOD(extack, "Crypto offload is not supported"); 492 return -EINVAL; 493 } 494 495 break; 496 case XFRM_DEV_OFFLOAD_PACKET: 497 if (!(mlx5_ipsec_device_caps(mdev) & 498 MLX5_IPSEC_CAP_PACKET_OFFLOAD)) { 499 NL_SET_ERR_MSG_MOD(extack, "Packet offload is not supported"); 500 return -EINVAL; 501 } 502 503 if (x->props.mode == XFRM_MODE_TUNNEL && 504 !(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_TUNNEL)) { 505 NL_SET_ERR_MSG_MOD(extack, "Packet offload is not supported for tunnel mode"); 506 return -EINVAL; 507 } 508 509 if (x->replay_esn && x->xso.dir == XFRM_DEV_OFFLOAD_IN && 510 x->replay_esn->replay_window != 32 && 511 x->replay_esn->replay_window != 64 && 512 x->replay_esn->replay_window != 128 && 513 x->replay_esn->replay_window != 256) { 514 NL_SET_ERR_MSG_MOD(extack, "Unsupported replay window size"); 515 return -EINVAL; 516 } 517 518 if (!x->props.reqid) { 519 NL_SET_ERR_MSG_MOD(extack, "Cannot offload without reqid"); 520 return -EINVAL; 521 } 522 523 if (x->lft.soft_byte_limit >= x->lft.hard_byte_limit && 524 x->lft.hard_byte_limit != XFRM_INF) { 525 /* XFRM stack doesn't prevent such configuration :(. */ 526 NL_SET_ERR_MSG_MOD(extack, "Hard byte limit must be greater than soft one"); 527 return -EINVAL; 528 } 529 530 if (!x->lft.soft_byte_limit || !x->lft.hard_byte_limit) { 531 NL_SET_ERR_MSG_MOD(extack, "Soft/hard byte limits can't be 0"); 532 return -EINVAL; 533 } 534 535 if (x->lft.soft_packet_limit >= x->lft.hard_packet_limit && 536 x->lft.hard_packet_limit != XFRM_INF) { 537 /* XFRM stack doesn't prevent such configuration :(. */ 538 NL_SET_ERR_MSG_MOD(extack, "Hard packet limit must be greater than soft one"); 539 return -EINVAL; 540 } 541 542 if (!x->lft.soft_packet_limit || !x->lft.hard_packet_limit) { 543 NL_SET_ERR_MSG_MOD(extack, "Soft/hard packet limits can't be 0"); 544 return -EINVAL; 545 } 546 break; 547 default: 548 NL_SET_ERR_MSG_MOD(extack, "Unsupported xfrm offload type"); 549 return -EINVAL; 550 } 551 return 0; 552} 553 554static void mlx5e_ipsec_modify_state(struct work_struct *_work) 555{ 556 struct mlx5e_ipsec_work *work = 557 container_of(_work, struct mlx5e_ipsec_work, work); 558 struct mlx5e_ipsec_sa_entry *sa_entry = work->sa_entry; 559 struct mlx5_accel_esp_xfrm_attrs *attrs; 560 561 attrs = &((struct mlx5e_ipsec_sa_entry *)work->data)->attrs; 562 563 mlx5_accel_esp_modify_xfrm(sa_entry, attrs); 564} 565 566static void mlx5e_ipsec_set_esn_ops(struct mlx5e_ipsec_sa_entry *sa_entry) 567{ 568 struct xfrm_state *x = sa_entry->x; 569 570 if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO || 571 x->xso.dir != XFRM_DEV_OFFLOAD_OUT) 572 return; 573 574 if (x->props.flags & XFRM_STATE_ESN) { 575 sa_entry->set_iv_op = mlx5e_ipsec_set_iv_esn; 576 return; 577 } 578 579 sa_entry->set_iv_op = mlx5e_ipsec_set_iv; 580} 581 582static void mlx5e_ipsec_handle_netdev_event(struct work_struct *_work) 583{ 584 struct mlx5e_ipsec_work *work = 585 container_of(_work, struct mlx5e_ipsec_work, work); 586 struct mlx5e_ipsec_sa_entry *sa_entry = work->sa_entry; 587 struct mlx5e_ipsec_netevent_data *data = work->data; 588 struct mlx5_accel_esp_xfrm_attrs *attrs; 589 590 attrs = &sa_entry->attrs; 591 592 switch (attrs->dir) { 593 case XFRM_DEV_OFFLOAD_IN: 594 ether_addr_copy(attrs->smac, data->addr); 595 break; 596 case XFRM_DEV_OFFLOAD_OUT: 597 ether_addr_copy(attrs->dmac, data->addr); 598 break; 599 default: 600 WARN_ON_ONCE(true); 601 } 602 attrs->drop = false; 603 mlx5e_accel_ipsec_fs_modify(sa_entry); 604} 605 606static int mlx5_ipsec_create_work(struct mlx5e_ipsec_sa_entry *sa_entry) 607{ 608 struct xfrm_state *x = sa_entry->x; 609 struct mlx5e_ipsec_work *work; 610 void *data = NULL; 611 612 switch (x->xso.type) { 613 case XFRM_DEV_OFFLOAD_CRYPTO: 614 if (!(x->props.flags & XFRM_STATE_ESN)) 615 return 0; 616 break; 617 case XFRM_DEV_OFFLOAD_PACKET: 618 if (x->props.mode != XFRM_MODE_TUNNEL) 619 return 0; 620 break; 621 default: 622 break; 623 } 624 625 work = kzalloc(sizeof(*work), GFP_KERNEL); 626 if (!work) 627 return -ENOMEM; 628 629 switch (x->xso.type) { 630 case XFRM_DEV_OFFLOAD_CRYPTO: 631 data = kzalloc(sizeof(*sa_entry), GFP_KERNEL); 632 if (!data) 633 goto free_work; 634 635 INIT_WORK(&work->work, mlx5e_ipsec_modify_state); 636 break; 637 case XFRM_DEV_OFFLOAD_PACKET: 638 data = kzalloc(sizeof(struct mlx5e_ipsec_netevent_data), 639 GFP_KERNEL); 640 if (!data) 641 goto free_work; 642 643 INIT_WORK(&work->work, mlx5e_ipsec_handle_netdev_event); 644 break; 645 default: 646 break; 647 } 648 649 work->data = data; 650 work->sa_entry = sa_entry; 651 sa_entry->work = work; 652 return 0; 653 654free_work: 655 kfree(work); 656 return -ENOMEM; 657} 658 659static int mlx5e_ipsec_create_dwork(struct mlx5e_ipsec_sa_entry *sa_entry) 660{ 661 struct xfrm_state *x = sa_entry->x; 662 struct mlx5e_ipsec_dwork *dwork; 663 664 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) 665 return 0; 666 667 if (x->lft.soft_packet_limit == XFRM_INF && 668 x->lft.hard_packet_limit == XFRM_INF && 669 x->lft.soft_byte_limit == XFRM_INF && 670 x->lft.hard_byte_limit == XFRM_INF) 671 return 0; 672 673 dwork = kzalloc(sizeof(*dwork), GFP_KERNEL); 674 if (!dwork) 675 return -ENOMEM; 676 677 dwork->sa_entry = sa_entry; 678 INIT_DELAYED_WORK(&dwork->dwork, mlx5e_ipsec_handle_sw_limits); 679 sa_entry->dwork = dwork; 680 return 0; 681} 682 683static int mlx5e_xfrm_add_state(struct xfrm_state *x, 684 struct netlink_ext_ack *extack) 685{ 686 struct mlx5e_ipsec_sa_entry *sa_entry = NULL; 687 struct net_device *netdev = x->xso.real_dev; 688 struct mlx5e_ipsec *ipsec; 689 struct mlx5e_priv *priv; 690 gfp_t gfp; 691 int err; 692 693 priv = netdev_priv(netdev); 694 if (!priv->ipsec) 695 return -EOPNOTSUPP; 696 697 ipsec = priv->ipsec; 698 gfp = (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ) ? GFP_ATOMIC : GFP_KERNEL; 699 sa_entry = kzalloc(sizeof(*sa_entry), gfp); 700 if (!sa_entry) 701 return -ENOMEM; 702 703 sa_entry->x = x; 704 sa_entry->ipsec = ipsec; 705 /* Check if this SA is originated from acquire flow temporary SA */ 706 if (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ) 707 goto out; 708 709 err = mlx5e_xfrm_validate_state(priv->mdev, x, extack); 710 if (err) 711 goto err_xfrm; 712 713 if (!mlx5_eswitch_block_ipsec(priv->mdev)) { 714 err = -EBUSY; 715 goto err_xfrm; 716 } 717 718 /* check esn */ 719 if (x->props.flags & XFRM_STATE_ESN) 720 mlx5e_ipsec_update_esn_state(sa_entry); 721 722 mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &sa_entry->attrs); 723 724 err = mlx5_ipsec_create_work(sa_entry); 725 if (err) 726 goto unblock_ipsec; 727 728 err = mlx5e_ipsec_create_dwork(sa_entry); 729 if (err) 730 goto release_work; 731 732 /* create hw context */ 733 err = mlx5_ipsec_create_sa_ctx(sa_entry); 734 if (err) 735 goto release_dwork; 736 737 err = mlx5e_accel_ipsec_fs_add_rule(sa_entry); 738 if (err) 739 goto err_hw_ctx; 740 741 if (x->props.mode == XFRM_MODE_TUNNEL && 742 x->xso.type == XFRM_DEV_OFFLOAD_PACKET && 743 !mlx5e_ipsec_fs_tunnel_enabled(sa_entry)) { 744 NL_SET_ERR_MSG_MOD(extack, "Packet offload tunnel mode is disabled due to encap settings"); 745 err = -EINVAL; 746 goto err_add_rule; 747 } 748 749 /* We use *_bh() variant because xfrm_timer_handler(), which runs 750 * in softirq context, can reach our state delete logic and we need 751 * xa_erase_bh() there. 752 */ 753 err = xa_insert_bh(&ipsec->sadb, sa_entry->ipsec_obj_id, sa_entry, 754 GFP_KERNEL); 755 if (err) 756 goto err_add_rule; 757 758 mlx5e_ipsec_set_esn_ops(sa_entry); 759 760 if (sa_entry->dwork) 761 queue_delayed_work(ipsec->wq, &sa_entry->dwork->dwork, 762 MLX5_IPSEC_RESCHED); 763 764 if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET && 765 x->props.mode == XFRM_MODE_TUNNEL) 766 xa_set_mark(&ipsec->sadb, sa_entry->ipsec_obj_id, 767 MLX5E_IPSEC_TUNNEL_SA); 768 769out: 770 x->xso.offload_handle = (unsigned long)sa_entry; 771 return 0; 772 773err_add_rule: 774 mlx5e_accel_ipsec_fs_del_rule(sa_entry); 775err_hw_ctx: 776 mlx5_ipsec_free_sa_ctx(sa_entry); 777release_dwork: 778 kfree(sa_entry->dwork); 779release_work: 780 if (sa_entry->work) 781 kfree(sa_entry->work->data); 782 kfree(sa_entry->work); 783unblock_ipsec: 784 mlx5_eswitch_unblock_ipsec(priv->mdev); 785err_xfrm: 786 kfree(sa_entry); 787 NL_SET_ERR_MSG_WEAK_MOD(extack, "Device failed to offload this state"); 788 return err; 789} 790 791static void mlx5e_xfrm_del_state(struct xfrm_state *x) 792{ 793 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); 794 struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; 795 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 796 struct mlx5e_ipsec_sa_entry *old; 797 798 if (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ) 799 return; 800 801 old = xa_erase_bh(&ipsec->sadb, sa_entry->ipsec_obj_id); 802 WARN_ON(old != sa_entry); 803 804 if (attrs->mode == XFRM_MODE_TUNNEL && 805 attrs->type == XFRM_DEV_OFFLOAD_PACKET) 806 /* Make sure that no ARP requests are running in parallel */ 807 flush_workqueue(ipsec->wq); 808 809} 810 811static void mlx5e_xfrm_free_state(struct xfrm_state *x) 812{ 813 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); 814 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 815 816 if (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ) 817 goto sa_entry_free; 818 819 if (sa_entry->work) 820 cancel_work_sync(&sa_entry->work->work); 821 822 if (sa_entry->dwork) 823 cancel_delayed_work_sync(&sa_entry->dwork->dwork); 824 825 mlx5e_accel_ipsec_fs_del_rule(sa_entry); 826 mlx5_ipsec_free_sa_ctx(sa_entry); 827 kfree(sa_entry->dwork); 828 if (sa_entry->work) 829 kfree(sa_entry->work->data); 830 kfree(sa_entry->work); 831 mlx5_eswitch_unblock_ipsec(ipsec->mdev); 832sa_entry_free: 833 kfree(sa_entry); 834} 835 836static int mlx5e_ipsec_netevent_event(struct notifier_block *nb, 837 unsigned long event, void *ptr) 838{ 839 struct mlx5_accel_esp_xfrm_attrs *attrs; 840 struct mlx5e_ipsec_netevent_data *data; 841 struct mlx5e_ipsec_sa_entry *sa_entry; 842 struct mlx5e_ipsec *ipsec; 843 struct neighbour *n = ptr; 844 struct net_device *netdev; 845 struct xfrm_state *x; 846 unsigned long idx; 847 848 if (event != NETEVENT_NEIGH_UPDATE || !(n->nud_state & NUD_VALID)) 849 return NOTIFY_DONE; 850 851 ipsec = container_of(nb, struct mlx5e_ipsec, netevent_nb); 852 xa_for_each_marked(&ipsec->sadb, idx, sa_entry, MLX5E_IPSEC_TUNNEL_SA) { 853 attrs = &sa_entry->attrs; 854 855 if (attrs->family == AF_INET) { 856 if (!neigh_key_eq32(n, &attrs->saddr.a4) && 857 !neigh_key_eq32(n, &attrs->daddr.a4)) 858 continue; 859 } else { 860 if (!neigh_key_eq128(n, &attrs->saddr.a4) && 861 !neigh_key_eq128(n, &attrs->daddr.a4)) 862 continue; 863 } 864 865 x = sa_entry->x; 866 netdev = x->xso.real_dev; 867 data = sa_entry->work->data; 868 869 neigh_ha_snapshot(data->addr, n, netdev); 870 queue_work(ipsec->wq, &sa_entry->work->work); 871 } 872 873 return NOTIFY_DONE; 874} 875 876void mlx5e_ipsec_init(struct mlx5e_priv *priv) 877{ 878 struct mlx5e_ipsec *ipsec; 879 int ret = -ENOMEM; 880 881 if (!mlx5_ipsec_device_caps(priv->mdev)) { 882 netdev_dbg(priv->netdev, "Not an IPSec offload device\n"); 883 return; 884 } 885 886 ipsec = kzalloc(sizeof(*ipsec), GFP_KERNEL); 887 if (!ipsec) 888 return; 889 890 xa_init_flags(&ipsec->sadb, XA_FLAGS_ALLOC); 891 ipsec->mdev = priv->mdev; 892 init_completion(&ipsec->comp); 893 ipsec->wq = alloc_workqueue("mlx5e_ipsec: %s", WQ_UNBOUND, 0, 894 priv->netdev->name); 895 if (!ipsec->wq) 896 goto err_wq; 897 898 if (mlx5_ipsec_device_caps(priv->mdev) & 899 MLX5_IPSEC_CAP_PACKET_OFFLOAD) { 900 ret = mlx5e_ipsec_aso_init(ipsec); 901 if (ret) 902 goto err_aso; 903 } 904 905 if (mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_TUNNEL) { 906 ipsec->netevent_nb.notifier_call = mlx5e_ipsec_netevent_event; 907 ret = register_netevent_notifier(&ipsec->netevent_nb); 908 if (ret) 909 goto clear_aso; 910 } 911 912 ipsec->is_uplink_rep = mlx5e_is_uplink_rep(priv); 913 ret = mlx5e_accel_ipsec_fs_init(ipsec, &priv->devcom); 914 if (ret) 915 goto err_fs_init; 916 917 ipsec->fs = priv->fs; 918 priv->ipsec = ipsec; 919 netdev_dbg(priv->netdev, "IPSec attached to netdevice\n"); 920 return; 921 922err_fs_init: 923 if (mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_TUNNEL) 924 unregister_netevent_notifier(&ipsec->netevent_nb); 925clear_aso: 926 if (mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD) 927 mlx5e_ipsec_aso_cleanup(ipsec); 928err_aso: 929 destroy_workqueue(ipsec->wq); 930err_wq: 931 kfree(ipsec); 932 mlx5_core_err(priv->mdev, "IPSec initialization failed, %d\n", ret); 933 return; 934} 935 936void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv) 937{ 938 struct mlx5e_ipsec *ipsec = priv->ipsec; 939 940 if (!ipsec) 941 return; 942 943 mlx5e_accel_ipsec_fs_cleanup(ipsec); 944 if (ipsec->netevent_nb.notifier_call) { 945 unregister_netevent_notifier(&ipsec->netevent_nb); 946 ipsec->netevent_nb.notifier_call = NULL; 947 } 948 if (ipsec->aso) 949 mlx5e_ipsec_aso_cleanup(ipsec); 950 destroy_workqueue(ipsec->wq); 951 kfree(ipsec); 952 priv->ipsec = NULL; 953} 954 955static bool mlx5e_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x) 956{ 957 if (x->props.family == AF_INET) { 958 /* Offload with IPv4 options is not supported yet */ 959 if (ip_hdr(skb)->ihl > 5) 960 return false; 961 } else { 962 /* Offload with IPv6 extension headers is not support yet */ 963 if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr)) 964 return false; 965 } 966 967 return true; 968} 969 970static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x) 971{ 972 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); 973 struct mlx5e_ipsec_work *work = sa_entry->work; 974 struct mlx5e_ipsec_sa_entry *sa_entry_shadow; 975 bool need_update; 976 977 need_update = mlx5e_ipsec_update_esn_state(sa_entry); 978 if (!need_update) 979 return; 980 981 sa_entry_shadow = work->data; 982 memset(sa_entry_shadow, 0x00, sizeof(*sa_entry_shadow)); 983 mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &sa_entry_shadow->attrs); 984 queue_work(sa_entry->ipsec->wq, &work->work); 985} 986 987static void mlx5e_xfrm_update_stats(struct xfrm_state *x) 988{ 989 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); 990 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 991 struct net *net = dev_net(x->xso.dev); 992 u64 packets, bytes, lastuse; 993 994 lockdep_assert(lockdep_is_held(&x->lock) || 995 lockdep_is_held(&dev_net(x->xso.real_dev)->xfrm.xfrm_cfg_mutex) || 996 lockdep_is_held(&dev_net(x->xso.real_dev)->xfrm.xfrm_state_lock)); 997 998 if (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ) 999 return; 1000 1001 if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_IN) { 1002 mlx5_fc_query_cached(ipsec_rule->auth.fc, &bytes, &packets, &lastuse); 1003 x->stats.integrity_failed += packets; 1004 XFRM_ADD_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR, packets); 1005 1006 mlx5_fc_query_cached(ipsec_rule->trailer.fc, &bytes, &packets, &lastuse); 1007 XFRM_ADD_STATS(net, LINUX_MIB_XFRMINHDRERROR, packets); 1008 } 1009 1010 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) 1011 return; 1012 1013 mlx5_fc_query_cached(ipsec_rule->fc, &bytes, &packets, &lastuse); 1014 x->curlft.packets += packets; 1015 x->curlft.bytes += bytes; 1016 1017 if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_IN) { 1018 mlx5_fc_query_cached(ipsec_rule->replay.fc, &bytes, &packets, &lastuse); 1019 x->stats.replay += packets; 1020 XFRM_ADD_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR, packets); 1021 } 1022} 1023 1024static int mlx5e_xfrm_validate_policy(struct mlx5_core_dev *mdev, 1025 struct xfrm_policy *x, 1026 struct netlink_ext_ack *extack) 1027{ 1028 struct xfrm_selector *sel = &x->selector; 1029 1030 if (x->type != XFRM_POLICY_TYPE_MAIN) { 1031 NL_SET_ERR_MSG_MOD(extack, "Cannot offload non-main policy types"); 1032 return -EINVAL; 1033 } 1034 1035 /* Please pay attention that we support only one template */ 1036 if (x->xfrm_nr > 1) { 1037 NL_SET_ERR_MSG_MOD(extack, "Cannot offload more than one template"); 1038 return -EINVAL; 1039 } 1040 1041 if (x->xdo.dir != XFRM_DEV_OFFLOAD_IN && 1042 x->xdo.dir != XFRM_DEV_OFFLOAD_OUT) { 1043 NL_SET_ERR_MSG_MOD(extack, "Cannot offload forward policy"); 1044 return -EINVAL; 1045 } 1046 1047 if (!x->xfrm_vec[0].reqid && sel->proto == IPPROTO_IP && 1048 addr6_all_zero(sel->saddr.a6) && addr6_all_zero(sel->daddr.a6)) { 1049 NL_SET_ERR_MSG_MOD(extack, "Unsupported policy with reqid 0 without at least one of upper protocol or ip addr(s) different than 0"); 1050 return -EINVAL; 1051 } 1052 1053 if (x->xdo.type != XFRM_DEV_OFFLOAD_PACKET) { 1054 NL_SET_ERR_MSG_MOD(extack, "Unsupported xfrm offload type"); 1055 return -EINVAL; 1056 } 1057 1058 if (x->selector.proto != IPPROTO_IP && 1059 x->selector.proto != IPPROTO_UDP && 1060 x->selector.proto != IPPROTO_TCP) { 1061 NL_SET_ERR_MSG_MOD(extack, "Device does not support upper protocol other than TCP/UDP"); 1062 return -EINVAL; 1063 } 1064 1065 if (x->priority) { 1066 if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO)) { 1067 NL_SET_ERR_MSG_MOD(extack, "Device does not support policy priority"); 1068 return -EINVAL; 1069 } 1070 1071 if (x->priority == U32_MAX) { 1072 NL_SET_ERR_MSG_MOD(extack, "Device does not support requested policy priority"); 1073 return -EINVAL; 1074 } 1075 } 1076 1077 if (x->xdo.type == XFRM_DEV_OFFLOAD_PACKET && 1078 !(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)) { 1079 NL_SET_ERR_MSG_MOD(extack, "Packet offload is not supported"); 1080 return -EINVAL; 1081 } 1082 1083 return 0; 1084} 1085 1086static void 1087mlx5e_ipsec_build_accel_pol_attrs(struct mlx5e_ipsec_pol_entry *pol_entry, 1088 struct mlx5_accel_pol_xfrm_attrs *attrs) 1089{ 1090 struct xfrm_policy *x = pol_entry->x; 1091 struct xfrm_selector *sel; 1092 1093 sel = &x->selector; 1094 memset(attrs, 0, sizeof(*attrs)); 1095 1096 memcpy(&attrs->saddr, sel->saddr.a6, sizeof(attrs->saddr)); 1097 memcpy(&attrs->daddr, sel->daddr.a6, sizeof(attrs->daddr)); 1098 attrs->family = sel->family; 1099 attrs->dir = x->xdo.dir; 1100 attrs->action = x->action; 1101 attrs->type = XFRM_DEV_OFFLOAD_PACKET; 1102 attrs->reqid = x->xfrm_vec[0].reqid; 1103 attrs->upspec.dport = ntohs(sel->dport); 1104 attrs->upspec.dport_mask = ntohs(sel->dport_mask); 1105 attrs->upspec.sport = ntohs(sel->sport); 1106 attrs->upspec.sport_mask = ntohs(sel->sport_mask); 1107 attrs->upspec.proto = sel->proto; 1108 attrs->prio = x->priority; 1109} 1110 1111static int mlx5e_xfrm_add_policy(struct xfrm_policy *x, 1112 struct netlink_ext_ack *extack) 1113{ 1114 struct net_device *netdev = x->xdo.real_dev; 1115 struct mlx5e_ipsec_pol_entry *pol_entry; 1116 struct mlx5e_priv *priv; 1117 int err; 1118 1119 priv = netdev_priv(netdev); 1120 if (!priv->ipsec) { 1121 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPsec packet offload"); 1122 return -EOPNOTSUPP; 1123 } 1124 1125 err = mlx5e_xfrm_validate_policy(priv->mdev, x, extack); 1126 if (err) 1127 return err; 1128 1129 pol_entry = kzalloc(sizeof(*pol_entry), GFP_KERNEL); 1130 if (!pol_entry) 1131 return -ENOMEM; 1132 1133 pol_entry->x = x; 1134 pol_entry->ipsec = priv->ipsec; 1135 1136 if (!mlx5_eswitch_block_ipsec(priv->mdev)) { 1137 err = -EBUSY; 1138 goto ipsec_busy; 1139 } 1140 1141 mlx5e_ipsec_build_accel_pol_attrs(pol_entry, &pol_entry->attrs); 1142 err = mlx5e_accel_ipsec_fs_add_pol(pol_entry); 1143 if (err) 1144 goto err_fs; 1145 1146 x->xdo.offload_handle = (unsigned long)pol_entry; 1147 return 0; 1148 1149err_fs: 1150 mlx5_eswitch_unblock_ipsec(priv->mdev); 1151ipsec_busy: 1152 kfree(pol_entry); 1153 NL_SET_ERR_MSG_MOD(extack, "Device failed to offload this policy"); 1154 return err; 1155} 1156 1157static void mlx5e_xfrm_del_policy(struct xfrm_policy *x) 1158{ 1159 struct mlx5e_ipsec_pol_entry *pol_entry = to_ipsec_pol_entry(x); 1160 1161 mlx5e_accel_ipsec_fs_del_pol(pol_entry); 1162 mlx5_eswitch_unblock_ipsec(pol_entry->ipsec->mdev); 1163} 1164 1165static void mlx5e_xfrm_free_policy(struct xfrm_policy *x) 1166{ 1167 struct mlx5e_ipsec_pol_entry *pol_entry = to_ipsec_pol_entry(x); 1168 1169 kfree(pol_entry); 1170} 1171 1172static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = { 1173 .xdo_dev_state_add = mlx5e_xfrm_add_state, 1174 .xdo_dev_state_delete = mlx5e_xfrm_del_state, 1175 .xdo_dev_state_free = mlx5e_xfrm_free_state, 1176 .xdo_dev_offload_ok = mlx5e_ipsec_offload_ok, 1177 .xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state, 1178 1179 .xdo_dev_state_update_stats = mlx5e_xfrm_update_stats, 1180 .xdo_dev_policy_add = mlx5e_xfrm_add_policy, 1181 .xdo_dev_policy_delete = mlx5e_xfrm_del_policy, 1182 .xdo_dev_policy_free = mlx5e_xfrm_free_policy, 1183}; 1184 1185void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) 1186{ 1187 struct mlx5_core_dev *mdev = priv->mdev; 1188 struct net_device *netdev = priv->netdev; 1189 1190 if (!mlx5_ipsec_device_caps(mdev)) 1191 return; 1192 1193 mlx5_core_info(mdev, "mlx5e: IPSec ESP acceleration enabled\n"); 1194 1195 netdev->xfrmdev_ops = &mlx5e_ipsec_xfrmdev_ops; 1196 netdev->features |= NETIF_F_HW_ESP; 1197 netdev->hw_enc_features |= NETIF_F_HW_ESP; 1198 1199 if (!MLX5_CAP_ETH(mdev, swp_csum)) { 1200 mlx5_core_dbg(mdev, "mlx5e: SWP checksum not supported\n"); 1201 return; 1202 } 1203 1204 netdev->features |= NETIF_F_HW_ESP_TX_CSUM; 1205 netdev->hw_enc_features |= NETIF_F_HW_ESP_TX_CSUM; 1206 1207 if (!MLX5_CAP_ETH(mdev, swp_lso)) { 1208 mlx5_core_dbg(mdev, "mlx5e: ESP LSO not supported\n"); 1209 return; 1210 } 1211 1212 netdev->gso_partial_features |= NETIF_F_GSO_ESP; 1213 mlx5_core_dbg(mdev, "mlx5e: ESP GSO capability turned on\n"); 1214 netdev->features |= NETIF_F_GSO_ESP; 1215 netdev->hw_features |= NETIF_F_GSO_ESP; 1216 netdev->hw_enc_features |= NETIF_F_GSO_ESP; 1217} 1218