1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ 3 4#include "fs_core.h" 5#include "fs_cmd.h" 6#include "en.h" 7#include "lib/ipsec_fs_roce.h" 8#include "mlx5_core.h" 9#include <linux/random.h> 10 11struct mlx5_ipsec_miss { 12 struct mlx5_flow_group *group; 13 struct mlx5_flow_handle *rule; 14}; 15 16struct mlx5_ipsec_rx_roce { 17 struct mlx5_flow_group *g; 18 struct mlx5_flow_table *ft; 19 struct mlx5_flow_handle *rule; 20 struct mlx5_ipsec_miss roce_miss; 21 struct mlx5_flow_table *nic_master_ft; 22 struct mlx5_flow_group *nic_master_group; 23 struct mlx5_flow_handle *nic_master_rule; 24 struct mlx5_flow_table *goto_alias_ft; 25 u32 alias_id; 26 char key[ACCESS_KEY_LEN]; 27 28 struct mlx5_flow_table *ft_rdma; 29 struct mlx5_flow_namespace *ns_rdma; 30}; 31 32struct mlx5_ipsec_tx_roce { 33 struct mlx5_flow_group *g; 34 struct mlx5_flow_table *ft; 35 struct mlx5_flow_handle *rule; 36 struct mlx5_flow_table *goto_alias_ft; 37 u32 alias_id; 38 char key[ACCESS_KEY_LEN]; 39 struct mlx5_flow_namespace *ns; 40}; 41 42struct mlx5_ipsec_fs { 43 struct mlx5_ipsec_rx_roce ipv4_rx; 44 struct mlx5_ipsec_rx_roce ipv6_rx; 45 struct mlx5_ipsec_tx_roce tx; 46 struct mlx5_devcom_comp_dev **devcom; 47}; 48 49static void ipsec_fs_roce_setup_udp_dport(struct mlx5_flow_spec *spec, 50 u16 dport) 51{ 52 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 53 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); 54 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_UDP); 55 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.udp_dport); 56 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, dport); 57} 58 59static bool ipsec_fs_create_alias_supported_one(struct mlx5_core_dev *mdev) 60{ 61 u64 obj_allowed = MLX5_CAP_GEN_2_64(mdev, allowed_object_for_other_vhca_access); 62 u32 obj_supp = MLX5_CAP_GEN_2(mdev, cross_vhca_object_to_object_supported); 63 64 if (!(obj_supp & 65 MLX5_CROSS_VHCA_OBJ_TO_OBJ_SUPPORTED_LOCAL_FLOW_TABLE_TO_REMOTE_FLOW_TABLE_MISS)) 66 return false; 67 68 if (!(obj_allowed & MLX5_ALLOWED_OBJ_FOR_OTHER_VHCA_ACCESS_FLOW_TABLE)) 69 return false; 70 71 return true; 72} 73 74static bool ipsec_fs_create_alias_supported(struct mlx5_core_dev *mdev, 75 struct mlx5_core_dev *master_mdev) 76{ 77 if (ipsec_fs_create_alias_supported_one(mdev) && 78 ipsec_fs_create_alias_supported_one(master_mdev)) 79 return true; 80 81 return false; 82} 83 84static int ipsec_fs_create_aliased_ft(struct mlx5_core_dev *ibv_owner, 85 struct mlx5_core_dev *ibv_allowed, 86 struct mlx5_flow_table *ft, 87 u32 *obj_id, char *alias_key, bool from_event) 88{ 89 u32 aliased_object_id = (ft->type << FT_ID_FT_TYPE_OFFSET) | ft->id; 90 u16 vhca_id_to_be_accessed = MLX5_CAP_GEN(ibv_owner, vhca_id); 91 struct mlx5_cmd_allow_other_vhca_access_attr allow_attr = {}; 92 struct mlx5_cmd_alias_obj_create_attr alias_attr = {}; 93 int ret; 94 int i; 95 96 if (!ipsec_fs_create_alias_supported(ibv_owner, ibv_allowed)) 97 return -EOPNOTSUPP; 98 99 for (i = 0; i < ACCESS_KEY_LEN; i++) 100 if (!from_event) 101 alias_key[i] = get_random_u64() & 0xFF; 102 103 memcpy(allow_attr.access_key, alias_key, ACCESS_KEY_LEN); 104 allow_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS; 105 allow_attr.obj_id = aliased_object_id; 106 107 if (!from_event) { 108 ret = mlx5_cmd_allow_other_vhca_access(ibv_owner, &allow_attr); 109 if (ret) { 110 mlx5_core_err(ibv_owner, "Failed to allow other vhca access err=%d\n", 111 ret); 112 return ret; 113 } 114 } 115 116 memcpy(alias_attr.access_key, alias_key, ACCESS_KEY_LEN); 117 alias_attr.obj_id = aliased_object_id; 118 alias_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS; 119 alias_attr.vhca_id = vhca_id_to_be_accessed; 120 ret = mlx5_cmd_alias_obj_create(ibv_allowed, &alias_attr, obj_id); 121 if (ret) { 122 mlx5_core_err(ibv_allowed, "Failed to create alias object err=%d\n", 123 ret); 124 return ret; 125 } 126 127 return 0; 128} 129 130static int 131ipsec_fs_roce_rx_rule_setup(struct mlx5_core_dev *mdev, 132 struct mlx5_flow_destination *default_dst, 133 struct mlx5_ipsec_rx_roce *roce) 134{ 135 bool is_mpv_slave = mlx5_core_is_mp_slave(mdev); 136 struct mlx5_flow_destination dst = {}; 137 MLX5_DECLARE_FLOW_ACT(flow_act); 138 struct mlx5_flow_handle *rule; 139 struct mlx5_flow_spec *spec; 140 int err = 0; 141 142 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 143 if (!spec) 144 return -ENOMEM; 145 146 ipsec_fs_roce_setup_udp_dport(spec, ROCE_V2_UDP_DPORT); 147 148 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 149 if (is_mpv_slave) { 150 dst.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 151 dst.ft = roce->goto_alias_ft; 152 } else { 153 dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE; 154 dst.ft = roce->ft_rdma; 155 } 156 rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, &dst, 1); 157 if (IS_ERR(rule)) { 158 err = PTR_ERR(rule); 159 mlx5_core_err(mdev, "Fail to add RX RoCE IPsec rule err=%d\n", 160 err); 161 goto out; 162 } 163 164 roce->rule = rule; 165 166 memset(spec, 0, sizeof(*spec)); 167 rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, default_dst, 1); 168 if (IS_ERR(rule)) { 169 err = PTR_ERR(rule); 170 mlx5_core_err(mdev, "Fail to add RX RoCE IPsec miss rule err=%d\n", 171 err); 172 goto fail_add_default_rule; 173 } 174 175 roce->roce_miss.rule = rule; 176 177 if (!is_mpv_slave) 178 goto out; 179 180 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 181 dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE; 182 dst.ft = roce->ft_rdma; 183 rule = mlx5_add_flow_rules(roce->nic_master_ft, NULL, &flow_act, &dst, 184 1); 185 if (IS_ERR(rule)) { 186 err = PTR_ERR(rule); 187 mlx5_core_err(mdev, "Fail to add RX RoCE IPsec rule for alias err=%d\n", 188 err); 189 goto fail_add_nic_master_rule; 190 } 191 roce->nic_master_rule = rule; 192 193 kvfree(spec); 194 return 0; 195 196fail_add_nic_master_rule: 197 mlx5_del_flow_rules(roce->roce_miss.rule); 198fail_add_default_rule: 199 mlx5_del_flow_rules(roce->rule); 200out: 201 kvfree(spec); 202 return err; 203} 204 205static int ipsec_fs_roce_tx_rule_setup(struct mlx5_core_dev *mdev, 206 struct mlx5_ipsec_tx_roce *roce, 207 struct mlx5_flow_table *pol_ft) 208{ 209 struct mlx5_flow_destination dst = {}; 210 MLX5_DECLARE_FLOW_ACT(flow_act); 211 struct mlx5_flow_handle *rule; 212 int err = 0; 213 214 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 215 dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE; 216 dst.ft = pol_ft; 217 rule = mlx5_add_flow_rules(roce->ft, NULL, &flow_act, &dst, 218 1); 219 if (IS_ERR(rule)) { 220 err = PTR_ERR(rule); 221 mlx5_core_err(mdev, "Fail to add TX RoCE IPsec rule err=%d\n", 222 err); 223 goto out; 224 } 225 roce->rule = rule; 226 227out: 228 return err; 229} 230 231static int ipsec_fs_roce_tx_mpv_rule_setup(struct mlx5_core_dev *mdev, 232 struct mlx5_ipsec_tx_roce *roce, 233 struct mlx5_flow_table *pol_ft) 234{ 235 struct mlx5_flow_destination dst = {}; 236 MLX5_DECLARE_FLOW_ACT(flow_act); 237 struct mlx5_flow_handle *rule; 238 struct mlx5_flow_spec *spec; 239 int err = 0; 240 241 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 242 if (!spec) 243 return -ENOMEM; 244 245 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 246 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.source_vhca_port); 247 MLX5_SET(fte_match_param, spec->match_value, misc_parameters.source_vhca_port, 248 MLX5_CAP_GEN(mdev, native_port_num)); 249 250 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 251 dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE; 252 dst.ft = roce->goto_alias_ft; 253 rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, &dst, 1); 254 if (IS_ERR(rule)) { 255 err = PTR_ERR(rule); 256 mlx5_core_err(mdev, "Fail to add TX RoCE IPsec rule err=%d\n", 257 err); 258 goto out; 259 } 260 roce->rule = rule; 261 262 /* No need for miss rule, since on miss we go to next PRIO, in which 263 * if master is configured, he will catch the traffic to go to his 264 * encryption table. 265 */ 266 267out: 268 kvfree(spec); 269 return err; 270} 271 272#define MLX5_TX_ROCE_GROUP_SIZE BIT(0) 273#define MLX5_IPSEC_RDMA_TX_FT_LEVEL 0 274#define MLX5_IPSEC_NIC_GOTO_ALIAS_FT_LEVEL 3 /* Since last used level in NIC ipsec is 2 */ 275 276static int ipsec_fs_roce_tx_mpv_create_ft(struct mlx5_core_dev *mdev, 277 struct mlx5_ipsec_tx_roce *roce, 278 struct mlx5_flow_table *pol_ft, 279 struct mlx5e_priv *peer_priv, 280 bool from_event) 281{ 282 struct mlx5_flow_namespace *roce_ns, *nic_ns; 283 struct mlx5_flow_table_attr ft_attr = {}; 284 struct mlx5_flow_table next_ft; 285 struct mlx5_flow_table *ft; 286 int err; 287 288 roce_ns = mlx5_get_flow_namespace(peer_priv->mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC); 289 if (!roce_ns) 290 return -EOPNOTSUPP; 291 292 nic_ns = mlx5_get_flow_namespace(peer_priv->mdev, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC); 293 if (!nic_ns) 294 return -EOPNOTSUPP; 295 296 err = ipsec_fs_create_aliased_ft(mdev, peer_priv->mdev, pol_ft, &roce->alias_id, roce->key, 297 from_event); 298 if (err) 299 return err; 300 301 next_ft.id = roce->alias_id; 302 ft_attr.max_fte = 1; 303 ft_attr.next_ft = &next_ft; 304 ft_attr.level = MLX5_IPSEC_NIC_GOTO_ALIAS_FT_LEVEL; 305 ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED; 306 ft = mlx5_create_flow_table(nic_ns, &ft_attr); 307 if (IS_ERR(ft)) { 308 err = PTR_ERR(ft); 309 mlx5_core_err(mdev, "Fail to create RoCE IPsec goto alias ft err=%d\n", err); 310 goto destroy_alias; 311 } 312 313 roce->goto_alias_ft = ft; 314 315 memset(&ft_attr, 0, sizeof(ft_attr)); 316 ft_attr.max_fte = 1; 317 ft_attr.level = MLX5_IPSEC_RDMA_TX_FT_LEVEL; 318 ft = mlx5_create_flow_table(roce_ns, &ft_attr); 319 if (IS_ERR(ft)) { 320 err = PTR_ERR(ft); 321 mlx5_core_err(mdev, "Fail to create RoCE IPsec tx ft err=%d\n", err); 322 goto destroy_alias_ft; 323 } 324 325 roce->ft = ft; 326 327 return 0; 328 329destroy_alias_ft: 330 mlx5_destroy_flow_table(roce->goto_alias_ft); 331destroy_alias: 332 mlx5_cmd_alias_obj_destroy(peer_priv->mdev, roce->alias_id, 333 MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS); 334 return err; 335} 336 337static int ipsec_fs_roce_tx_mpv_create_group_rules(struct mlx5_core_dev *mdev, 338 struct mlx5_ipsec_tx_roce *roce, 339 struct mlx5_flow_table *pol_ft, 340 u32 *in) 341{ 342 struct mlx5_flow_group *g; 343 int ix = 0; 344 int err; 345 u8 *mc; 346 347 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 348 MLX5_SET_TO_ONES(fte_match_param, mc, misc_parameters.source_vhca_port); 349 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_MISC_PARAMETERS); 350 351 MLX5_SET_CFG(in, start_flow_index, ix); 352 ix += MLX5_TX_ROCE_GROUP_SIZE; 353 MLX5_SET_CFG(in, end_flow_index, ix - 1); 354 g = mlx5_create_flow_group(roce->ft, in); 355 if (IS_ERR(g)) { 356 err = PTR_ERR(g); 357 mlx5_core_err(mdev, "Fail to create RoCE IPsec tx group err=%d\n", err); 358 return err; 359 } 360 roce->g = g; 361 362 err = ipsec_fs_roce_tx_mpv_rule_setup(mdev, roce, pol_ft); 363 if (err) { 364 mlx5_core_err(mdev, "Fail to create RoCE IPsec tx rules err=%d\n", err); 365 goto destroy_group; 366 } 367 368 return 0; 369 370destroy_group: 371 mlx5_destroy_flow_group(roce->g); 372 return err; 373} 374 375static int ipsec_fs_roce_tx_mpv_create(struct mlx5_core_dev *mdev, 376 struct mlx5_ipsec_fs *ipsec_roce, 377 struct mlx5_flow_table *pol_ft, 378 u32 *in, bool from_event) 379{ 380 struct mlx5_devcom_comp_dev *tmp = NULL; 381 struct mlx5_ipsec_tx_roce *roce; 382 struct mlx5e_priv *peer_priv; 383 int err; 384 385 if (!mlx5_devcom_for_each_peer_begin(*ipsec_roce->devcom)) 386 return -EOPNOTSUPP; 387 388 peer_priv = mlx5_devcom_get_next_peer_data(*ipsec_roce->devcom, &tmp); 389 if (!peer_priv) { 390 err = -EOPNOTSUPP; 391 goto release_peer; 392 } 393 394 roce = &ipsec_roce->tx; 395 396 err = ipsec_fs_roce_tx_mpv_create_ft(mdev, roce, pol_ft, peer_priv, from_event); 397 if (err) { 398 mlx5_core_err(mdev, "Fail to create RoCE IPsec tables err=%d\n", err); 399 goto release_peer; 400 } 401 402 err = ipsec_fs_roce_tx_mpv_create_group_rules(mdev, roce, pol_ft, in); 403 if (err) { 404 mlx5_core_err(mdev, "Fail to create RoCE IPsec tx group/rule err=%d\n", err); 405 goto destroy_tables; 406 } 407 408 mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom); 409 return 0; 410 411destroy_tables: 412 mlx5_destroy_flow_table(roce->ft); 413 mlx5_destroy_flow_table(roce->goto_alias_ft); 414 mlx5_cmd_alias_obj_destroy(peer_priv->mdev, roce->alias_id, 415 MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS); 416release_peer: 417 mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom); 418 return err; 419} 420 421static void roce_rx_mpv_destroy_tables(struct mlx5_core_dev *mdev, struct mlx5_ipsec_rx_roce *roce) 422{ 423 mlx5_destroy_flow_table(roce->goto_alias_ft); 424 mlx5_cmd_alias_obj_destroy(mdev, roce->alias_id, 425 MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS); 426 mlx5_destroy_flow_group(roce->nic_master_group); 427 mlx5_destroy_flow_table(roce->nic_master_ft); 428} 429 430#define MLX5_RX_ROCE_GROUP_SIZE BIT(0) 431#define MLX5_IPSEC_RX_IPV4_FT_LEVEL 3 432#define MLX5_IPSEC_RX_IPV6_FT_LEVEL 2 433 434static int ipsec_fs_roce_rx_mpv_create(struct mlx5_core_dev *mdev, 435 struct mlx5_ipsec_fs *ipsec_roce, 436 struct mlx5_flow_namespace *ns, 437 u32 family, u32 level, u32 prio) 438{ 439 struct mlx5_flow_namespace *roce_ns, *nic_ns; 440 struct mlx5_flow_table_attr ft_attr = {}; 441 struct mlx5_devcom_comp_dev *tmp = NULL; 442 struct mlx5_ipsec_rx_roce *roce; 443 struct mlx5_flow_table next_ft; 444 struct mlx5_flow_table *ft; 445 struct mlx5_flow_group *g; 446 struct mlx5e_priv *peer_priv; 447 int ix = 0; 448 u32 *in; 449 int err; 450 451 roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx : 452 &ipsec_roce->ipv6_rx; 453 454 if (!mlx5_devcom_for_each_peer_begin(*ipsec_roce->devcom)) 455 return -EOPNOTSUPP; 456 457 peer_priv = mlx5_devcom_get_next_peer_data(*ipsec_roce->devcom, &tmp); 458 if (!peer_priv) { 459 err = -EOPNOTSUPP; 460 goto release_peer; 461 } 462 463 roce_ns = mlx5_get_flow_namespace(peer_priv->mdev, MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC); 464 if (!roce_ns) { 465 err = -EOPNOTSUPP; 466 goto release_peer; 467 } 468 469 nic_ns = mlx5_get_flow_namespace(peer_priv->mdev, MLX5_FLOW_NAMESPACE_KERNEL); 470 if (!nic_ns) { 471 err = -EOPNOTSUPP; 472 goto release_peer; 473 } 474 475 in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL); 476 if (!in) { 477 err = -ENOMEM; 478 goto release_peer; 479 } 480 481 ft_attr.level = (family == AF_INET) ? MLX5_IPSEC_RX_IPV4_FT_LEVEL : 482 MLX5_IPSEC_RX_IPV6_FT_LEVEL; 483 ft_attr.max_fte = 1; 484 ft = mlx5_create_flow_table(roce_ns, &ft_attr); 485 if (IS_ERR(ft)) { 486 err = PTR_ERR(ft); 487 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at rdma master err=%d\n", err); 488 goto free_in; 489 } 490 491 roce->ft_rdma = ft; 492 493 ft_attr.max_fte = 1; 494 ft_attr.prio = prio; 495 ft_attr.level = level + 2; 496 ft = mlx5_create_flow_table(nic_ns, &ft_attr); 497 if (IS_ERR(ft)) { 498 err = PTR_ERR(ft); 499 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at NIC master err=%d\n", err); 500 goto destroy_ft_rdma; 501 } 502 roce->nic_master_ft = ft; 503 504 MLX5_SET_CFG(in, start_flow_index, ix); 505 ix += 1; 506 MLX5_SET_CFG(in, end_flow_index, ix - 1); 507 g = mlx5_create_flow_group(roce->nic_master_ft, in); 508 if (IS_ERR(g)) { 509 err = PTR_ERR(g); 510 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx group aliased err=%d\n", err); 511 goto destroy_nic_master_ft; 512 } 513 roce->nic_master_group = g; 514 515 err = ipsec_fs_create_aliased_ft(peer_priv->mdev, mdev, roce->nic_master_ft, 516 &roce->alias_id, roce->key, false); 517 if (err) { 518 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx alias FT err=%d\n", err); 519 goto destroy_group; 520 } 521 522 next_ft.id = roce->alias_id; 523 ft_attr.max_fte = 1; 524 ft_attr.prio = prio; 525 ft_attr.level = roce->ft->level + 1; 526 ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED; 527 ft_attr.next_ft = &next_ft; 528 ft = mlx5_create_flow_table(ns, &ft_attr); 529 if (IS_ERR(ft)) { 530 err = PTR_ERR(ft); 531 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at NIC slave err=%d\n", err); 532 goto destroy_alias; 533 } 534 roce->goto_alias_ft = ft; 535 536 kvfree(in); 537 mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom); 538 return 0; 539 540destroy_alias: 541 mlx5_cmd_alias_obj_destroy(mdev, roce->alias_id, 542 MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS); 543destroy_group: 544 mlx5_destroy_flow_group(roce->nic_master_group); 545destroy_nic_master_ft: 546 mlx5_destroy_flow_table(roce->nic_master_ft); 547destroy_ft_rdma: 548 mlx5_destroy_flow_table(roce->ft_rdma); 549free_in: 550 kvfree(in); 551release_peer: 552 mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom); 553 return err; 554} 555 556void mlx5_ipsec_fs_roce_tx_destroy(struct mlx5_ipsec_fs *ipsec_roce, 557 struct mlx5_core_dev *mdev) 558{ 559 struct mlx5_devcom_comp_dev *tmp = NULL; 560 struct mlx5_ipsec_tx_roce *tx_roce; 561 struct mlx5e_priv *peer_priv; 562 563 if (!ipsec_roce) 564 return; 565 566 tx_roce = &ipsec_roce->tx; 567 568 if (!tx_roce->ft) 569 return; /* Incase RoCE was cleaned from MPV event flow */ 570 571 mlx5_del_flow_rules(tx_roce->rule); 572 mlx5_destroy_flow_group(tx_roce->g); 573 mlx5_destroy_flow_table(tx_roce->ft); 574 575 if (!mlx5_core_is_mp_slave(mdev)) 576 return; 577 578 if (!mlx5_devcom_for_each_peer_begin(*ipsec_roce->devcom)) 579 return; 580 581 peer_priv = mlx5_devcom_get_next_peer_data(*ipsec_roce->devcom, &tmp); 582 if (!peer_priv) { 583 mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom); 584 return; 585 } 586 587 mlx5_destroy_flow_table(tx_roce->goto_alias_ft); 588 mlx5_cmd_alias_obj_destroy(peer_priv->mdev, tx_roce->alias_id, 589 MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS); 590 mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom); 591 tx_roce->ft = NULL; 592} 593 594int mlx5_ipsec_fs_roce_tx_create(struct mlx5_core_dev *mdev, 595 struct mlx5_ipsec_fs *ipsec_roce, 596 struct mlx5_flow_table *pol_ft, 597 bool from_event) 598{ 599 struct mlx5_flow_table_attr ft_attr = {}; 600 struct mlx5_ipsec_tx_roce *roce; 601 struct mlx5_flow_table *ft; 602 struct mlx5_flow_group *g; 603 int ix = 0; 604 int err; 605 u32 *in; 606 607 if (!ipsec_roce) 608 return 0; 609 610 roce = &ipsec_roce->tx; 611 612 in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL); 613 if (!in) 614 return -ENOMEM; 615 616 if (mlx5_core_is_mp_slave(mdev)) { 617 err = ipsec_fs_roce_tx_mpv_create(mdev, ipsec_roce, pol_ft, in, from_event); 618 goto free_in; 619 } 620 621 ft_attr.max_fte = 1; 622 ft_attr.prio = 1; 623 ft_attr.level = MLX5_IPSEC_RDMA_TX_FT_LEVEL; 624 ft = mlx5_create_flow_table(roce->ns, &ft_attr); 625 if (IS_ERR(ft)) { 626 err = PTR_ERR(ft); 627 mlx5_core_err(mdev, "Fail to create RoCE IPsec tx ft err=%d\n", err); 628 goto free_in; 629 } 630 631 roce->ft = ft; 632 633 MLX5_SET_CFG(in, start_flow_index, ix); 634 ix += MLX5_TX_ROCE_GROUP_SIZE; 635 MLX5_SET_CFG(in, end_flow_index, ix - 1); 636 g = mlx5_create_flow_group(ft, in); 637 if (IS_ERR(g)) { 638 err = PTR_ERR(g); 639 mlx5_core_err(mdev, "Fail to create RoCE IPsec tx group err=%d\n", err); 640 goto destroy_table; 641 } 642 roce->g = g; 643 644 err = ipsec_fs_roce_tx_rule_setup(mdev, roce, pol_ft); 645 if (err) { 646 mlx5_core_err(mdev, "Fail to create RoCE IPsec tx rules err=%d\n", err); 647 goto destroy_group; 648 } 649 650 kvfree(in); 651 return 0; 652 653destroy_group: 654 mlx5_destroy_flow_group(roce->g); 655destroy_table: 656 mlx5_destroy_flow_table(ft); 657free_in: 658 kvfree(in); 659 return err; 660} 661 662struct mlx5_flow_table *mlx5_ipsec_fs_roce_ft_get(struct mlx5_ipsec_fs *ipsec_roce, u32 family) 663{ 664 struct mlx5_ipsec_rx_roce *rx_roce; 665 666 if (!ipsec_roce) 667 return NULL; 668 669 rx_roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx : 670 &ipsec_roce->ipv6_rx; 671 672 return rx_roce->ft; 673} 674 675void mlx5_ipsec_fs_roce_rx_destroy(struct mlx5_ipsec_fs *ipsec_roce, u32 family, 676 struct mlx5_core_dev *mdev) 677{ 678 bool is_mpv_slave = mlx5_core_is_mp_slave(mdev); 679 struct mlx5_ipsec_rx_roce *rx_roce; 680 681 if (!ipsec_roce) 682 return; 683 684 rx_roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx : 685 &ipsec_roce->ipv6_rx; 686 if (!rx_roce->ft) 687 return; /* Incase RoCE was cleaned from MPV event flow */ 688 689 if (is_mpv_slave) 690 mlx5_del_flow_rules(rx_roce->nic_master_rule); 691 mlx5_del_flow_rules(rx_roce->roce_miss.rule); 692 mlx5_del_flow_rules(rx_roce->rule); 693 if (is_mpv_slave) 694 roce_rx_mpv_destroy_tables(mdev, rx_roce); 695 mlx5_destroy_flow_table(rx_roce->ft_rdma); 696 mlx5_destroy_flow_group(rx_roce->roce_miss.group); 697 mlx5_destroy_flow_group(rx_roce->g); 698 mlx5_destroy_flow_table(rx_roce->ft); 699 rx_roce->ft = NULL; 700} 701 702int mlx5_ipsec_fs_roce_rx_create(struct mlx5_core_dev *mdev, 703 struct mlx5_ipsec_fs *ipsec_roce, 704 struct mlx5_flow_namespace *ns, 705 struct mlx5_flow_destination *default_dst, 706 u32 family, u32 level, u32 prio) 707{ 708 bool is_mpv_slave = mlx5_core_is_mp_slave(mdev); 709 struct mlx5_flow_table_attr ft_attr = {}; 710 struct mlx5_ipsec_rx_roce *roce; 711 struct mlx5_flow_table *ft; 712 struct mlx5_flow_group *g; 713 void *outer_headers_c; 714 int ix = 0; 715 u32 *in; 716 int err; 717 u8 *mc; 718 719 if (!ipsec_roce) 720 return 0; 721 722 roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx : 723 &ipsec_roce->ipv6_rx; 724 725 ft_attr.max_fte = 2; 726 ft_attr.level = level; 727 ft_attr.prio = prio; 728 ft = mlx5_create_flow_table(ns, &ft_attr); 729 if (IS_ERR(ft)) { 730 err = PTR_ERR(ft); 731 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at nic err=%d\n", err); 732 return err; 733 } 734 735 roce->ft = ft; 736 737 in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL); 738 if (!in) { 739 err = -ENOMEM; 740 goto fail_nomem; 741 } 742 743 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 744 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers); 745 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol); 746 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport); 747 748 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 749 MLX5_SET_CFG(in, start_flow_index, ix); 750 ix += MLX5_RX_ROCE_GROUP_SIZE; 751 MLX5_SET_CFG(in, end_flow_index, ix - 1); 752 g = mlx5_create_flow_group(ft, in); 753 if (IS_ERR(g)) { 754 err = PTR_ERR(g); 755 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx group at nic err=%d\n", err); 756 goto fail_group; 757 } 758 roce->g = g; 759 760 memset(in, 0, MLX5_ST_SZ_BYTES(create_flow_group_in)); 761 MLX5_SET_CFG(in, start_flow_index, ix); 762 ix += MLX5_RX_ROCE_GROUP_SIZE; 763 MLX5_SET_CFG(in, end_flow_index, ix - 1); 764 g = mlx5_create_flow_group(ft, in); 765 if (IS_ERR(g)) { 766 err = PTR_ERR(g); 767 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx miss group at nic err=%d\n", err); 768 goto fail_mgroup; 769 } 770 roce->roce_miss.group = g; 771 772 if (is_mpv_slave) { 773 err = ipsec_fs_roce_rx_mpv_create(mdev, ipsec_roce, ns, family, level, prio); 774 if (err) { 775 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx alias err=%d\n", err); 776 goto fail_mpv_create; 777 } 778 } else { 779 memset(&ft_attr, 0, sizeof(ft_attr)); 780 if (family == AF_INET) 781 ft_attr.level = 1; 782 ft_attr.max_fte = 1; 783 ft = mlx5_create_flow_table(roce->ns_rdma, &ft_attr); 784 if (IS_ERR(ft)) { 785 err = PTR_ERR(ft); 786 mlx5_core_err(mdev, 787 "Fail to create RoCE IPsec rx ft at rdma err=%d\n", err); 788 goto fail_rdma_table; 789 } 790 791 roce->ft_rdma = ft; 792 } 793 794 err = ipsec_fs_roce_rx_rule_setup(mdev, default_dst, roce); 795 if (err) { 796 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx rules err=%d\n", err); 797 goto fail_setup_rule; 798 } 799 800 kvfree(in); 801 return 0; 802 803fail_setup_rule: 804 if (is_mpv_slave) 805 roce_rx_mpv_destroy_tables(mdev, roce); 806 mlx5_destroy_flow_table(roce->ft_rdma); 807fail_mpv_create: 808fail_rdma_table: 809 mlx5_destroy_flow_group(roce->roce_miss.group); 810fail_mgroup: 811 mlx5_destroy_flow_group(roce->g); 812fail_group: 813 kvfree(in); 814fail_nomem: 815 mlx5_destroy_flow_table(roce->ft); 816 return err; 817} 818 819bool mlx5_ipsec_fs_is_mpv_roce_supported(struct mlx5_core_dev *mdev) 820{ 821 if (!mlx5_core_mp_enabled(mdev)) 822 return true; 823 824 if (ipsec_fs_create_alias_supported_one(mdev)) 825 return true; 826 827 return false; 828} 829 830void mlx5_ipsec_fs_roce_cleanup(struct mlx5_ipsec_fs *ipsec_roce) 831{ 832 kfree(ipsec_roce); 833} 834 835struct mlx5_ipsec_fs *mlx5_ipsec_fs_roce_init(struct mlx5_core_dev *mdev, 836 struct mlx5_devcom_comp_dev **devcom) 837{ 838 struct mlx5_ipsec_fs *roce_ipsec; 839 struct mlx5_flow_namespace *ns; 840 841 ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC); 842 if (!ns) { 843 mlx5_core_err(mdev, "Failed to get RoCE rx ns\n"); 844 return NULL; 845 } 846 847 roce_ipsec = kzalloc(sizeof(*roce_ipsec), GFP_KERNEL); 848 if (!roce_ipsec) 849 return NULL; 850 851 roce_ipsec->ipv4_rx.ns_rdma = ns; 852 roce_ipsec->ipv6_rx.ns_rdma = ns; 853 854 ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC); 855 if (!ns) { 856 mlx5_core_err(mdev, "Failed to get RoCE tx ns\n"); 857 goto err_tx; 858 } 859 860 roce_ipsec->tx.ns = ns; 861 862 roce_ipsec->devcom = devcom; 863 864 return roce_ipsec; 865 866err_tx: 867 kfree(roce_ipsec); 868 return NULL; 869} 870