1/*- 2 * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD: stable/11/sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c 353224 2019-10-07 09:24:12Z hselasky $ 26 */ 27 28#include <linux/module.h> 29#include <dev/mlx5/driver.h> 30#include "mlx5_core.h" 31#include "fs_core.h" 32#include <linux/string.h> 33#include <linux/compiler.h> 34 35#define INIT_TREE_NODE_ARRAY_SIZE(...) (sizeof((struct init_tree_node[]){__VA_ARGS__}) /\ 36 sizeof(struct init_tree_node)) 37 38#define ADD_PRIO(name_val, flags_val, min_level_val, max_ft_val, caps_val, \ 39 ...) {.type = FS_TYPE_PRIO,\ 40 .name = name_val,\ 41 .min_ft_level = min_level_val,\ 42 .flags = flags_val,\ 43 .max_ft = max_ft_val,\ 44 .caps = caps_val,\ 45 .children = (struct init_tree_node[]) {__VA_ARGS__},\ 46 .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \ 47} 48 49#define ADD_FT_PRIO(name_val, flags_val, max_ft_val, ...)\ 50 ADD_PRIO(name_val, flags_val, 0, max_ft_val, {},\ 51 __VA_ARGS__)\ 52 53#define ADD_NS(name_val, ...) {.type = FS_TYPE_NAMESPACE,\ 54 .name = name_val,\ 55 .children = (struct init_tree_node[]) {__VA_ARGS__},\ 56 .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \ 57} 58 59#define INIT_CAPS_ARRAY_SIZE(...) (sizeof((long[]){__VA_ARGS__}) /\ 60 sizeof(long)) 61 62#define FS_CAP(cap) (__mlx5_bit_off(flow_table_nic_cap, cap)) 63 64#define FS_REQUIRED_CAPS(...) {.arr_sz = INIT_CAPS_ARRAY_SIZE(__VA_ARGS__), \ 65 .caps = (long[]) {__VA_ARGS__}} 66 67#define BYPASS_MAX_FT 5 68#define BYPASS_PRIO_MAX_FT 1 69#define KERNEL_MAX_FT 3 70#define LEFTOVER_MAX_FT 1 71#define KENREL_MIN_LEVEL 3 72#define LEFTOVER_MIN_LEVEL KENREL_MIN_LEVEL + 1 73#define BYPASS_MIN_LEVEL MLX5_NUM_BYPASS_FTS + LEFTOVER_MIN_LEVEL 74struct node_caps { 75 size_t arr_sz; 76 long *caps; 77}; 78 79struct init_tree_node { 80 enum fs_type type; 81 const char *name; 82 struct init_tree_node *children; 83 int ar_size; 84 struct node_caps caps; 85 u8 flags; 86 int min_ft_level; 87 int prio; 88 int max_ft; 89} root_fs = { 90 .type = FS_TYPE_NAMESPACE, 91 .name = "root", 92 .ar_size = 3, 93 .children = (struct init_tree_node[]) { 94 ADD_PRIO("by_pass_prio", 0, BYPASS_MIN_LEVEL, 0, 95 FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), 96 FS_CAP(flow_table_properties_nic_receive.modify_root)), 97 ADD_NS("by_pass_ns", 98 ADD_FT_PRIO("prio0", 0, 99 BYPASS_PRIO_MAX_FT), 100 ADD_FT_PRIO("prio1", 0, 101 BYPASS_PRIO_MAX_FT), 102 ADD_FT_PRIO("prio2", 0, 103 BYPASS_PRIO_MAX_FT), 104 ADD_FT_PRIO("prio3", 0, 105 BYPASS_PRIO_MAX_FT), 106 ADD_FT_PRIO("prio4", 0, 107 BYPASS_PRIO_MAX_FT), 108 ADD_FT_PRIO("prio5", 0, 109 BYPASS_PRIO_MAX_FT), 110 ADD_FT_PRIO("prio6", 0, 111 BYPASS_PRIO_MAX_FT), 112 ADD_FT_PRIO("prio7", 0, 113 BYPASS_PRIO_MAX_FT), 114 ADD_FT_PRIO("prio-mcast", 0, 115 BYPASS_PRIO_MAX_FT))), 116 ADD_PRIO("kernel_prio", 0, KENREL_MIN_LEVEL, 0, {}, 117 ADD_NS("kernel_ns", 118 ADD_FT_PRIO("prio_kernel-0", 0, 119 KERNEL_MAX_FT))), 120 ADD_PRIO("leftovers_prio", MLX5_CORE_FS_PRIO_SHARED, 121 LEFTOVER_MIN_LEVEL, 0, 122 FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), 123 FS_CAP(flow_table_properties_nic_receive.modify_root)), 124 ADD_NS("leftover_ns", 125 ADD_FT_PRIO("leftovers_prio-0", 126 MLX5_CORE_FS_PRIO_SHARED, 127 LEFTOVER_MAX_FT))) 128 } 129}; 130 131/* Tree creation functions */ 132 133static struct mlx5_flow_root_namespace *find_root(struct fs_base *node) 134{ 135 struct fs_base *parent; 136 137 /* Make sure we only read it once while we go up the tree */ 138 while ((parent = node->parent)) 139 node = parent; 140 141 if (node->type != FS_TYPE_NAMESPACE) { 142 return NULL; 143 } 144 145 return container_of(container_of(node, 146 struct mlx5_flow_namespace, 147 base), 148 struct mlx5_flow_root_namespace, 149 ns); 150} 151 152static inline struct mlx5_core_dev *fs_get_dev(struct fs_base *node) 153{ 154 struct mlx5_flow_root_namespace *root = find_root(node); 155 156 if (root) 157 return root->dev; 158 return NULL; 159} 160 161static void fs_init_node(struct fs_base *node, 162 unsigned int refcount) 163{ 164 kref_init(&node->refcount); 165 atomic_set(&node->users_refcount, refcount); 166 init_completion(&node->complete); 167 INIT_LIST_HEAD(&node->list); 168 mutex_init(&node->lock); 169} 170 171static void _fs_add_node(struct fs_base *node, 172 const char *name, 173 struct fs_base *parent) 174{ 175 if (parent) 176 atomic_inc(&parent->users_refcount); 177 node->name = kstrdup_const(name, GFP_KERNEL); 178 node->parent = parent; 179} 180 181static void fs_add_node(struct fs_base *node, 182 struct fs_base *parent, const char *name, 183 unsigned int refcount) 184{ 185 fs_init_node(node, refcount); 186 _fs_add_node(node, name, parent); 187} 188 189static void _fs_put(struct fs_base *node, void (*kref_cb)(struct kref *kref), 190 bool parent_locked); 191 192static void fs_del_dst(struct mlx5_flow_rule *dst); 193static void _fs_del_ft(struct mlx5_flow_table *ft); 194static void fs_del_fg(struct mlx5_flow_group *fg); 195static void fs_del_fte(struct fs_fte *fte); 196 197static void cmd_remove_node(struct fs_base *base) 198{ 199 switch (base->type) { 200 case FS_TYPE_FLOW_DEST: 201 fs_del_dst(container_of(base, struct mlx5_flow_rule, base)); 202 break; 203 case FS_TYPE_FLOW_TABLE: 204 _fs_del_ft(container_of(base, struct mlx5_flow_table, base)); 205 break; 206 case FS_TYPE_FLOW_GROUP: 207 fs_del_fg(container_of(base, struct mlx5_flow_group, base)); 208 break; 209 case FS_TYPE_FLOW_ENTRY: 210 fs_del_fte(container_of(base, struct fs_fte, base)); 211 break; 212 default: 213 break; 214 } 215} 216 217static void __fs_remove_node(struct kref *kref) 218{ 219 struct fs_base *node = container_of(kref, struct fs_base, refcount); 220 221 if (node->parent) 222 mutex_lock(&node->parent->lock); 223 mutex_lock(&node->lock); 224 cmd_remove_node(node); 225 mutex_unlock(&node->lock); 226 complete(&node->complete); 227 if (node->parent) { 228 mutex_unlock(&node->parent->lock); 229 _fs_put(node->parent, _fs_remove_node, false); 230 } 231} 232 233void _fs_remove_node(struct kref *kref) 234{ 235 struct fs_base *node = container_of(kref, struct fs_base, refcount); 236 237 __fs_remove_node(kref); 238 kfree_const(node->name); 239 kfree(node); 240} 241 242static void fs_get(struct fs_base *node) 243{ 244 atomic_inc(&node->users_refcount); 245} 246 247static void _fs_put(struct fs_base *node, void (*kref_cb)(struct kref *kref), 248 bool parent_locked) 249{ 250 struct fs_base *parent_node = node->parent; 251 252 if (parent_node && !parent_locked) 253 mutex_lock(&parent_node->lock); 254 if (atomic_dec_and_test(&node->users_refcount)) { 255 if (parent_node) { 256 /*remove from parent's list*/ 257 list_del_init(&node->list); 258 mutex_unlock(&parent_node->lock); 259 } 260 kref_put(&node->refcount, kref_cb); 261 if (parent_node && parent_locked) 262 mutex_lock(&parent_node->lock); 263 } else if (parent_node && !parent_locked) { 264 mutex_unlock(&parent_node->lock); 265 } 266} 267 268static void fs_put(struct fs_base *node) 269{ 270 _fs_put(node, __fs_remove_node, false); 271} 272 273static void fs_put_parent_locked(struct fs_base *node) 274{ 275 _fs_put(node, __fs_remove_node, true); 276} 277 278static void fs_remove_node(struct fs_base *node) 279{ 280 fs_put(node); 281 wait_for_completion(&node->complete); 282 kfree_const(node->name); 283 kfree(node); 284} 285 286static void fs_remove_node_parent_locked(struct fs_base *node) 287{ 288 fs_put_parent_locked(node); 289 wait_for_completion(&node->complete); 290 kfree_const(node->name); 291 kfree(node); 292} 293 294static struct fs_fte *fs_alloc_fte(u8 action, 295 u32 flow_tag, 296 u32 *match_value, 297 unsigned int index) 298{ 299 struct fs_fte *fte; 300 301 302 fte = kzalloc(sizeof(*fte), GFP_KERNEL); 303 if (!fte) 304 return ERR_PTR(-ENOMEM); 305 306 memcpy(fte->val, match_value, sizeof(fte->val)); 307 fte->base.type = FS_TYPE_FLOW_ENTRY; 308 fte->dests_size = 0; 309 fte->flow_tag = flow_tag; 310 fte->index = index; 311 INIT_LIST_HEAD(&fte->dests); 312 fte->action = action; 313 314 return fte; 315} 316 317static struct fs_fte *alloc_star_ft_entry(struct mlx5_flow_table *ft, 318 struct mlx5_flow_group *fg, 319 u32 *match_value, 320 unsigned int index) 321{ 322 int err; 323 struct fs_fte *fte; 324 struct mlx5_flow_rule *dst; 325 326 if (fg->num_ftes == fg->max_ftes) 327 return ERR_PTR(-ENOSPC); 328 329 fte = fs_alloc_fte(MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, 330 MLX5_FS_DEFAULT_FLOW_TAG, match_value, index); 331 if (IS_ERR(fte)) 332 return fte; 333 334 /*create dst*/ 335 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 336 if (!dst) { 337 err = -ENOMEM; 338 goto free_fte; 339 } 340 341 fte->base.parent = &fg->base; 342 fte->dests_size = 1; 343 dst->dest_attr.type = MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE; 344 dst->base.parent = &fte->base; 345 list_add(&dst->base.list, &fte->dests); 346 /* assumed that the callee creates the star rules sorted by index */ 347 list_add_tail(&fte->base.list, &fg->ftes); 348 fg->num_ftes++; 349 350 return fte; 351 352free_fte: 353 kfree(fte); 354 return ERR_PTR(err); 355} 356 357/* assume that fte can't be changed */ 358static void free_star_fte_entry(struct fs_fte *fte) 359{ 360 struct mlx5_flow_group *fg; 361 struct mlx5_flow_rule *dst, *temp; 362 363 fs_get_parent(fg, fte); 364 365 list_for_each_entry_safe(dst, temp, &fte->dests, base.list) { 366 fte->dests_size--; 367 list_del(&dst->base.list); 368 kfree(dst); 369 } 370 371 list_del(&fte->base.list); 372 fg->num_ftes--; 373 kfree(fte); 374} 375 376static struct mlx5_flow_group *fs_alloc_fg(u32 *create_fg_in) 377{ 378 struct mlx5_flow_group *fg; 379 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in, 380 create_fg_in, match_criteria); 381 u8 match_criteria_enable = MLX5_GET(create_flow_group_in, 382 create_fg_in, 383 match_criteria_enable); 384 fg = kzalloc(sizeof(*fg), GFP_KERNEL); 385 if (!fg) 386 return ERR_PTR(-ENOMEM); 387 388 INIT_LIST_HEAD(&fg->ftes); 389 fg->mask.match_criteria_enable = match_criteria_enable; 390 memcpy(&fg->mask.match_criteria, match_criteria, 391 sizeof(fg->mask.match_criteria)); 392 fg->base.type = FS_TYPE_FLOW_GROUP; 393 fg->start_index = MLX5_GET(create_flow_group_in, create_fg_in, 394 start_flow_index); 395 fg->max_ftes = MLX5_GET(create_flow_group_in, create_fg_in, 396 end_flow_index) - fg->start_index + 1; 397 return fg; 398} 399 400static struct mlx5_flow_table *find_next_ft(struct fs_prio *prio); 401static struct mlx5_flow_table *find_prev_ft(struct mlx5_flow_table *curr, 402 struct fs_prio *prio); 403 404/* assumed src_ft and dst_ft can't be freed */ 405static int fs_set_star_rule(struct mlx5_core_dev *dev, 406 struct mlx5_flow_table *src_ft, 407 struct mlx5_flow_table *dst_ft) 408{ 409 struct mlx5_flow_rule *src_dst; 410 struct fs_fte *src_fte; 411 int err = 0; 412 u32 *match_value; 413 int match_len = MLX5_ST_SZ_BYTES(fte_match_param); 414 415 src_dst = list_first_entry(&src_ft->star_rule.fte->dests, 416 struct mlx5_flow_rule, base.list); 417 match_value = mlx5_vzalloc(match_len); 418 if (!match_value) { 419 mlx5_core_warn(dev, "failed to allocate inbox\n"); 420 return -ENOMEM; 421 } 422 /*Create match context*/ 423 424 fs_get_parent(src_fte, src_dst); 425 426 src_dst->dest_attr.ft = dst_ft; 427 if (dst_ft) { 428 err = mlx5_cmd_fs_set_fte(dev, 429 src_ft->vport, 430 &src_fte->status, 431 match_value, src_ft->type, 432 src_ft->id, src_fte->index, 433 src_ft->star_rule.fg->id, 434 src_fte->flow_tag, 435 src_fte->action, 436 src_fte->dests_size, 437 &src_fte->dests); 438 if (err) 439 goto free; 440 441 fs_get(&dst_ft->base); 442 } else { 443 mlx5_cmd_fs_delete_fte(dev, 444 src_ft->vport, 445 &src_fte->status, 446 src_ft->type, src_ft->id, 447 src_fte->index); 448 } 449 450free: 451 kvfree(match_value); 452 return err; 453} 454 455static int connect_prev_fts(struct fs_prio *locked_prio, 456 struct fs_prio *prev_prio, 457 struct mlx5_flow_table *next_ft) 458{ 459 struct mlx5_flow_table *iter; 460 int err = 0; 461 struct mlx5_core_dev *dev = fs_get_dev(&prev_prio->base); 462 463 if (!dev) 464 return -ENODEV; 465 466 mutex_lock(&prev_prio->base.lock); 467 fs_for_each_ft(iter, prev_prio) { 468 struct mlx5_flow_rule *src_dst = 469 list_first_entry(&iter->star_rule.fte->dests, 470 struct mlx5_flow_rule, base.list); 471 struct mlx5_flow_table *prev_ft = src_dst->dest_attr.ft; 472 473 if (prev_ft == next_ft) 474 continue; 475 476 err = fs_set_star_rule(dev, iter, next_ft); 477 if (err) { 478 mlx5_core_warn(dev, 479 "mlx5: flow steering can't connect prev and next\n"); 480 goto unlock; 481 } else { 482 /* Assume ft's prio is locked */ 483 if (prev_ft) { 484 struct fs_prio *prio; 485 486 fs_get_parent(prio, prev_ft); 487 if (prio == locked_prio) 488 fs_put_parent_locked(&prev_ft->base); 489 else 490 fs_put(&prev_ft->base); 491 } 492 } 493 } 494 495unlock: 496 mutex_unlock(&prev_prio->base.lock); 497 return 0; 498} 499 500static int create_star_rule(struct mlx5_flow_table *ft, struct fs_prio *prio) 501{ 502 struct mlx5_flow_group *fg; 503 int err; 504 u32 *fg_in; 505 u32 *match_value; 506 struct mlx5_flow_table *next_ft; 507 struct mlx5_flow_table *prev_ft; 508 struct mlx5_flow_root_namespace *root = find_root(&prio->base); 509 int fg_inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 510 int match_len = MLX5_ST_SZ_BYTES(fte_match_param); 511 512 fg_in = mlx5_vzalloc(fg_inlen); 513 if (!fg_in) { 514 mlx5_core_warn(root->dev, "failed to allocate inbox\n"); 515 return -ENOMEM; 516 } 517 518 match_value = mlx5_vzalloc(match_len); 519 if (!match_value) { 520 mlx5_core_warn(root->dev, "failed to allocate inbox\n"); 521 kvfree(fg_in); 522 return -ENOMEM; 523 } 524 525 MLX5_SET(create_flow_group_in, fg_in, start_flow_index, ft->max_fte); 526 MLX5_SET(create_flow_group_in, fg_in, end_flow_index, ft->max_fte); 527 fg = fs_alloc_fg(fg_in); 528 if (IS_ERR(fg)) { 529 err = PTR_ERR(fg); 530 goto out; 531 } 532 ft->star_rule.fg = fg; 533 err = mlx5_cmd_fs_create_fg(fs_get_dev(&prio->base), 534 fg_in, ft->vport, ft->type, 535 ft->id, 536 &fg->id); 537 if (err) 538 goto free_fg; 539 540 ft->star_rule.fte = alloc_star_ft_entry(ft, fg, 541 match_value, 542 ft->max_fte); 543 if (IS_ERR(ft->star_rule.fte)) 544 goto free_star_rule; 545 546 mutex_lock(&root->fs_chain_lock); 547 next_ft = find_next_ft(prio); 548 err = fs_set_star_rule(root->dev, ft, next_ft); 549 if (err) { 550 mutex_unlock(&root->fs_chain_lock); 551 goto free_star_rule; 552 } 553 if (next_ft) { 554 struct fs_prio *parent; 555 556 fs_get_parent(parent, next_ft); 557 fs_put(&next_ft->base); 558 } 559 prev_ft = find_prev_ft(ft, prio); 560 if (prev_ft) { 561 struct fs_prio *prev_parent; 562 563 fs_get_parent(prev_parent, prev_ft); 564 565 err = connect_prev_fts(NULL, prev_parent, ft); 566 if (err) { 567 mutex_unlock(&root->fs_chain_lock); 568 goto destroy_chained_star_rule; 569 } 570 fs_put(&prev_ft->base); 571 } 572 mutex_unlock(&root->fs_chain_lock); 573 kvfree(fg_in); 574 kvfree(match_value); 575 576 return 0; 577 578destroy_chained_star_rule: 579 fs_set_star_rule(fs_get_dev(&prio->base), ft, NULL); 580 if (next_ft) 581 fs_put(&next_ft->base); 582free_star_rule: 583 free_star_fte_entry(ft->star_rule.fte); 584 mlx5_cmd_fs_destroy_fg(fs_get_dev(&ft->base), ft->vport, 585 ft->type, ft->id, 586 fg->id); 587free_fg: 588 kfree(fg); 589out: 590 kvfree(fg_in); 591 kvfree(match_value); 592 return err; 593} 594 595static void destroy_star_rule(struct mlx5_flow_table *ft, struct fs_prio *prio) 596{ 597 int err; 598 struct mlx5_flow_root_namespace *root; 599 struct mlx5_core_dev *dev = fs_get_dev(&prio->base); 600 struct mlx5_flow_table *prev_ft, *next_ft; 601 struct fs_prio *prev_prio; 602 603 WARN_ON(!dev); 604 605 root = find_root(&prio->base); 606 if (!root) 607 mlx5_core_err(dev, 608 "flow steering failed to find root of priority %s", 609 prio->base.name); 610 611 /* In order to ensure atomic deletion, first update 612 * prev ft to point on the next ft. 613 */ 614 mutex_lock(&root->fs_chain_lock); 615 prev_ft = find_prev_ft(ft, prio); 616 next_ft = find_next_ft(prio); 617 if (prev_ft) { 618 fs_get_parent(prev_prio, prev_ft); 619 /*Prev is connected to ft, only if ft is the first(last) in the prio*/ 620 err = connect_prev_fts(prio, prev_prio, next_ft); 621 if (err) 622 mlx5_core_warn(root->dev, 623 "flow steering can't connect prev and next of flow table\n"); 624 fs_put(&prev_ft->base); 625 } 626 627 err = fs_set_star_rule(root->dev, ft, NULL); 628 /*One put is for fs_get in find next ft*/ 629 if (next_ft) { 630 fs_put(&next_ft->base); 631 if (!err) 632 fs_put(&next_ft->base); 633 } 634 635 mutex_unlock(&root->fs_chain_lock); 636 err = mlx5_cmd_fs_destroy_fg(dev, ft->vport, ft->type, ft->id, 637 ft->star_rule.fg->id); 638 if (err) 639 mlx5_core_warn(dev, 640 "flow steering can't destroy star entry group(index:%d) of ft:%s\n", ft->star_rule.fg->start_index, 641 ft->base.name); 642 free_star_fte_entry(ft->star_rule.fte); 643 644 kfree(ft->star_rule.fg); 645 ft->star_rule.fg = NULL; 646} 647 648static struct fs_prio *find_prio(struct mlx5_flow_namespace *ns, 649 unsigned int prio) 650{ 651 struct fs_prio *iter_prio; 652 653 fs_for_each_prio(iter_prio, ns) { 654 if (iter_prio->prio == prio) 655 return iter_prio; 656 } 657 658 return NULL; 659} 660 661static unsigned int _alloc_new_level(struct fs_prio *prio, 662 struct mlx5_flow_namespace *match); 663 664static unsigned int __alloc_new_level(struct mlx5_flow_namespace *ns, 665 struct fs_prio *prio) 666{ 667 unsigned int level = 0; 668 struct fs_prio *p; 669 670 if (!ns) 671 return 0; 672 673 mutex_lock(&ns->base.lock); 674 fs_for_each_prio(p, ns) { 675 if (p != prio) 676 level += p->max_ft; 677 else 678 break; 679 } 680 mutex_unlock(&ns->base.lock); 681 682 fs_get_parent(prio, ns); 683 if (prio) 684 WARN_ON(prio->base.type != FS_TYPE_PRIO); 685 686 return level + _alloc_new_level(prio, ns); 687} 688 689/* Called under lock of priority, hence locking all upper objects */ 690static unsigned int _alloc_new_level(struct fs_prio *prio, 691 struct mlx5_flow_namespace *match) 692{ 693 struct mlx5_flow_namespace *ns; 694 struct fs_base *it; 695 unsigned int level = 0; 696 697 if (!prio) 698 return 0; 699 700 mutex_lock(&prio->base.lock); 701 fs_for_each_ns_or_ft_reverse(it, prio) { 702 if (it->type == FS_TYPE_NAMESPACE) { 703 struct fs_prio *p; 704 705 fs_get_obj(ns, it); 706 707 if (match != ns) { 708 mutex_lock(&ns->base.lock); 709 fs_for_each_prio(p, ns) 710 level += p->max_ft; 711 mutex_unlock(&ns->base.lock); 712 } else { 713 break; 714 } 715 } else { 716 struct mlx5_flow_table *ft; 717 718 fs_get_obj(ft, it); 719 mutex_unlock(&prio->base.lock); 720 return level + ft->level + 1; 721 } 722 } 723 724 fs_get_parent(ns, prio); 725 mutex_unlock(&prio->base.lock); 726 return __alloc_new_level(ns, prio) + level; 727} 728 729static unsigned int alloc_new_level(struct fs_prio *prio) 730{ 731 return _alloc_new_level(prio, NULL); 732} 733 734static int update_root_ft_create(struct mlx5_flow_root_namespace *root, 735 struct mlx5_flow_table *ft) 736{ 737 int err = 0; 738 int min_level = INT_MAX; 739 740 if (root->root_ft) 741 min_level = root->root_ft->level; 742 743 if (ft->level < min_level) 744 err = mlx5_cmd_update_root_ft(root->dev, ft->type, 745 ft->id); 746 else 747 return err; 748 749 if (err) 750 mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n", 751 ft->id); 752 else 753 root->root_ft = ft; 754 755 return err; 756} 757 758static struct mlx5_flow_table *_create_ft_common(struct mlx5_flow_namespace *ns, 759 u16 vport, 760 struct fs_prio *fs_prio, 761 int max_fte, 762 const char *name) 763{ 764 struct mlx5_flow_table *ft; 765 int err; 766 int log_table_sz; 767 int ft_size; 768 char gen_name[20]; 769 struct mlx5_flow_root_namespace *root = find_root(&ns->base); 770 struct mlx5_core_dev *dev = fs_get_dev(&ns->base); 771 772 if (!root) { 773 mlx5_core_err(dev, 774 "flow steering failed to find root of namespace %s", 775 ns->base.name); 776 return ERR_PTR(-ENODEV); 777 } 778 779 if (fs_prio->num_ft == fs_prio->max_ft) 780 return ERR_PTR(-ENOSPC); 781 782 ft = kzalloc(sizeof(*ft), GFP_KERNEL); 783 if (!ft) 784 return ERR_PTR(-ENOMEM); 785 786 fs_init_node(&ft->base, 1); 787 INIT_LIST_HEAD(&ft->fgs); 788 789 /* Temporarily WA until we expose the level set in the API */ 790 if (root->table_type == FS_FT_ESW_EGRESS_ACL || 791 root->table_type == FS_FT_ESW_INGRESS_ACL) 792 ft->level = 0; 793 else 794 ft->level = alloc_new_level(fs_prio); 795 796 ft->base.type = FS_TYPE_FLOW_TABLE; 797 ft->vport = vport; 798 ft->type = root->table_type; 799 /*Two entries are reserved for star rules*/ 800 ft_size = roundup_pow_of_two(max_fte + 2); 801 /*User isn't aware to those rules*/ 802 ft->max_fte = ft_size - 2; 803 log_table_sz = ilog2(ft_size); 804 err = mlx5_cmd_fs_create_ft(root->dev, ft->vport, ft->type, 805 ft->level, log_table_sz, &ft->id); 806 if (err) 807 goto free_ft; 808 809 err = create_star_rule(ft, fs_prio); 810 if (err) 811 goto del_ft; 812 813 if ((root->table_type == FS_FT_NIC_RX) && MLX5_CAP_FLOWTABLE(root->dev, 814 flow_table_properties_nic_receive.modify_root)) { 815 err = update_root_ft_create(root, ft); 816 if (err) 817 goto destroy_star_rule; 818 } 819 820 if (!name || !strlen(name)) { 821 snprintf(gen_name, 20, "flow_table_%u", ft->id); 822 _fs_add_node(&ft->base, gen_name, &fs_prio->base); 823 } else { 824 _fs_add_node(&ft->base, name, &fs_prio->base); 825 } 826 list_add_tail(&ft->base.list, &fs_prio->objs); 827 fs_prio->num_ft++; 828 829 return ft; 830 831destroy_star_rule: 832 destroy_star_rule(ft, fs_prio); 833del_ft: 834 mlx5_cmd_fs_destroy_ft(root->dev, ft->vport, ft->type, ft->id); 835free_ft: 836 kfree(ft); 837 return ERR_PTR(err); 838} 839 840static struct mlx5_flow_table *create_ft_common(struct mlx5_flow_namespace *ns, 841 u16 vport, 842 unsigned int prio, 843 int max_fte, 844 const char *name) 845{ 846 struct fs_prio *fs_prio = NULL; 847 fs_prio = find_prio(ns, prio); 848 if (!fs_prio) 849 return ERR_PTR(-EINVAL); 850 851 return _create_ft_common(ns, vport, fs_prio, max_fte, name); 852} 853 854 855static struct mlx5_flow_table *find_first_ft_in_ns(struct mlx5_flow_namespace *ns, 856 struct list_head *start); 857 858static struct mlx5_flow_table *find_first_ft_in_prio(struct fs_prio *prio, 859 struct list_head *start); 860 861static struct mlx5_flow_table *mlx5_create_autogrouped_shared_flow_table(struct fs_prio *fs_prio) 862{ 863 struct mlx5_flow_table *ft; 864 865 ft = find_first_ft_in_prio(fs_prio, &fs_prio->objs); 866 if (ft) { 867 ft->shared_refcount++; 868 return ft; 869 } 870 871 return NULL; 872} 873 874struct mlx5_flow_table *mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns, 875 int prio, 876 const char *name, 877 int num_flow_table_entries, 878 int max_num_groups) 879{ 880 struct mlx5_flow_table *ft = NULL; 881 struct fs_prio *fs_prio; 882 bool is_shared_prio; 883 884 fs_prio = find_prio(ns, prio); 885 if (!fs_prio) 886 return ERR_PTR(-EINVAL); 887 888 is_shared_prio = fs_prio->flags & MLX5_CORE_FS_PRIO_SHARED; 889 if (is_shared_prio) { 890 mutex_lock(&fs_prio->shared_lock); 891 ft = mlx5_create_autogrouped_shared_flow_table(fs_prio); 892 } 893 894 if (ft) 895 goto return_ft; 896 897 ft = create_ft_common(ns, 0, prio, num_flow_table_entries, 898 name); 899 if (IS_ERR(ft)) 900 goto return_ft; 901 902 ft->autogroup.active = true; 903 ft->autogroup.max_types = max_num_groups; 904 if (is_shared_prio) 905 ft->shared_refcount = 1; 906 907return_ft: 908 if (is_shared_prio) 909 mutex_unlock(&fs_prio->shared_lock); 910 return ft; 911} 912EXPORT_SYMBOL(mlx5_create_auto_grouped_flow_table); 913 914struct mlx5_flow_table *mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns, 915 u16 vport, 916 int prio, 917 const char *name, 918 int num_flow_table_entries) 919{ 920 return create_ft_common(ns, vport, prio, num_flow_table_entries, name); 921} 922EXPORT_SYMBOL(mlx5_create_vport_flow_table); 923 924struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns, 925 int prio, 926 const char *name, 927 int num_flow_table_entries) 928{ 929 return create_ft_common(ns, 0, prio, num_flow_table_entries, name); 930} 931EXPORT_SYMBOL(mlx5_create_flow_table); 932 933static void _fs_del_ft(struct mlx5_flow_table *ft) 934{ 935 int err; 936 struct mlx5_core_dev *dev = fs_get_dev(&ft->base); 937 struct fs_prio *prio; 938 939 err = mlx5_cmd_fs_destroy_ft(dev, ft->vport, ft->type, ft->id); 940 if (err) 941 mlx5_core_warn(dev, "flow steering can't destroy ft %s\n", 942 ft->base.name); 943 944 fs_get_parent(prio, ft); 945 prio->num_ft--; 946} 947 948static int update_root_ft_destroy(struct mlx5_flow_root_namespace *root, 949 struct mlx5_flow_table *ft) 950{ 951 int err = 0; 952 struct fs_prio *prio; 953 struct mlx5_flow_table *next_ft = NULL; 954 struct mlx5_flow_table *put_ft = NULL; 955 956 if (root->root_ft != ft) 957 return 0; 958 959 fs_get_parent(prio, ft); 960 /*Assuming objs containis only flow tables and 961 * flow tables are sorted by level. 962 */ 963 if (!list_is_last(&ft->base.list, &prio->objs)) { 964 next_ft = list_next_entry(ft, base.list); 965 } else { 966 next_ft = find_next_ft(prio); 967 put_ft = next_ft; 968 } 969 970 if (next_ft) { 971 err = mlx5_cmd_update_root_ft(root->dev, next_ft->type, 972 next_ft->id); 973 if (err) 974 mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n", 975 ft->id); 976 } 977 if (!err) 978 root->root_ft = next_ft; 979 980 if (put_ft) 981 fs_put(&put_ft->base); 982 983 return err; 984} 985 986/*Objects in the same prio are destroyed in the reverse order they were createrd*/ 987int mlx5_destroy_flow_table(struct mlx5_flow_table *ft) 988{ 989 int err = 0; 990 struct fs_prio *prio; 991 struct mlx5_flow_root_namespace *root; 992 bool is_shared_prio; 993 struct mlx5_core_dev *dev; 994 995 fs_get_parent(prio, ft); 996 root = find_root(&prio->base); 997 dev = fs_get_dev(&prio->base); 998 999 if (!root) { 1000 mlx5_core_err(dev, 1001 "flow steering failed to find root of priority %s", 1002 prio->base.name); 1003 return -ENODEV; 1004 } 1005 1006 is_shared_prio = prio->flags & MLX5_CORE_FS_PRIO_SHARED; 1007 if (is_shared_prio) { 1008 mutex_lock(&prio->shared_lock); 1009 if (ft->shared_refcount > 1) { 1010 --ft->shared_refcount; 1011 fs_put(&ft->base); 1012 mutex_unlock(&prio->shared_lock); 1013 return 0; 1014 } 1015 } 1016 1017 mutex_lock(&prio->base.lock); 1018 mutex_lock(&ft->base.lock); 1019 1020 err = update_root_ft_destroy(root, ft); 1021 if (err) 1022 goto unlock_ft; 1023 1024 /* delete two last entries */ 1025 destroy_star_rule(ft, prio); 1026 1027 mutex_unlock(&ft->base.lock); 1028 fs_remove_node_parent_locked(&ft->base); 1029 mutex_unlock(&prio->base.lock); 1030 if (is_shared_prio) 1031 mutex_unlock(&prio->shared_lock); 1032 1033 return err; 1034 1035unlock_ft: 1036 mutex_unlock(&ft->base.lock); 1037 mutex_unlock(&prio->base.lock); 1038 if (is_shared_prio) 1039 mutex_unlock(&prio->shared_lock); 1040 1041 return err; 1042} 1043EXPORT_SYMBOL(mlx5_destroy_flow_table); 1044 1045static struct mlx5_flow_group *fs_create_fg(struct mlx5_core_dev *dev, 1046 struct mlx5_flow_table *ft, 1047 struct list_head *prev, 1048 u32 *fg_in, 1049 int refcount) 1050{ 1051 struct mlx5_flow_group *fg; 1052 int err; 1053 unsigned int end_index; 1054 char name[20]; 1055 1056 fg = fs_alloc_fg(fg_in); 1057 if (IS_ERR(fg)) 1058 return fg; 1059 1060 end_index = fg->start_index + fg->max_ftes - 1; 1061 err = mlx5_cmd_fs_create_fg(dev, fg_in, 1062 ft->vport, ft->type, ft->id, 1063 &fg->id); 1064 if (err) 1065 goto free_fg; 1066 1067 mutex_lock(&ft->base.lock); 1068 if (ft->autogroup.active) 1069 ft->autogroup.num_types++; 1070 1071 snprintf(name, sizeof(name), "group_%u", fg->id); 1072 /*Add node to tree*/ 1073 fs_add_node(&fg->base, &ft->base, name, refcount); 1074 /*Add node to group list*/ 1075 list_add(&fg->base.list, prev); 1076 mutex_unlock(&ft->base.lock); 1077 1078 return fg; 1079 1080free_fg: 1081 kfree(fg); 1082 return ERR_PTR(err); 1083} 1084 1085struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft, 1086 u32 *in) 1087{ 1088 struct mlx5_flow_group *fg; 1089 struct mlx5_core_dev *dev = fs_get_dev(&ft->base); 1090 1091 if (!dev) 1092 return ERR_PTR(-ENODEV); 1093 1094 if (ft->autogroup.active) 1095 return ERR_PTR(-EPERM); 1096 1097 fg = fs_create_fg(dev, ft, ft->fgs.prev, in, 1); 1098 1099 return fg; 1100} 1101EXPORT_SYMBOL(mlx5_create_flow_group); 1102 1103/*Group is destoyed when all the rules in the group were removed*/ 1104static void fs_del_fg(struct mlx5_flow_group *fg) 1105{ 1106 struct mlx5_flow_table *parent_ft; 1107 struct mlx5_core_dev *dev; 1108 1109 fs_get_parent(parent_ft, fg); 1110 dev = fs_get_dev(&parent_ft->base); 1111 WARN_ON(!dev); 1112 1113 if (parent_ft->autogroup.active) 1114 parent_ft->autogroup.num_types--; 1115 1116 if (mlx5_cmd_fs_destroy_fg(dev, parent_ft->vport, 1117 parent_ft->type, 1118 parent_ft->id, fg->id)) 1119 mlx5_core_warn(dev, "flow steering can't destroy fg\n"); 1120} 1121 1122void mlx5_destroy_flow_group(struct mlx5_flow_group *fg) 1123{ 1124 fs_remove_node(&fg->base); 1125} 1126EXPORT_SYMBOL(mlx5_destroy_flow_group); 1127 1128static bool _fs_match_exact_val(void *mask, void *val1, void *val2, size_t size) 1129{ 1130 unsigned int i; 1131 1132 /* TODO: optimize by comparing 64bits when possible */ 1133 for (i = 0; i < size; i++, mask++, val1++, val2++) 1134 if ((*((u8 *)val1) & (*(u8 *)mask)) != 1135 ((*(u8 *)val2) & (*(u8 *)mask))) 1136 return false; 1137 1138 return true; 1139} 1140 1141bool fs_match_exact_val(struct mlx5_core_fs_mask *mask, 1142 void *val1, void *val2) 1143{ 1144 if (mask->match_criteria_enable & 1145 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS) { 1146 void *fte_match1 = MLX5_ADDR_OF(fte_match_param, 1147 val1, outer_headers); 1148 void *fte_match2 = MLX5_ADDR_OF(fte_match_param, 1149 val2, outer_headers); 1150 void *fte_mask = MLX5_ADDR_OF(fte_match_param, 1151 mask->match_criteria, outer_headers); 1152 1153 if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2, 1154 MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4))) 1155 return false; 1156 } 1157 1158 if (mask->match_criteria_enable & 1159 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS) { 1160 void *fte_match1 = MLX5_ADDR_OF(fte_match_param, 1161 val1, misc_parameters); 1162 void *fte_match2 = MLX5_ADDR_OF(fte_match_param, 1163 val2, misc_parameters); 1164 void *fte_mask = MLX5_ADDR_OF(fte_match_param, 1165 mask->match_criteria, misc_parameters); 1166 1167 if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2, 1168 MLX5_ST_SZ_BYTES(fte_match_set_misc))) 1169 return false; 1170 } 1171 if (mask->match_criteria_enable & 1172 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS) { 1173 void *fte_match1 = MLX5_ADDR_OF(fte_match_param, 1174 val1, inner_headers); 1175 void *fte_match2 = MLX5_ADDR_OF(fte_match_param, 1176 val2, inner_headers); 1177 void *fte_mask = MLX5_ADDR_OF(fte_match_param, 1178 mask->match_criteria, inner_headers); 1179 1180 if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2, 1181 MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4))) 1182 return false; 1183 } 1184 return true; 1185} 1186 1187bool fs_match_exact_mask(u8 match_criteria_enable1, 1188 u8 match_criteria_enable2, 1189 void *mask1, void *mask2) 1190{ 1191 return match_criteria_enable1 == match_criteria_enable2 && 1192 !memcmp(mask1, mask2, MLX5_ST_SZ_BYTES(fte_match_param)); 1193} 1194 1195static struct mlx5_flow_table *find_first_ft_in_ns_reverse(struct mlx5_flow_namespace *ns, 1196 struct list_head *start); 1197 1198static struct mlx5_flow_table *_find_first_ft_in_prio_reverse(struct fs_prio *prio, 1199 struct list_head *start) 1200{ 1201 struct fs_base *it = container_of(start, struct fs_base, list); 1202 1203 if (!prio) 1204 return NULL; 1205 1206 fs_for_each_ns_or_ft_continue_reverse(it, prio) { 1207 struct mlx5_flow_namespace *ns; 1208 struct mlx5_flow_table *ft; 1209 1210 if (it->type == FS_TYPE_FLOW_TABLE) { 1211 fs_get_obj(ft, it); 1212 fs_get(&ft->base); 1213 return ft; 1214 } 1215 1216 fs_get_obj(ns, it); 1217 WARN_ON(ns->base.type != FS_TYPE_NAMESPACE); 1218 1219 ft = find_first_ft_in_ns_reverse(ns, &ns->prios); 1220 if (ft) 1221 return ft; 1222 } 1223 1224 return NULL; 1225} 1226 1227static struct mlx5_flow_table *find_first_ft_in_prio_reverse(struct fs_prio *prio, 1228 struct list_head *start) 1229{ 1230 struct mlx5_flow_table *ft; 1231 1232 if (!prio) 1233 return NULL; 1234 1235 mutex_lock(&prio->base.lock); 1236 ft = _find_first_ft_in_prio_reverse(prio, start); 1237 mutex_unlock(&prio->base.lock); 1238 1239 return ft; 1240} 1241 1242static struct mlx5_flow_table *find_first_ft_in_ns_reverse(struct mlx5_flow_namespace *ns, 1243 struct list_head *start) 1244{ 1245 struct fs_prio *prio; 1246 1247 if (!ns) 1248 return NULL; 1249 1250 fs_get_obj(prio, container_of(start, struct fs_base, list)); 1251 mutex_lock(&ns->base.lock); 1252 fs_for_each_prio_continue_reverse(prio, ns) { 1253 struct mlx5_flow_table *ft; 1254 1255 ft = find_first_ft_in_prio_reverse(prio, &prio->objs); 1256 if (ft) { 1257 mutex_unlock(&ns->base.lock); 1258 return ft; 1259 } 1260 } 1261 mutex_unlock(&ns->base.lock); 1262 1263 return NULL; 1264} 1265 1266/* Returned a held ft, assumed curr is protected, assumed curr's parent is 1267 * locked 1268 */ 1269static struct mlx5_flow_table *find_prev_ft(struct mlx5_flow_table *curr, 1270 struct fs_prio *prio) 1271{ 1272 struct mlx5_flow_table *ft = NULL; 1273 struct fs_base *curr_base; 1274 1275 if (!curr) 1276 return NULL; 1277 1278 /* prio has either namespace or flow-tables, but not both */ 1279 if (!list_empty(&prio->objs) && 1280 list_first_entry(&prio->objs, struct mlx5_flow_table, base.list) != 1281 curr) 1282 return NULL; 1283 1284 while (!ft && prio) { 1285 struct mlx5_flow_namespace *ns; 1286 1287 fs_get_parent(ns, prio); 1288 ft = find_first_ft_in_ns_reverse(ns, &prio->base.list); 1289 curr_base = &ns->base; 1290 fs_get_parent(prio, ns); 1291 1292 if (prio && !ft) 1293 ft = find_first_ft_in_prio_reverse(prio, 1294 &curr_base->list); 1295 } 1296 return ft; 1297} 1298 1299static struct mlx5_flow_table *_find_first_ft_in_prio(struct fs_prio *prio, 1300 struct list_head *start) 1301{ 1302 struct fs_base *it = container_of(start, struct fs_base, list); 1303 1304 if (!prio) 1305 return NULL; 1306 1307 fs_for_each_ns_or_ft_continue(it, prio) { 1308 struct mlx5_flow_namespace *ns; 1309 struct mlx5_flow_table *ft; 1310 1311 if (it->type == FS_TYPE_FLOW_TABLE) { 1312 fs_get_obj(ft, it); 1313 fs_get(&ft->base); 1314 return ft; 1315 } 1316 1317 fs_get_obj(ns, it); 1318 WARN_ON(ns->base.type != FS_TYPE_NAMESPACE); 1319 1320 ft = find_first_ft_in_ns(ns, &ns->prios); 1321 if (ft) 1322 return ft; 1323 } 1324 1325 return NULL; 1326} 1327 1328static struct mlx5_flow_table *find_first_ft_in_prio(struct fs_prio *prio, 1329 struct list_head *start) 1330{ 1331 struct mlx5_flow_table *ft; 1332 1333 if (!prio) 1334 return NULL; 1335 1336 mutex_lock(&prio->base.lock); 1337 ft = _find_first_ft_in_prio(prio, start); 1338 mutex_unlock(&prio->base.lock); 1339 1340 return ft; 1341} 1342 1343static struct mlx5_flow_table *find_first_ft_in_ns(struct mlx5_flow_namespace *ns, 1344 struct list_head *start) 1345{ 1346 struct fs_prio *prio; 1347 1348 if (!ns) 1349 return NULL; 1350 1351 fs_get_obj(prio, container_of(start, struct fs_base, list)); 1352 mutex_lock(&ns->base.lock); 1353 fs_for_each_prio_continue(prio, ns) { 1354 struct mlx5_flow_table *ft; 1355 1356 ft = find_first_ft_in_prio(prio, &prio->objs); 1357 if (ft) { 1358 mutex_unlock(&ns->base.lock); 1359 return ft; 1360 } 1361 } 1362 mutex_unlock(&ns->base.lock); 1363 1364 return NULL; 1365} 1366 1367/* returned a held ft, assumed curr is protected, assumed curr's parent is 1368 * locked 1369 */ 1370static struct mlx5_flow_table *find_next_ft(struct fs_prio *prio) 1371{ 1372 struct mlx5_flow_table *ft = NULL; 1373 struct fs_base *curr_base; 1374 1375 while (!ft && prio) { 1376 struct mlx5_flow_namespace *ns; 1377 1378 fs_get_parent(ns, prio); 1379 ft = find_first_ft_in_ns(ns, &prio->base.list); 1380 curr_base = &ns->base; 1381 fs_get_parent(prio, ns); 1382 1383 if (!ft && prio) 1384 ft = _find_first_ft_in_prio(prio, &curr_base->list); 1385 } 1386 return ft; 1387} 1388 1389 1390/* called under ft mutex lock */ 1391static struct mlx5_flow_group *create_autogroup(struct mlx5_flow_table *ft, 1392 u8 match_criteria_enable, 1393 u32 *match_criteria) 1394{ 1395 unsigned int group_size; 1396 unsigned int candidate_index = 0; 1397 unsigned int candidate_group_num = 0; 1398 struct mlx5_flow_group *g; 1399 struct mlx5_flow_group *ret; 1400 struct list_head *prev = &ft->fgs; 1401 struct mlx5_core_dev *dev; 1402 u32 *in; 1403 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1404 void *match_criteria_addr; 1405 1406 if (!ft->autogroup.active) 1407 return ERR_PTR(-ENOENT); 1408 1409 dev = fs_get_dev(&ft->base); 1410 if (!dev) 1411 return ERR_PTR(-ENODEV); 1412 1413 in = mlx5_vzalloc(inlen); 1414 if (!in) { 1415 mlx5_core_warn(dev, "failed to allocate inbox\n"); 1416 return ERR_PTR(-ENOMEM); 1417 } 1418 1419 1420 if (ft->autogroup.num_types < ft->autogroup.max_types) 1421 group_size = ft->max_fte / (ft->autogroup.max_types + 1); 1422 else 1423 group_size = 1; 1424 1425 if (group_size == 0) { 1426 mlx5_core_warn(dev, 1427 "flow steering can't create group size of 0\n"); 1428 ret = ERR_PTR(-EINVAL); 1429 goto out; 1430 } 1431 1432 /* sorted by start_index */ 1433 fs_for_each_fg(g, ft) { 1434 candidate_group_num++; 1435 if (candidate_index + group_size > g->start_index) 1436 candidate_index = g->start_index + g->max_ftes; 1437 else 1438 break; 1439 prev = &g->base.list; 1440 } 1441 1442 if (candidate_index + group_size > ft->max_fte) { 1443 ret = ERR_PTR(-ENOSPC); 1444 goto out; 1445 } 1446 1447 MLX5_SET(create_flow_group_in, in, match_criteria_enable, 1448 match_criteria_enable); 1449 MLX5_SET(create_flow_group_in, in, start_flow_index, candidate_index); 1450 MLX5_SET(create_flow_group_in, in, end_flow_index, candidate_index + 1451 group_size - 1); 1452 match_criteria_addr = MLX5_ADDR_OF(create_flow_group_in, 1453 in, match_criteria); 1454 memcpy(match_criteria_addr, match_criteria, 1455 MLX5_ST_SZ_BYTES(fte_match_param)); 1456 1457 ret = fs_create_fg(dev, ft, prev, in, 0); 1458out: 1459 kvfree(in); 1460 return ret; 1461} 1462 1463static struct mlx5_flow_namespace *get_ns_with_notifiers(struct fs_base *node) 1464{ 1465 struct mlx5_flow_namespace *ns = NULL; 1466 1467 while (node && (node->type != FS_TYPE_NAMESPACE || 1468 list_empty(&container_of(node, struct 1469 mlx5_flow_namespace, 1470 base)->list_notifiers))) 1471 node = node->parent; 1472 1473 if (node) 1474 fs_get_obj(ns, node); 1475 1476 return ns; 1477} 1478 1479 1480/*Assumption- fte is locked*/ 1481static void call_to_add_rule_notifiers(struct mlx5_flow_rule *dst, 1482 struct fs_fte *fte) 1483{ 1484 struct mlx5_flow_namespace *ns; 1485 struct mlx5_flow_handler *iter_handler; 1486 struct fs_client_priv_data *iter_client; 1487 void *data; 1488 bool is_new_rule = list_first_entry(&fte->dests, 1489 struct mlx5_flow_rule, 1490 base.list) == dst; 1491 int err; 1492 1493 ns = get_ns_with_notifiers(&fte->base); 1494 if (!ns) 1495 return; 1496 1497 down_read(&ns->notifiers_rw_sem); 1498 list_for_each_entry(iter_handler, &ns->list_notifiers, 1499 list) { 1500 if (iter_handler->add_dst_cb) { 1501 data = NULL; 1502 mutex_lock(&dst->clients_lock); 1503 list_for_each_entry( 1504 iter_client, &dst->clients_data, list) { 1505 if (iter_client->fs_handler == iter_handler) { 1506 data = iter_client->client_dst_data; 1507 break; 1508 } 1509 } 1510 mutex_unlock(&dst->clients_lock); 1511 err = iter_handler->add_dst_cb(dst, 1512 is_new_rule, 1513 NULL, 1514 iter_handler->client_context); 1515 if (err) 1516 break; 1517 } 1518 } 1519 up_read(&ns->notifiers_rw_sem); 1520} 1521 1522static void call_to_del_rule_notifiers(struct mlx5_flow_rule *dst, 1523 struct fs_fte *fte) 1524{ 1525 struct mlx5_flow_namespace *ns; 1526 struct mlx5_flow_handler *iter_handler; 1527 struct fs_client_priv_data *iter_client; 1528 void *data; 1529 bool ctx_changed = (fte->dests_size == 0); 1530 1531 ns = get_ns_with_notifiers(&fte->base); 1532 if (!ns) 1533 return; 1534 down_read(&ns->notifiers_rw_sem); 1535 list_for_each_entry(iter_handler, &ns->list_notifiers, 1536 list) { 1537 data = NULL; 1538 mutex_lock(&dst->clients_lock); 1539 list_for_each_entry(iter_client, &dst->clients_data, list) { 1540 if (iter_client->fs_handler == iter_handler) { 1541 data = iter_client->client_dst_data; 1542 break; 1543 } 1544 } 1545 mutex_unlock(&dst->clients_lock); 1546 if (iter_handler->del_dst_cb) { 1547 iter_handler->del_dst_cb(dst, ctx_changed, data, 1548 iter_handler->client_context); 1549 } 1550 } 1551 up_read(&ns->notifiers_rw_sem); 1552} 1553 1554/* fte should not be deleted while calling this function */ 1555static struct mlx5_flow_rule *_fs_add_dst_fte(struct fs_fte *fte, 1556 struct mlx5_flow_group *fg, 1557 struct mlx5_flow_destination *dest) 1558{ 1559 struct mlx5_flow_table *ft; 1560 struct mlx5_flow_rule *dst; 1561 int err; 1562 1563 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 1564 if (!dst) 1565 return ERR_PTR(-ENOMEM); 1566 1567 memcpy(&dst->dest_attr, dest, sizeof(*dest)); 1568 dst->base.type = FS_TYPE_FLOW_DEST; 1569 INIT_LIST_HEAD(&dst->clients_data); 1570 mutex_init(&dst->clients_lock); 1571 fs_get_parent(ft, fg); 1572 /*Add dest to dests list- added as first element after the head*/ 1573 list_add_tail(&dst->base.list, &fte->dests); 1574 fte->dests_size++; 1575 err = mlx5_cmd_fs_set_fte(fs_get_dev(&ft->base), 1576 ft->vport, 1577 &fte->status, 1578 fte->val, ft->type, 1579 ft->id, fte->index, fg->id, fte->flow_tag, 1580 fte->action, fte->dests_size, &fte->dests); 1581 if (err) 1582 goto free_dst; 1583 1584 list_del(&dst->base.list); 1585 1586 return dst; 1587 1588free_dst: 1589 list_del(&dst->base.list); 1590 kfree(dst); 1591 fte->dests_size--; 1592 return ERR_PTR(err); 1593} 1594 1595static char *get_dest_name(struct mlx5_flow_destination *dest) 1596{ 1597 char *name = kzalloc(sizeof(char) * 20, GFP_KERNEL); 1598 1599 switch (dest->type) { 1600 case MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE: 1601 snprintf(name, 20, "dest_%s_%u", "flow_table", 1602 dest->ft->id); 1603 return name; 1604 case MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT: 1605 snprintf(name, 20, "dest_%s_%u", "vport", 1606 dest->vport_num); 1607 return name; 1608 case MLX5_FLOW_CONTEXT_DEST_TYPE_TIR: 1609 snprintf(name, 20, "dest_%s_%u", "tir", dest->tir_num); 1610 return name; 1611 default: 1612 kfree(name); 1613 return NULL; 1614 } 1615} 1616 1617/* assumed fg is locked */ 1618static unsigned int fs_get_free_fg_index(struct mlx5_flow_group *fg, 1619 struct list_head **prev) 1620{ 1621 struct fs_fte *fte; 1622 unsigned int start = fg->start_index; 1623 1624 if (prev) 1625 *prev = &fg->ftes; 1626 1627 /* assumed list is sorted by index */ 1628 fs_for_each_fte(fte, fg) { 1629 if (fte->index != start) 1630 return start; 1631 start++; 1632 if (prev) 1633 *prev = &fte->base.list; 1634 } 1635 1636 return start; 1637} 1638 1639 1640static struct fs_fte *fs_create_fte(struct mlx5_flow_group *fg, 1641 u32 *match_value, 1642 u8 action, 1643 u32 flow_tag, 1644 struct list_head **prev) 1645{ 1646 struct fs_fte *fte; 1647 int index = 0; 1648 1649 index = fs_get_free_fg_index(fg, prev); 1650 fte = fs_alloc_fte(action, flow_tag, match_value, index); 1651 if (IS_ERR(fte)) 1652 return fte; 1653 1654 return fte; 1655} 1656 1657static void add_rule_to_tree(struct mlx5_flow_rule *rule, 1658 struct fs_fte *fte) 1659{ 1660 char *dest_name; 1661 1662 dest_name = get_dest_name(&rule->dest_attr); 1663 fs_add_node(&rule->base, &fte->base, dest_name, 1); 1664 /* re-add to list, since fs_add_node reset our list */ 1665 list_add_tail(&rule->base.list, &fte->dests); 1666 kfree(dest_name); 1667 call_to_add_rule_notifiers(rule, fte); 1668} 1669 1670static void fs_del_dst(struct mlx5_flow_rule *dst) 1671{ 1672 struct mlx5_flow_table *ft; 1673 struct mlx5_flow_group *fg; 1674 struct fs_fte *fte; 1675 u32 *match_value; 1676 struct mlx5_core_dev *dev = fs_get_dev(&dst->base); 1677 int match_len = MLX5_ST_SZ_BYTES(fte_match_param); 1678 int err; 1679 1680 WARN_ON(!dev); 1681 1682 match_value = mlx5_vzalloc(match_len); 1683 if (!match_value) { 1684 mlx5_core_warn(dev, "failed to allocate inbox\n"); 1685 return; 1686 } 1687 1688 fs_get_parent(fte, dst); 1689 fs_get_parent(fg, fte); 1690 mutex_lock(&fg->base.lock); 1691 memcpy(match_value, fte->val, sizeof(fte->val)); 1692 /* ft can't be changed as fg is locked */ 1693 fs_get_parent(ft, fg); 1694 list_del(&dst->base.list); 1695 fte->dests_size--; 1696 if (fte->dests_size) { 1697 err = mlx5_cmd_fs_set_fte(dev, ft->vport, 1698 &fte->status, match_value, ft->type, 1699 ft->id, fte->index, fg->id, 1700 fte->flow_tag, fte->action, 1701 fte->dests_size, &fte->dests); 1702 if (err) { 1703 mlx5_core_warn(dev, "%s can't delete dst %s\n", 1704 __func__, dst->base.name); 1705 goto err; 1706 } 1707 } 1708 call_to_del_rule_notifiers(dst, fte); 1709err: 1710 mutex_unlock(&fg->base.lock); 1711 kvfree(match_value); 1712} 1713 1714static void fs_del_fte(struct fs_fte *fte) 1715{ 1716 struct mlx5_flow_table *ft; 1717 struct mlx5_flow_group *fg; 1718 int err; 1719 struct mlx5_core_dev *dev; 1720 1721 fs_get_parent(fg, fte); 1722 fs_get_parent(ft, fg); 1723 1724 dev = fs_get_dev(&ft->base); 1725 WARN_ON(!dev); 1726 1727 err = mlx5_cmd_fs_delete_fte(dev, ft->vport, &fte->status, 1728 ft->type, ft->id, fte->index); 1729 if (err) 1730 mlx5_core_warn(dev, "flow steering can't delete fte %s\n", 1731 fte->base.name); 1732 1733 fg->num_ftes--; 1734} 1735 1736/* assuming parent fg is locked */ 1737/* Add dst algorithm */ 1738static struct mlx5_flow_rule *fs_add_dst_fg(struct mlx5_flow_group *fg, 1739 u32 *match_value, 1740 u8 action, 1741 u32 flow_tag, 1742 struct mlx5_flow_destination *dest) 1743{ 1744 struct fs_fte *fte; 1745 struct mlx5_flow_rule *dst; 1746 struct mlx5_flow_table *ft; 1747 struct list_head *prev; 1748 char fte_name[20]; 1749 1750 mutex_lock(&fg->base.lock); 1751 fs_for_each_fte(fte, fg) { 1752 /* TODO: Check of size against PRM max size */ 1753 mutex_lock(&fte->base.lock); 1754 if (fs_match_exact_val(&fg->mask, match_value, &fte->val) && 1755 action == fte->action && flow_tag == fte->flow_tag) { 1756 dst = _fs_add_dst_fte(fte, fg, dest); 1757 mutex_unlock(&fte->base.lock); 1758 if (IS_ERR(dst)) 1759 goto unlock_fg; 1760 goto add_rule; 1761 } 1762 mutex_unlock(&fte->base.lock); 1763 } 1764 1765 fs_get_parent(ft, fg); 1766 if (fg->num_ftes == fg->max_ftes) { 1767 dst = ERR_PTR(-ENOSPC); 1768 goto unlock_fg; 1769 } 1770 1771 fte = fs_create_fte(fg, match_value, action, flow_tag, &prev); 1772 if (IS_ERR(fte)) { 1773 dst = (void *)fte; 1774 goto unlock_fg; 1775 } 1776 dst = _fs_add_dst_fte(fte, fg, dest); 1777 if (IS_ERR(dst)) { 1778 kfree(fte); 1779 goto unlock_fg; 1780 } 1781 1782 fg->num_ftes++; 1783 1784 snprintf(fte_name, sizeof(fte_name), "fte%u", fte->index); 1785 /* Add node to tree */ 1786 fs_add_node(&fte->base, &fg->base, fte_name, 0); 1787 list_add(&fte->base.list, prev); 1788add_rule: 1789 add_rule_to_tree(dst, fte); 1790unlock_fg: 1791 mutex_unlock(&fg->base.lock); 1792 return dst; 1793} 1794 1795static struct mlx5_flow_rule *fs_add_dst_ft(struct mlx5_flow_table *ft, 1796 u8 match_criteria_enable, 1797 u32 *match_criteria, 1798 u32 *match_value, 1799 u8 action, u32 flow_tag, 1800 struct mlx5_flow_destination *dest) 1801{ 1802 /*? where dst_entry is allocated*/ 1803 struct mlx5_flow_group *g; 1804 struct mlx5_flow_rule *dst; 1805 1806 fs_get(&ft->base); 1807 mutex_lock(&ft->base.lock); 1808 fs_for_each_fg(g, ft) 1809 if (fs_match_exact_mask(g->mask.match_criteria_enable, 1810 match_criteria_enable, 1811 g->mask.match_criteria, 1812 match_criteria)) { 1813 mutex_unlock(&ft->base.lock); 1814 1815 dst = fs_add_dst_fg(g, match_value, 1816 action, flow_tag, dest); 1817 if (PTR_ERR(dst) && PTR_ERR(dst) != -ENOSPC) 1818 goto unlock; 1819 } 1820 mutex_unlock(&ft->base.lock); 1821 1822 g = create_autogroup(ft, match_criteria_enable, match_criteria); 1823 if (IS_ERR(g)) { 1824 dst = (void *)g; 1825 goto unlock; 1826 } 1827 1828 dst = fs_add_dst_fg(g, match_value, 1829 action, flow_tag, dest); 1830 if (IS_ERR(dst)) { 1831 /* Remove assumes refcount > 0 and autogroup creates a group 1832 * with a refcount = 0. 1833 */ 1834 fs_get(&g->base); 1835 fs_remove_node(&g->base); 1836 goto unlock; 1837 } 1838 1839unlock: 1840 fs_put(&ft->base); 1841 return dst; 1842} 1843 1844struct mlx5_flow_rule * 1845mlx5_add_flow_rule(struct mlx5_flow_table *ft, 1846 u8 match_criteria_enable, 1847 u32 *match_criteria, 1848 u32 *match_value, 1849 u32 action, 1850 u32 flow_tag, 1851 struct mlx5_flow_destination *dest) 1852{ 1853 struct mlx5_flow_rule *dst; 1854 struct mlx5_flow_namespace *ns; 1855 1856 ns = get_ns_with_notifiers(&ft->base); 1857 if (ns) 1858 down_read(&ns->dests_rw_sem); 1859 dst = fs_add_dst_ft(ft, match_criteria_enable, match_criteria, 1860 match_value, action, flow_tag, dest); 1861 if (ns) 1862 up_read(&ns->dests_rw_sem); 1863 1864 return dst; 1865 1866 1867} 1868EXPORT_SYMBOL(mlx5_add_flow_rule); 1869 1870void mlx5_del_flow_rule(struct mlx5_flow_rule *dst) 1871{ 1872 struct mlx5_flow_namespace *ns; 1873 1874 ns = get_ns_with_notifiers(&dst->base); 1875 if (ns) 1876 down_read(&ns->dests_rw_sem); 1877 fs_remove_node(&dst->base); 1878 if (ns) 1879 up_read(&ns->dests_rw_sem); 1880} 1881EXPORT_SYMBOL(mlx5_del_flow_rule); 1882 1883#define MLX5_CORE_FS_ROOT_NS_NAME "root" 1884#define MLX5_CORE_FS_ESW_EGRESS_ACL "esw_egress_root" 1885#define MLX5_CORE_FS_ESW_INGRESS_ACL "esw_ingress_root" 1886#define MLX5_CORE_FS_FDB_ROOT_NS_NAME "fdb_root" 1887#define MLX5_CORE_FS_SNIFFER_RX_ROOT_NS_NAME "sniffer_rx_root" 1888#define MLX5_CORE_FS_SNIFFER_TX_ROOT_NS_NAME "sniffer_tx_root" 1889#define MLX5_CORE_FS_PRIO_MAX_FT 4 1890#define MLX5_CORE_FS_PRIO_MAX_NS 1 1891 1892static struct fs_prio *fs_create_prio(struct mlx5_flow_namespace *ns, 1893 unsigned prio, int max_ft, 1894 const char *name, u8 flags) 1895{ 1896 struct fs_prio *fs_prio; 1897 1898 fs_prio = kzalloc(sizeof(*fs_prio), GFP_KERNEL); 1899 if (!fs_prio) 1900 return ERR_PTR(-ENOMEM); 1901 1902 fs_prio->base.type = FS_TYPE_PRIO; 1903 fs_add_node(&fs_prio->base, &ns->base, name, 1); 1904 fs_prio->max_ft = max_ft; 1905 fs_prio->max_ns = MLX5_CORE_FS_PRIO_MAX_NS; 1906 fs_prio->prio = prio; 1907 fs_prio->flags = flags; 1908 list_add_tail(&fs_prio->base.list, &ns->prios); 1909 INIT_LIST_HEAD(&fs_prio->objs); 1910 mutex_init(&fs_prio->shared_lock); 1911 1912 return fs_prio; 1913} 1914 1915static void cleanup_root_ns(struct mlx5_core_dev *dev) 1916{ 1917 struct mlx5_flow_root_namespace *root_ns = dev->root_ns; 1918 struct fs_prio *iter_prio; 1919 1920 if (!root_ns) 1921 return; 1922 1923 /* stage 1 */ 1924 fs_for_each_prio(iter_prio, &root_ns->ns) { 1925 struct mlx5_flow_namespace *iter_ns; 1926 1927 fs_for_each_ns(iter_ns, iter_prio) { 1928 while (!list_empty(&iter_ns->prios)) { 1929 struct fs_base *iter_prio2 = 1930 list_first_entry(&iter_ns->prios, 1931 struct fs_base, 1932 list); 1933 1934 fs_remove_node(iter_prio2); 1935 } 1936 } 1937 } 1938 1939 /* stage 2 */ 1940 fs_for_each_prio(iter_prio, &root_ns->ns) { 1941 while (!list_empty(&iter_prio->objs)) { 1942 struct fs_base *iter_ns = 1943 list_first_entry(&iter_prio->objs, 1944 struct fs_base, 1945 list); 1946 1947 fs_remove_node(iter_ns); 1948 } 1949 } 1950 /* stage 3 */ 1951 while (!list_empty(&root_ns->ns.prios)) { 1952 struct fs_base *iter_prio = 1953 list_first_entry(&root_ns->ns.prios, 1954 struct fs_base, 1955 list); 1956 1957 fs_remove_node(iter_prio); 1958 } 1959 1960 fs_remove_node(&root_ns->ns.base); 1961 dev->root_ns = NULL; 1962} 1963 1964static void cleanup_single_prio_root_ns(struct mlx5_core_dev *dev, 1965 struct mlx5_flow_root_namespace *root_ns) 1966{ 1967 struct fs_base *prio; 1968 1969 if (!root_ns) 1970 return; 1971 1972 if (!list_empty(&root_ns->ns.prios)) { 1973 prio = list_first_entry(&root_ns->ns.prios, 1974 struct fs_base, 1975 list); 1976 fs_remove_node(prio); 1977 } 1978 fs_remove_node(&root_ns->ns.base); 1979 root_ns = NULL; 1980} 1981 1982void mlx5_cleanup_fs(struct mlx5_core_dev *dev) 1983{ 1984 cleanup_root_ns(dev); 1985 cleanup_single_prio_root_ns(dev, dev->sniffer_rx_root_ns); 1986 cleanup_single_prio_root_ns(dev, dev->sniffer_tx_root_ns); 1987 cleanup_single_prio_root_ns(dev, dev->fdb_root_ns); 1988 cleanup_single_prio_root_ns(dev, dev->esw_egress_root_ns); 1989 cleanup_single_prio_root_ns(dev, dev->esw_ingress_root_ns); 1990} 1991 1992static struct mlx5_flow_namespace *fs_init_namespace(struct mlx5_flow_namespace 1993 *ns) 1994{ 1995 ns->base.type = FS_TYPE_NAMESPACE; 1996 init_rwsem(&ns->dests_rw_sem); 1997 init_rwsem(&ns->notifiers_rw_sem); 1998 INIT_LIST_HEAD(&ns->prios); 1999 INIT_LIST_HEAD(&ns->list_notifiers); 2000 2001 return ns; 2002} 2003 2004static struct mlx5_flow_root_namespace *create_root_ns(struct mlx5_core_dev *dev, 2005 enum fs_ft_type 2006 table_type, 2007 char *name) 2008{ 2009 struct mlx5_flow_root_namespace *root_ns; 2010 struct mlx5_flow_namespace *ns; 2011 2012 /* create the root namespace */ 2013 root_ns = mlx5_vzalloc(sizeof(*root_ns)); 2014 if (!root_ns) 2015 goto err; 2016 2017 root_ns->dev = dev; 2018 root_ns->table_type = table_type; 2019 mutex_init(&root_ns->fs_chain_lock); 2020 2021 ns = &root_ns->ns; 2022 fs_init_namespace(ns); 2023 fs_add_node(&ns->base, NULL, name, 1); 2024 2025 return root_ns; 2026err: 2027 return NULL; 2028} 2029 2030static int init_fdb_root_ns(struct mlx5_core_dev *dev) 2031{ 2032 struct fs_prio *prio; 2033 2034 dev->fdb_root_ns = create_root_ns(dev, FS_FT_FDB, 2035 MLX5_CORE_FS_FDB_ROOT_NS_NAME); 2036 if (!dev->fdb_root_ns) 2037 return -ENOMEM; 2038 2039 /* create 1 prio*/ 2040 prio = fs_create_prio(&dev->fdb_root_ns->ns, 0, 1, "fdb_prio", 0); 2041 if (IS_ERR(prio)) 2042 return PTR_ERR(prio); 2043 else 2044 return 0; 2045} 2046 2047#define MAX_VPORTS 128 2048 2049static int init_egress_acl_root_ns(struct mlx5_core_dev *dev) 2050{ 2051 struct fs_prio *prio; 2052 2053 dev->esw_egress_root_ns = create_root_ns(dev, FS_FT_ESW_EGRESS_ACL, 2054 MLX5_CORE_FS_ESW_EGRESS_ACL); 2055 if (!dev->esw_egress_root_ns) 2056 return -ENOMEM; 2057 2058 /* create 1 prio*/ 2059 prio = fs_create_prio(&dev->esw_egress_root_ns->ns, 0, MAX_VPORTS, 2060 "esw_egress_prio", 0); 2061 if (IS_ERR(prio)) 2062 return PTR_ERR(prio); 2063 else 2064 return 0; 2065} 2066 2067static int init_ingress_acl_root_ns(struct mlx5_core_dev *dev) 2068{ 2069 struct fs_prio *prio; 2070 2071 dev->esw_ingress_root_ns = create_root_ns(dev, FS_FT_ESW_INGRESS_ACL, 2072 MLX5_CORE_FS_ESW_INGRESS_ACL); 2073 if (!dev->esw_ingress_root_ns) 2074 return -ENOMEM; 2075 2076 /* create 1 prio*/ 2077 prio = fs_create_prio(&dev->esw_ingress_root_ns->ns, 0, MAX_VPORTS, 2078 "esw_ingress_prio", 0); 2079 if (IS_ERR(prio)) 2080 return PTR_ERR(prio); 2081 else 2082 return 0; 2083} 2084 2085static int init_sniffer_rx_root_ns(struct mlx5_core_dev *dev) 2086{ 2087 struct fs_prio *prio; 2088 2089 dev->sniffer_rx_root_ns = create_root_ns(dev, FS_FT_SNIFFER_RX, 2090 MLX5_CORE_FS_SNIFFER_RX_ROOT_NS_NAME); 2091 if (!dev->sniffer_rx_root_ns) 2092 return -ENOMEM; 2093 2094 /* create 1 prio*/ 2095 prio = fs_create_prio(&dev->sniffer_rx_root_ns->ns, 0, 1, 2096 "sniffer_prio", 0); 2097 if (IS_ERR(prio)) 2098 return PTR_ERR(prio); 2099 else 2100 return 0; 2101} 2102 2103 2104static int init_sniffer_tx_root_ns(struct mlx5_core_dev *dev) 2105{ 2106 struct fs_prio *prio; 2107 2108 dev->sniffer_tx_root_ns = create_root_ns(dev, FS_FT_SNIFFER_TX, 2109 MLX5_CORE_FS_SNIFFER_TX_ROOT_NS_NAME); 2110 if (!dev->sniffer_tx_root_ns) 2111 return -ENOMEM; 2112 2113 /* create 1 prio*/ 2114 prio = fs_create_prio(&dev->sniffer_tx_root_ns->ns, 0, 1, 2115 "sniffer_prio", 0); 2116 if (IS_ERR(prio)) 2117 return PTR_ERR(prio); 2118 else 2119 return 0; 2120} 2121 2122static struct mlx5_flow_namespace *fs_create_namespace(struct fs_prio *prio, 2123 const char *name) 2124{ 2125 struct mlx5_flow_namespace *ns; 2126 2127 ns = kzalloc(sizeof(*ns), GFP_KERNEL); 2128 if (!ns) 2129 return ERR_PTR(-ENOMEM); 2130 2131 fs_init_namespace(ns); 2132 fs_add_node(&ns->base, &prio->base, name, 1); 2133 list_add_tail(&ns->base.list, &prio->objs); 2134 2135 return ns; 2136} 2137 2138#define FLOW_TABLE_BIT_SZ 1 2139#define GET_FLOW_TABLE_CAP(dev, offset) \ 2140 ((be32_to_cpu(*((__be32 *)(dev->hca_caps_cur[MLX5_CAP_FLOW_TABLE]) + \ 2141 offset / 32)) >> \ 2142 (32 - FLOW_TABLE_BIT_SZ - (offset & 0x1f))) & FLOW_TABLE_BIT_SZ) 2143 2144static bool has_required_caps(struct mlx5_core_dev *dev, struct node_caps *caps) 2145{ 2146 int i; 2147 2148 for (i = 0; i < caps->arr_sz; i++) { 2149 if (!GET_FLOW_TABLE_CAP(dev, caps->caps[i])) 2150 return false; 2151 } 2152 return true; 2153} 2154 2155static int _init_root_tree(struct mlx5_core_dev *dev, int max_ft_level, 2156 struct init_tree_node *node, struct fs_base *base_parent, 2157 struct init_tree_node *tree_parent) 2158{ 2159 struct mlx5_flow_namespace *fs_ns; 2160 struct fs_prio *fs_prio; 2161 int priority; 2162 struct fs_base *base; 2163 int i; 2164 int err = 0; 2165 2166 if (node->type == FS_TYPE_PRIO) { 2167 if ((node->min_ft_level > max_ft_level) || 2168 !has_required_caps(dev, &node->caps)) 2169 goto out; 2170 2171 fs_get_obj(fs_ns, base_parent); 2172 priority = node - tree_parent->children; 2173 fs_prio = fs_create_prio(fs_ns, priority, 2174 node->max_ft, 2175 node->name, node->flags); 2176 if (IS_ERR(fs_prio)) { 2177 err = PTR_ERR(fs_prio); 2178 goto out; 2179 } 2180 base = &fs_prio->base; 2181 } else if (node->type == FS_TYPE_NAMESPACE) { 2182 fs_get_obj(fs_prio, base_parent); 2183 fs_ns = fs_create_namespace(fs_prio, node->name); 2184 if (IS_ERR(fs_ns)) { 2185 err = PTR_ERR(fs_ns); 2186 goto out; 2187 } 2188 base = &fs_ns->base; 2189 } else { 2190 return -EINVAL; 2191 } 2192 for (i = 0; i < node->ar_size; i++) { 2193 err = _init_root_tree(dev, max_ft_level, &node->children[i], base, 2194 node); 2195 if (err) 2196 break; 2197 } 2198out: 2199 return err; 2200} 2201 2202static int init_root_tree(struct mlx5_core_dev *dev, int max_ft_level, 2203 struct init_tree_node *node, struct fs_base *parent) 2204{ 2205 int i; 2206 struct mlx5_flow_namespace *fs_ns; 2207 int err = 0; 2208 2209 fs_get_obj(fs_ns, parent); 2210 for (i = 0; i < node->ar_size; i++) { 2211 err = _init_root_tree(dev, max_ft_level, 2212 &node->children[i], &fs_ns->base, node); 2213 if (err) 2214 break; 2215 } 2216 return err; 2217} 2218 2219static int sum_max_ft_in_prio(struct fs_prio *prio); 2220static int sum_max_ft_in_ns(struct mlx5_flow_namespace *ns) 2221{ 2222 struct fs_prio *prio; 2223 int sum = 0; 2224 2225 fs_for_each_prio(prio, ns) { 2226 sum += sum_max_ft_in_prio(prio); 2227 } 2228 return sum; 2229} 2230 2231static int sum_max_ft_in_prio(struct fs_prio *prio) 2232{ 2233 int sum = 0; 2234 struct fs_base *it; 2235 struct mlx5_flow_namespace *ns; 2236 2237 if (prio->max_ft) 2238 return prio->max_ft; 2239 2240 fs_for_each_ns_or_ft(it, prio) { 2241 if (it->type == FS_TYPE_FLOW_TABLE) 2242 continue; 2243 2244 fs_get_obj(ns, it); 2245 sum += sum_max_ft_in_ns(ns); 2246 } 2247 prio->max_ft = sum; 2248 return sum; 2249} 2250 2251static void set_max_ft(struct mlx5_flow_namespace *ns) 2252{ 2253 struct fs_prio *prio; 2254 2255 if (!ns) 2256 return; 2257 2258 fs_for_each_prio(prio, ns) 2259 sum_max_ft_in_prio(prio); 2260} 2261 2262static int init_root_ns(struct mlx5_core_dev *dev) 2263{ 2264 int max_ft_level = MLX5_CAP_FLOWTABLE(dev, 2265 flow_table_properties_nic_receive. 2266 max_ft_level); 2267 2268 dev->root_ns = create_root_ns(dev, FS_FT_NIC_RX, 2269 MLX5_CORE_FS_ROOT_NS_NAME); 2270 if (IS_ERR_OR_NULL(dev->root_ns)) 2271 goto err; 2272 2273 2274 if (init_root_tree(dev, max_ft_level, &root_fs, &dev->root_ns->ns.base)) 2275 goto err; 2276 2277 set_max_ft(&dev->root_ns->ns); 2278 2279 return 0; 2280err: 2281 return -ENOMEM; 2282} 2283 2284u8 mlx5_get_match_criteria_enable(struct mlx5_flow_rule *rule) 2285{ 2286 struct fs_base *pbase; 2287 struct mlx5_flow_group *fg; 2288 2289 pbase = rule->base.parent; 2290 WARN_ON(!pbase); 2291 pbase = pbase->parent; 2292 WARN_ON(!pbase); 2293 2294 fs_get_obj(fg, pbase); 2295 return fg->mask.match_criteria_enable; 2296} 2297 2298void mlx5_get_match_value(u32 *match_value, 2299 struct mlx5_flow_rule *rule) 2300{ 2301 struct fs_base *pbase; 2302 struct fs_fte *fte; 2303 2304 pbase = rule->base.parent; 2305 WARN_ON(!pbase); 2306 fs_get_obj(fte, pbase); 2307 2308 memcpy(match_value, fte->val, sizeof(fte->val)); 2309} 2310 2311void mlx5_get_match_criteria(u32 *match_criteria, 2312 struct mlx5_flow_rule *rule) 2313{ 2314 struct fs_base *pbase; 2315 struct mlx5_flow_group *fg; 2316 2317 pbase = rule->base.parent; 2318 WARN_ON(!pbase); 2319 pbase = pbase->parent; 2320 WARN_ON(!pbase); 2321 2322 fs_get_obj(fg, pbase); 2323 memcpy(match_criteria, &fg->mask.match_criteria, 2324 sizeof(fg->mask.match_criteria)); 2325} 2326 2327int mlx5_init_fs(struct mlx5_core_dev *dev) 2328{ 2329 int err; 2330 2331 if (MLX5_CAP_GEN(dev, nic_flow_table)) { 2332 err = init_root_ns(dev); 2333 if (err) 2334 goto err; 2335 } 2336 2337 err = init_fdb_root_ns(dev); 2338 if (err) 2339 goto err; 2340 2341 err = init_egress_acl_root_ns(dev); 2342 if (err) 2343 goto err; 2344 2345 err = init_ingress_acl_root_ns(dev); 2346 if (err) 2347 goto err; 2348 2349 err = init_sniffer_tx_root_ns(dev); 2350 if (err) 2351 goto err; 2352 2353 err = init_sniffer_rx_root_ns(dev); 2354 if (err) 2355 goto err; 2356 2357 return 0; 2358err: 2359 mlx5_cleanup_fs(dev); 2360 return err; 2361} 2362 2363struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev, 2364 enum mlx5_flow_namespace_type type) 2365{ 2366 struct mlx5_flow_root_namespace *root_ns = dev->root_ns; 2367 int prio; 2368 static struct fs_prio *fs_prio; 2369 struct mlx5_flow_namespace *ns; 2370 2371 switch (type) { 2372 case MLX5_FLOW_NAMESPACE_BYPASS: 2373 prio = 0; 2374 break; 2375 case MLX5_FLOW_NAMESPACE_KERNEL: 2376 prio = 1; 2377 break; 2378 case MLX5_FLOW_NAMESPACE_LEFTOVERS: 2379 prio = 2; 2380 break; 2381 case MLX5_FLOW_NAMESPACE_FDB: 2382 if (dev->fdb_root_ns) 2383 return &dev->fdb_root_ns->ns; 2384 else 2385 return NULL; 2386 case MLX5_FLOW_NAMESPACE_ESW_EGRESS: 2387 if (dev->esw_egress_root_ns) 2388 return &dev->esw_egress_root_ns->ns; 2389 else 2390 return NULL; 2391 case MLX5_FLOW_NAMESPACE_ESW_INGRESS: 2392 if (dev->esw_ingress_root_ns) 2393 return &dev->esw_ingress_root_ns->ns; 2394 else 2395 return NULL; 2396 case MLX5_FLOW_NAMESPACE_SNIFFER_RX: 2397 if (dev->sniffer_rx_root_ns) 2398 return &dev->sniffer_rx_root_ns->ns; 2399 else 2400 return NULL; 2401 case MLX5_FLOW_NAMESPACE_SNIFFER_TX: 2402 if (dev->sniffer_tx_root_ns) 2403 return &dev->sniffer_tx_root_ns->ns; 2404 else 2405 return NULL; 2406 default: 2407 return NULL; 2408 } 2409 2410 if (!root_ns) 2411 return NULL; 2412 2413 fs_prio = find_prio(&root_ns->ns, prio); 2414 if (!fs_prio) 2415 return NULL; 2416 2417 ns = list_first_entry(&fs_prio->objs, 2418 typeof(*ns), 2419 base.list); 2420 2421 return ns; 2422} 2423EXPORT_SYMBOL(mlx5_get_flow_namespace); 2424 2425 2426int mlx5_set_rule_private_data(struct mlx5_flow_rule *rule, 2427 struct mlx5_flow_handler *fs_handler, 2428 void *client_data) 2429{ 2430 struct fs_client_priv_data *priv_data; 2431 2432 mutex_lock(&rule->clients_lock); 2433 /*Check that hanlder isn't exists in the list already*/ 2434 list_for_each_entry(priv_data, &rule->clients_data, list) { 2435 if (priv_data->fs_handler == fs_handler) { 2436 priv_data->client_dst_data = client_data; 2437 goto unlock; 2438 } 2439 } 2440 priv_data = kzalloc(sizeof(*priv_data), GFP_KERNEL); 2441 if (!priv_data) { 2442 mutex_unlock(&rule->clients_lock); 2443 return -ENOMEM; 2444 } 2445 2446 priv_data->client_dst_data = client_data; 2447 priv_data->fs_handler = fs_handler; 2448 list_add(&priv_data->list, &rule->clients_data); 2449 2450unlock: 2451 mutex_unlock(&rule->clients_lock); 2452 2453 return 0; 2454} 2455 2456static int remove_from_clients(struct mlx5_flow_rule *rule, 2457 bool ctx_changed, 2458 void *client_data, 2459 void *context) 2460{ 2461 struct fs_client_priv_data *iter_client; 2462 struct fs_client_priv_data *temp_client; 2463 struct mlx5_flow_handler *handler = (struct 2464 mlx5_flow_handler*)context; 2465 2466 mutex_lock(&rule->clients_lock); 2467 list_for_each_entry_safe(iter_client, temp_client, 2468 &rule->clients_data, list) { 2469 if (iter_client->fs_handler == handler) { 2470 list_del(&iter_client->list); 2471 kfree(iter_client); 2472 break; 2473 } 2474 } 2475 mutex_unlock(&rule->clients_lock); 2476 2477 return 0; 2478} 2479 2480struct mlx5_flow_handler *mlx5_register_rule_notifier(struct mlx5_core_dev *dev, 2481 enum mlx5_flow_namespace_type ns_type, 2482 rule_event_fn add_cb, 2483 rule_event_fn del_cb, 2484 void *context) 2485{ 2486 struct mlx5_flow_namespace *ns; 2487 struct mlx5_flow_handler *handler; 2488 2489 ns = mlx5_get_flow_namespace(dev, ns_type); 2490 if (!ns) 2491 return ERR_PTR(-EINVAL); 2492 2493 handler = kzalloc(sizeof(*handler), GFP_KERNEL); 2494 if (!handler) 2495 return ERR_PTR(-ENOMEM); 2496 2497 handler->add_dst_cb = add_cb; 2498 handler->del_dst_cb = del_cb; 2499 handler->client_context = context; 2500 handler->ns = ns; 2501 down_write(&ns->notifiers_rw_sem); 2502 list_add_tail(&handler->list, &ns->list_notifiers); 2503 up_write(&ns->notifiers_rw_sem); 2504 2505 return handler; 2506} 2507 2508static void iterate_rules_in_ns(struct mlx5_flow_namespace *ns, 2509 rule_event_fn add_rule_cb, 2510 void *context); 2511 2512void mlx5_unregister_rule_notifier(struct mlx5_flow_handler *handler) 2513{ 2514 struct mlx5_flow_namespace *ns = handler->ns; 2515 2516 /*Remove from dst's clients*/ 2517 down_write(&ns->dests_rw_sem); 2518 down_write(&ns->notifiers_rw_sem); 2519 iterate_rules_in_ns(ns, remove_from_clients, handler); 2520 list_del(&handler->list); 2521 up_write(&ns->notifiers_rw_sem); 2522 up_write(&ns->dests_rw_sem); 2523 kfree(handler); 2524} 2525 2526static void iterate_rules_in_ft(struct mlx5_flow_table *ft, 2527 rule_event_fn add_rule_cb, 2528 void *context) 2529{ 2530 struct mlx5_flow_group *iter_fg; 2531 struct fs_fte *iter_fte; 2532 struct mlx5_flow_rule *iter_rule; 2533 int err = 0; 2534 bool is_new_rule; 2535 2536 mutex_lock(&ft->base.lock); 2537 fs_for_each_fg(iter_fg, ft) { 2538 mutex_lock(&iter_fg->base.lock); 2539 fs_for_each_fte(iter_fte, iter_fg) { 2540 mutex_lock(&iter_fte->base.lock); 2541 is_new_rule = true; 2542 fs_for_each_dst(iter_rule, iter_fte) { 2543 fs_get(&iter_rule->base); 2544 err = add_rule_cb(iter_rule, 2545 is_new_rule, 2546 NULL, 2547 context); 2548 fs_put_parent_locked(&iter_rule->base); 2549 if (err) 2550 break; 2551 is_new_rule = false; 2552 } 2553 mutex_unlock(&iter_fte->base.lock); 2554 if (err) 2555 break; 2556 } 2557 mutex_unlock(&iter_fg->base.lock); 2558 if (err) 2559 break; 2560 } 2561 mutex_unlock(&ft->base.lock); 2562} 2563 2564static void iterate_rules_in_prio(struct fs_prio *prio, 2565 rule_event_fn add_rule_cb, 2566 void *context) 2567{ 2568 struct fs_base *it; 2569 2570 mutex_lock(&prio->base.lock); 2571 fs_for_each_ns_or_ft(it, prio) { 2572 if (it->type == FS_TYPE_FLOW_TABLE) { 2573 struct mlx5_flow_table *ft; 2574 2575 fs_get_obj(ft, it); 2576 iterate_rules_in_ft(ft, add_rule_cb, context); 2577 } else { 2578 struct mlx5_flow_namespace *ns; 2579 2580 fs_get_obj(ns, it); 2581 iterate_rules_in_ns(ns, add_rule_cb, context); 2582 } 2583 } 2584 mutex_unlock(&prio->base.lock); 2585} 2586 2587static void iterate_rules_in_ns(struct mlx5_flow_namespace *ns, 2588 rule_event_fn add_rule_cb, 2589 void *context) 2590{ 2591 struct fs_prio *iter_prio; 2592 2593 mutex_lock(&ns->base.lock); 2594 fs_for_each_prio(iter_prio, ns) { 2595 iterate_rules_in_prio(iter_prio, add_rule_cb, context); 2596 } 2597 mutex_unlock(&ns->base.lock); 2598} 2599 2600void mlx5_flow_iterate_existing_rules(struct mlx5_flow_namespace *ns, 2601 rule_event_fn add_rule_cb, 2602 void *context) 2603{ 2604 down_write(&ns->dests_rw_sem); 2605 down_read(&ns->notifiers_rw_sem); 2606 iterate_rules_in_ns(ns, add_rule_cb, context); 2607 up_read(&ns->notifiers_rw_sem); 2608 up_write(&ns->dests_rw_sem); 2609} 2610 2611 2612void mlx5_del_flow_rules_list(struct mlx5_flow_rules_list *rules_list) 2613{ 2614 struct mlx5_flow_rule_node *iter_node; 2615 struct mlx5_flow_rule_node *temp_node; 2616 2617 list_for_each_entry_safe(iter_node, temp_node, &rules_list->head, list) { 2618 list_del(&iter_node->list); 2619 kfree(iter_node); 2620 } 2621 2622 kfree(rules_list); 2623} 2624 2625#define ROCEV1_ETHERTYPE 0x8915 2626static int set_rocev1_rules(struct list_head *rules_list) 2627{ 2628 struct mlx5_flow_rule_node *rocev1_rule; 2629 2630 rocev1_rule = kzalloc(sizeof(*rocev1_rule), GFP_KERNEL); 2631 if (!rocev1_rule) 2632 return -ENOMEM; 2633 2634 rocev1_rule->match_criteria_enable = 2635 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS; 2636 MLX5_SET(fte_match_set_lyr_2_4, rocev1_rule->match_criteria, ethertype, 2637 0xffff); 2638 MLX5_SET(fte_match_set_lyr_2_4, rocev1_rule->match_value, ethertype, 2639 ROCEV1_ETHERTYPE); 2640 2641 list_add_tail(&rocev1_rule->list, rules_list); 2642 2643 return 0; 2644} 2645 2646#define ROCEV2_UDP_PORT 4791 2647static int set_rocev2_rules(struct list_head *rules_list) 2648{ 2649 struct mlx5_flow_rule_node *ipv4_rule; 2650 struct mlx5_flow_rule_node *ipv6_rule; 2651 2652 ipv4_rule = kzalloc(sizeof(*ipv4_rule), GFP_KERNEL); 2653 if (!ipv4_rule) 2654 return -ENOMEM; 2655 2656 ipv6_rule = kzalloc(sizeof(*ipv6_rule), GFP_KERNEL); 2657 if (!ipv6_rule) { 2658 kfree(ipv4_rule); 2659 return -ENOMEM; 2660 } 2661 2662 ipv4_rule->match_criteria_enable = 2663 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS; 2664 MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, ethertype, 2665 0xffff); 2666 MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, ethertype, 2667 0x0800); 2668 MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, ip_protocol, 2669 0xff); 2670 MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, ip_protocol, 2671 IPPROTO_UDP); 2672 MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, udp_dport, 2673 0xffff); 2674 MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, udp_dport, 2675 ROCEV2_UDP_PORT); 2676 2677 ipv6_rule->match_criteria_enable = 2678 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS; 2679 MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, ethertype, 2680 0xffff); 2681 MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, ethertype, 2682 0x86dd); 2683 MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, ip_protocol, 2684 0xff); 2685 MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, ip_protocol, 2686 IPPROTO_UDP); 2687 MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, udp_dport, 2688 0xffff); 2689 MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, udp_dport, 2690 ROCEV2_UDP_PORT); 2691 2692 list_add_tail(&ipv4_rule->list, rules_list); 2693 list_add_tail(&ipv6_rule->list, rules_list); 2694 2695 return 0; 2696} 2697 2698 2699struct mlx5_flow_rules_list *get_roce_flow_rules(u8 roce_mode) 2700{ 2701 int err = 0; 2702 struct mlx5_flow_rules_list *rules_list = 2703 kzalloc(sizeof(*rules_list), GFP_KERNEL); 2704 2705 if (!rules_list) 2706 return NULL; 2707 2708 INIT_LIST_HEAD(&rules_list->head); 2709 2710 if (roce_mode & MLX5_ROCE_VERSION_1_CAP) { 2711 err = set_rocev1_rules(&rules_list->head); 2712 if (err) 2713 goto free_list; 2714 } 2715 if (roce_mode & MLX5_ROCE_VERSION_2_CAP) 2716 err = set_rocev2_rules(&rules_list->head); 2717 if (err) 2718 goto free_list; 2719 2720 return rules_list; 2721 2722free_list: 2723 mlx5_del_flow_rules_list(rules_list); 2724 return NULL; 2725} 2726