1/*- 2 * Copyright (c) 2013-2021, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26#include "opt_rss.h" 27#include "opt_ratelimit.h" 28 29#include <linux/module.h> 30#include <dev/mlx5/driver.h> 31#include <dev/mlx5/mlx5_core/mlx5_core.h> 32#include <dev/mlx5/mlx5_core/fs_core.h> 33#include <linux/string.h> 34#include <linux/compiler.h> 35 36#define INIT_TREE_NODE_ARRAY_SIZE(...) (sizeof((struct init_tree_node[]){__VA_ARGS__}) /\ 37 sizeof(struct init_tree_node)) 38 39#define ADD_PRIO(name_val, flags_val, min_level_val, max_ft_val, caps_val, \ 40 ...) {.type = FS_TYPE_PRIO,\ 41 .name = name_val,\ 42 .min_ft_level = min_level_val,\ 43 .flags = flags_val,\ 44 .max_ft = max_ft_val,\ 45 .caps = caps_val,\ 46 .children = (struct init_tree_node[]) {__VA_ARGS__},\ 47 .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \ 48} 49 50#define ADD_FT_PRIO(name_val, flags_val, max_ft_val, ...)\ 51 ADD_PRIO(name_val, flags_val, 0, max_ft_val, {},\ 52 __VA_ARGS__)\ 53 54#define ADD_NS(name_val, ...) {.type = FS_TYPE_NAMESPACE,\ 55 .name = name_val,\ 56 .children = (struct init_tree_node[]) {__VA_ARGS__},\ 57 .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \ 58} 59 60#define INIT_CAPS_ARRAY_SIZE(...) (sizeof((long[]){__VA_ARGS__}) /\ 61 sizeof(long)) 62 63#define FS_CAP(cap) (__mlx5_bit_off(flow_table_nic_cap, cap)) 64 65#define FS_REQUIRED_CAPS(...) {.arr_sz = INIT_CAPS_ARRAY_SIZE(__VA_ARGS__), \ 66 .caps = (long[]) {__VA_ARGS__}} 67 68/* Flowtable sizes: */ 69#define BYPASS_MAX_FT 5 70#define BYPASS_PRIO_MAX_FT 1 71#define OFFLOADS_MAX_FT 2 72#define KERNEL_MAX_FT 5 73#define LEFTOVER_MAX_FT 1 74 75/* Flowtable levels: */ 76#define OFFLOADS_MIN_LEVEL 3 77#define KERNEL_MIN_LEVEL (OFFLOADS_MIN_LEVEL + 1) 78#define LEFTOVER_MIN_LEVEL (KERNEL_MIN_LEVEL + 1) 79#define BYPASS_MIN_LEVEL (MLX5_NUM_BYPASS_FTS + LEFTOVER_MIN_LEVEL) 80 81struct node_caps { 82 size_t arr_sz; 83 long *caps; 84}; 85 86struct init_tree_node { 87 enum fs_type type; 88 const char *name; 89 struct init_tree_node *children; 90 int ar_size; 91 struct node_caps caps; 92 u8 flags; 93 int min_ft_level; 94 int prio; 95 int max_ft; 96} root_fs = { 97 .type = FS_TYPE_NAMESPACE, 98 .name = "root", 99 .ar_size = 4, 100 .children = (struct init_tree_node[]) { 101 ADD_PRIO("by_pass_prio", 0, BYPASS_MIN_LEVEL, 0, 102 FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), 103 FS_CAP(flow_table_properties_nic_receive.modify_root)), 104 ADD_NS("by_pass_ns", 105 ADD_FT_PRIO("prio0", 0, 106 BYPASS_PRIO_MAX_FT), 107 ADD_FT_PRIO("prio1", 0, 108 BYPASS_PRIO_MAX_FT), 109 ADD_FT_PRIO("prio2", 0, 110 BYPASS_PRIO_MAX_FT), 111 ADD_FT_PRIO("prio3", 0, 112 BYPASS_PRIO_MAX_FT), 113 ADD_FT_PRIO("prio4", 0, 114 BYPASS_PRIO_MAX_FT), 115 ADD_FT_PRIO("prio5", 0, 116 BYPASS_PRIO_MAX_FT), 117 ADD_FT_PRIO("prio6", 0, 118 BYPASS_PRIO_MAX_FT), 119 ADD_FT_PRIO("prio7", 0, 120 BYPASS_PRIO_MAX_FT), 121 ADD_FT_PRIO("prio-mcast", 0, 122 BYPASS_PRIO_MAX_FT))), 123 ADD_PRIO("offloads_prio", 0, OFFLOADS_MIN_LEVEL, 0, {}, 124 ADD_NS("offloads_ns", 125 ADD_FT_PRIO("prio_offloads-0", 0, 126 OFFLOADS_MAX_FT))), 127 ADD_PRIO("kernel_prio", 0, KERNEL_MIN_LEVEL, 0, {}, 128 ADD_NS("kernel_ns", 129 ADD_FT_PRIO("prio_kernel-0", 0, 130 KERNEL_MAX_FT))), 131 ADD_PRIO("leftovers_prio", MLX5_CORE_FS_PRIO_SHARED, 132 LEFTOVER_MIN_LEVEL, 0, 133 FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), 134 FS_CAP(flow_table_properties_nic_receive.modify_root)), 135 ADD_NS("leftover_ns", 136 ADD_FT_PRIO("leftovers_prio-0", 137 MLX5_CORE_FS_PRIO_SHARED, 138 LEFTOVER_MAX_FT))) 139 } 140}; 141 142/* Tree creation functions */ 143 144static struct mlx5_flow_root_namespace *find_root(struct fs_base *node) 145{ 146 struct fs_base *parent; 147 148 /* Make sure we only read it once while we go up the tree */ 149 while ((parent = node->parent)) 150 node = parent; 151 152 if (node->type != FS_TYPE_NAMESPACE) { 153 return NULL; 154 } 155 156 return container_of(container_of(node, 157 struct mlx5_flow_namespace, 158 base), 159 struct mlx5_flow_root_namespace, 160 ns); 161} 162 163static inline struct mlx5_core_dev *fs_get_dev(struct fs_base *node) 164{ 165 struct mlx5_flow_root_namespace *root = find_root(node); 166 167 if (root) 168 return root->dev; 169 return NULL; 170} 171 172static void fs_init_node(struct fs_base *node, 173 unsigned int refcount) 174{ 175 kref_init(&node->refcount); 176 atomic_set(&node->users_refcount, refcount); 177 init_completion(&node->complete); 178 INIT_LIST_HEAD(&node->list); 179 mutex_init(&node->lock); 180} 181 182static void _fs_add_node(struct fs_base *node, 183 const char *name, 184 struct fs_base *parent) 185{ 186 if (parent) 187 atomic_inc(&parent->users_refcount); 188 node->name = kstrdup_const(name, GFP_KERNEL); 189 node->parent = parent; 190} 191 192static void fs_add_node(struct fs_base *node, 193 struct fs_base *parent, const char *name, 194 unsigned int refcount) 195{ 196 fs_init_node(node, refcount); 197 _fs_add_node(node, name, parent); 198} 199 200static void _fs_put(struct fs_base *node, void (*kref_cb)(struct kref *kref), 201 bool parent_locked); 202 203static void fs_del_dst(struct mlx5_flow_rule *dst); 204static void _fs_del_ft(struct mlx5_flow_table *ft); 205static void fs_del_fg(struct mlx5_flow_group *fg); 206static void fs_del_fte(struct fs_fte *fte); 207 208static void cmd_remove_node(struct fs_base *base) 209{ 210 switch (base->type) { 211 case FS_TYPE_FLOW_DEST: 212 fs_del_dst(container_of(base, struct mlx5_flow_rule, base)); 213 break; 214 case FS_TYPE_FLOW_TABLE: 215 _fs_del_ft(container_of(base, struct mlx5_flow_table, base)); 216 break; 217 case FS_TYPE_FLOW_GROUP: 218 fs_del_fg(container_of(base, struct mlx5_flow_group, base)); 219 break; 220 case FS_TYPE_FLOW_ENTRY: 221 fs_del_fte(container_of(base, struct fs_fte, base)); 222 break; 223 default: 224 break; 225 } 226} 227 228static void __fs_remove_node(struct kref *kref) 229{ 230 struct fs_base *node = container_of(kref, struct fs_base, refcount); 231 232 if (node->parent) { 233 if (node->type == FS_TYPE_FLOW_DEST) 234 mutex_lock(&node->parent->parent->lock); 235 mutex_lock(&node->parent->lock); 236 } 237 mutex_lock(&node->lock); 238 cmd_remove_node(node); 239 mutex_unlock(&node->lock); 240 complete(&node->complete); 241 if (node->parent) { 242 mutex_unlock(&node->parent->lock); 243 if (node->type == FS_TYPE_FLOW_DEST) 244 mutex_unlock(&node->parent->parent->lock); 245 _fs_put(node->parent, _fs_remove_node, false); 246 } 247} 248 249void _fs_remove_node(struct kref *kref) 250{ 251 struct fs_base *node = container_of(kref, struct fs_base, refcount); 252 253 __fs_remove_node(kref); 254 kfree_const(node->name); 255 kfree(node); 256} 257 258static void fs_get(struct fs_base *node) 259{ 260 atomic_inc(&node->users_refcount); 261} 262 263static void _fs_put(struct fs_base *node, void (*kref_cb)(struct kref *kref), 264 bool parent_locked) 265{ 266 struct fs_base *parent_node = node->parent; 267 268 if (parent_node && !parent_locked) 269 mutex_lock(&parent_node->lock); 270 if (atomic_dec_and_test(&node->users_refcount)) { 271 if (parent_node) { 272 /*remove from parent's list*/ 273 list_del_init(&node->list); 274 mutex_unlock(&parent_node->lock); 275 } 276 kref_put(&node->refcount, kref_cb); 277 if (parent_node && parent_locked) 278 mutex_lock(&parent_node->lock); 279 } else if (parent_node && !parent_locked) { 280 mutex_unlock(&parent_node->lock); 281 } 282} 283 284static void fs_put(struct fs_base *node) 285{ 286 _fs_put(node, __fs_remove_node, false); 287} 288 289static void fs_put_parent_locked(struct fs_base *node) 290{ 291 _fs_put(node, __fs_remove_node, true); 292} 293 294static void fs_remove_node(struct fs_base *node) 295{ 296 fs_put(node); 297 wait_for_completion(&node->complete); 298 kfree_const(node->name); 299 kfree(node); 300} 301 302static void fs_remove_node_parent_locked(struct fs_base *node) 303{ 304 fs_put_parent_locked(node); 305 wait_for_completion(&node->complete); 306 kfree_const(node->name); 307 kfree(node); 308} 309 310static struct fs_fte *fs_alloc_fte(u32 sw_action, 311 struct mlx5_flow_act *flow_act, 312 u32 *match_value, 313 unsigned int index) 314{ 315 struct fs_fte *fte; 316 317 318 fte = kzalloc(sizeof(*fte), GFP_KERNEL); 319 if (!fte) 320 return ERR_PTR(-ENOMEM); 321 322 memcpy(fte->val, match_value, sizeof(fte->val)); 323 fte->base.type = FS_TYPE_FLOW_ENTRY; 324 fte->dests_size = 0; 325 fte->index = index; 326 INIT_LIST_HEAD(&fte->dests); 327 fte->flow_act = *flow_act; 328 fte->sw_action = sw_action; 329 330 return fte; 331} 332 333static struct fs_fte *alloc_star_ft_entry(struct mlx5_flow_table *ft, 334 struct mlx5_flow_group *fg, 335 u32 *match_value, 336 unsigned int index) 337{ 338 int err; 339 struct fs_fte *fte; 340 struct mlx5_flow_rule *dst; 341 struct mlx5_flow_act flow_act = { 342 .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG, 343 .flow_tag = MLX5_FS_DEFAULT_FLOW_TAG, 344 }; 345 346 if (fg->num_ftes == fg->max_ftes) 347 return ERR_PTR(-ENOSPC); 348 349 fte = fs_alloc_fte(MLX5_FLOW_RULE_FWD_ACTION_DEST, 350 &flow_act, match_value, index); 351 if (IS_ERR(fte)) 352 return fte; 353 354 /*create dst*/ 355 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 356 if (!dst) { 357 err = -ENOMEM; 358 goto free_fte; 359 } 360 361 fte->base.parent = &fg->base; 362 fte->dests_size = 1; 363 dst->dest_attr.type = MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE; 364 dst->base.parent = &fte->base; 365 list_add(&dst->base.list, &fte->dests); 366 /* assumed that the callee creates the star rules sorted by index */ 367 list_add_tail(&fte->base.list, &fg->ftes); 368 fg->num_ftes++; 369 370 return fte; 371 372free_fte: 373 kfree(fte); 374 return ERR_PTR(err); 375} 376 377/* assume that fte can't be changed */ 378static void free_star_fte_entry(struct fs_fte *fte) 379{ 380 struct mlx5_flow_group *fg; 381 struct mlx5_flow_rule *dst, *temp; 382 383 fs_get_parent(fg, fte); 384 385 list_for_each_entry_safe(dst, temp, &fte->dests, base.list) { 386 fte->dests_size--; 387 list_del(&dst->base.list); 388 kfree(dst); 389 } 390 391 list_del(&fte->base.list); 392 fg->num_ftes--; 393 kfree(fte); 394} 395 396static struct mlx5_flow_group *fs_alloc_fg(u32 *create_fg_in) 397{ 398 struct mlx5_flow_group *fg; 399 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in, 400 create_fg_in, match_criteria); 401 u8 match_criteria_enable = MLX5_GET(create_flow_group_in, 402 create_fg_in, 403 match_criteria_enable); 404 fg = kzalloc(sizeof(*fg), GFP_KERNEL); 405 if (!fg) 406 return ERR_PTR(-ENOMEM); 407 408 INIT_LIST_HEAD(&fg->ftes); 409 fg->mask.match_criteria_enable = match_criteria_enable; 410 memcpy(&fg->mask.match_criteria, match_criteria, 411 sizeof(fg->mask.match_criteria)); 412 fg->base.type = FS_TYPE_FLOW_GROUP; 413 fg->start_index = MLX5_GET(create_flow_group_in, create_fg_in, 414 start_flow_index); 415 fg->max_ftes = MLX5_GET(create_flow_group_in, create_fg_in, 416 end_flow_index) - fg->start_index + 1; 417 return fg; 418} 419 420static struct mlx5_flow_table *find_next_ft(struct fs_prio *prio); 421static struct mlx5_flow_table *find_prev_ft(struct mlx5_flow_table *curr, 422 struct fs_prio *prio); 423 424/* assumed src_ft and dst_ft can't be freed */ 425static int fs_set_star_rule(struct mlx5_core_dev *dev, 426 struct mlx5_flow_table *src_ft, 427 struct mlx5_flow_table *dst_ft) 428{ 429 struct mlx5_flow_rule *src_dst; 430 struct fs_fte *src_fte; 431 int err = 0; 432 u32 *match_value; 433 int match_len = MLX5_ST_SZ_BYTES(fte_match_param); 434 435 src_dst = list_first_entry(&src_ft->star_rule.fte->dests, 436 struct mlx5_flow_rule, base.list); 437 match_value = mlx5_vzalloc(match_len); 438 if (!match_value) { 439 mlx5_core_warn(dev, "failed to allocate inbox\n"); 440 return -ENOMEM; 441 } 442 /*Create match context*/ 443 444 fs_get_parent(src_fte, src_dst); 445 446 src_dst->dest_attr.ft = dst_ft; 447 if (dst_ft) { 448 err = mlx5_cmd_fs_set_fte(dev, 449 src_ft->vport, 450 &src_fte->status, 451 match_value, src_ft->type, 452 src_ft->id, src_fte->index, 453 src_ft->star_rule.fg->id, 454 &src_fte->flow_act, 455 src_fte->sw_action, 456 src_fte->dests_size, 457 &src_fte->dests); 458 if (err) 459 goto free; 460 461 fs_get(&dst_ft->base); 462 } else { 463 mlx5_cmd_fs_delete_fte(dev, 464 src_ft->vport, 465 &src_fte->status, 466 src_ft->type, src_ft->id, 467 src_fte->index); 468 } 469 470free: 471 kvfree(match_value); 472 return err; 473} 474 475static int connect_prev_fts(struct fs_prio *locked_prio, 476 struct fs_prio *prev_prio, 477 struct mlx5_flow_table *next_ft) 478{ 479 struct mlx5_flow_table *iter; 480 int err = 0; 481 struct mlx5_core_dev *dev = fs_get_dev(&prev_prio->base); 482 483 if (!dev) 484 return -ENODEV; 485 486 mutex_lock(&prev_prio->base.lock); 487 fs_for_each_ft(iter, prev_prio) { 488 struct mlx5_flow_rule *src_dst = 489 list_first_entry(&iter->star_rule.fte->dests, 490 struct mlx5_flow_rule, base.list); 491 struct mlx5_flow_table *prev_ft = src_dst->dest_attr.ft; 492 493 if (prev_ft == next_ft) 494 continue; 495 496 err = fs_set_star_rule(dev, iter, next_ft); 497 if (err) { 498 mlx5_core_warn(dev, 499 "mlx5: flow steering can't connect prev and next\n"); 500 goto unlock; 501 } else { 502 /* Assume ft's prio is locked */ 503 if (prev_ft) { 504 struct fs_prio *prio; 505 506 fs_get_parent(prio, prev_ft); 507 if (prio == locked_prio) 508 fs_put_parent_locked(&prev_ft->base); 509 else 510 fs_put(&prev_ft->base); 511 } 512 } 513 } 514 515unlock: 516 mutex_unlock(&prev_prio->base.lock); 517 return 0; 518} 519 520static int create_star_rule(struct mlx5_flow_table *ft, struct fs_prio *prio) 521{ 522 struct mlx5_flow_group *fg; 523 int err; 524 u32 *fg_in; 525 u32 *match_value; 526 struct mlx5_flow_table *next_ft; 527 struct mlx5_flow_table *prev_ft; 528 struct mlx5_flow_root_namespace *root = find_root(&prio->base); 529 int fg_inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 530 int match_len = MLX5_ST_SZ_BYTES(fte_match_param); 531 532 fg_in = mlx5_vzalloc(fg_inlen); 533 if (!fg_in) { 534 mlx5_core_warn(root->dev, "failed to allocate inbox\n"); 535 return -ENOMEM; 536 } 537 538 match_value = mlx5_vzalloc(match_len); 539 if (!match_value) { 540 mlx5_core_warn(root->dev, "failed to allocate inbox\n"); 541 kvfree(fg_in); 542 return -ENOMEM; 543 } 544 545 MLX5_SET(create_flow_group_in, fg_in, start_flow_index, ft->max_fte); 546 MLX5_SET(create_flow_group_in, fg_in, end_flow_index, ft->max_fte); 547 fg = fs_alloc_fg(fg_in); 548 if (IS_ERR(fg)) { 549 err = PTR_ERR(fg); 550 goto out; 551 } 552 ft->star_rule.fg = fg; 553 err = mlx5_cmd_fs_create_fg(fs_get_dev(&prio->base), 554 fg_in, ft->vport, ft->type, 555 ft->id, 556 &fg->id); 557 if (err) 558 goto free_fg; 559 560 ft->star_rule.fte = alloc_star_ft_entry(ft, fg, 561 match_value, 562 ft->max_fte); 563 if (IS_ERR(ft->star_rule.fte)) 564 goto free_star_rule; 565 566 mutex_lock(&root->fs_chain_lock); 567 next_ft = find_next_ft(prio); 568 err = fs_set_star_rule(root->dev, ft, next_ft); 569 if (err) { 570 mutex_unlock(&root->fs_chain_lock); 571 goto free_star_rule; 572 } 573 if (next_ft) { 574 struct fs_prio *parent; 575 576 fs_get_parent(parent, next_ft); 577 fs_put(&next_ft->base); 578 } 579 prev_ft = find_prev_ft(ft, prio); 580 if (prev_ft) { 581 struct fs_prio *prev_parent; 582 583 fs_get_parent(prev_parent, prev_ft); 584 585 err = connect_prev_fts(NULL, prev_parent, ft); 586 if (err) { 587 mutex_unlock(&root->fs_chain_lock); 588 goto destroy_chained_star_rule; 589 } 590 fs_put(&prev_ft->base); 591 } 592 mutex_unlock(&root->fs_chain_lock); 593 kvfree(fg_in); 594 kvfree(match_value); 595 596 return 0; 597 598destroy_chained_star_rule: 599 fs_set_star_rule(fs_get_dev(&prio->base), ft, NULL); 600 if (next_ft) 601 fs_put(&next_ft->base); 602free_star_rule: 603 free_star_fte_entry(ft->star_rule.fte); 604 mlx5_cmd_fs_destroy_fg(fs_get_dev(&ft->base), ft->vport, 605 ft->type, ft->id, 606 fg->id); 607free_fg: 608 kfree(fg); 609out: 610 kvfree(fg_in); 611 kvfree(match_value); 612 return err; 613} 614 615static void destroy_star_rule(struct mlx5_flow_table *ft, struct fs_prio *prio) 616{ 617 int err; 618 struct mlx5_flow_root_namespace *root; 619 struct mlx5_core_dev *dev = fs_get_dev(&prio->base); 620 struct mlx5_flow_table *prev_ft, *next_ft; 621 struct fs_prio *prev_prio; 622 623 WARN_ON(!dev); 624 625 root = find_root(&prio->base); 626 if (!root) 627 mlx5_core_err(dev, 628 "flow steering failed to find root of priority %s", 629 prio->base.name); 630 631 /* In order to ensure atomic deletion, first update 632 * prev ft to point on the next ft. 633 */ 634 mutex_lock(&root->fs_chain_lock); 635 prev_ft = find_prev_ft(ft, prio); 636 next_ft = find_next_ft(prio); 637 if (prev_ft) { 638 fs_get_parent(prev_prio, prev_ft); 639 /*Prev is connected to ft, only if ft is the first(last) in the prio*/ 640 err = connect_prev_fts(prio, prev_prio, next_ft); 641 if (err) 642 mlx5_core_warn(root->dev, 643 "flow steering can't connect prev and next of flow table\n"); 644 fs_put(&prev_ft->base); 645 } 646 647 err = fs_set_star_rule(root->dev, ft, NULL); 648 /*One put is for fs_get in find next ft*/ 649 if (next_ft) { 650 fs_put(&next_ft->base); 651 if (!err) 652 fs_put(&next_ft->base); 653 } 654 655 mutex_unlock(&root->fs_chain_lock); 656 err = mlx5_cmd_fs_destroy_fg(dev, ft->vport, ft->type, ft->id, 657 ft->star_rule.fg->id); 658 if (err) 659 mlx5_core_warn(dev, 660 "flow steering can't destroy star entry group(index:%d) of ft:%s\n", ft->star_rule.fg->start_index, 661 ft->base.name); 662 free_star_fte_entry(ft->star_rule.fte); 663 664 kfree(ft->star_rule.fg); 665 ft->star_rule.fg = NULL; 666} 667 668static struct fs_prio *find_prio(struct mlx5_flow_namespace *ns, 669 unsigned int prio) 670{ 671 struct fs_prio *iter_prio; 672 673 fs_for_each_prio(iter_prio, ns) { 674 if (iter_prio->prio == prio) 675 return iter_prio; 676 } 677 678 return NULL; 679} 680 681static unsigned int _alloc_new_level(struct fs_prio *prio, 682 struct mlx5_flow_namespace *match); 683 684static unsigned int __alloc_new_level(struct mlx5_flow_namespace *ns, 685 struct fs_prio *prio) 686{ 687 unsigned int level = 0; 688 struct fs_prio *p; 689 690 if (!ns) 691 return 0; 692 693 mutex_lock(&ns->base.lock); 694 fs_for_each_prio(p, ns) { 695 if (p != prio) 696 level += p->max_ft; 697 else 698 break; 699 } 700 mutex_unlock(&ns->base.lock); 701 702 fs_get_parent(prio, ns); 703 if (prio) 704 WARN_ON(prio->base.type != FS_TYPE_PRIO); 705 706 return level + _alloc_new_level(prio, ns); 707} 708 709/* Called under lock of priority, hence locking all upper objects */ 710static unsigned int _alloc_new_level(struct fs_prio *prio, 711 struct mlx5_flow_namespace *match) 712{ 713 struct mlx5_flow_namespace *ns; 714 struct fs_base *it; 715 unsigned int level = 0; 716 717 if (!prio) 718 return 0; 719 720 mutex_lock(&prio->base.lock); 721 fs_for_each_ns_or_ft_reverse(it, prio) { 722 if (it->type == FS_TYPE_NAMESPACE) { 723 struct fs_prio *p; 724 725 fs_get_obj(ns, it); 726 727 if (match != ns) { 728 mutex_lock(&ns->base.lock); 729 fs_for_each_prio(p, ns) 730 level += p->max_ft; 731 mutex_unlock(&ns->base.lock); 732 } else { 733 break; 734 } 735 } else { 736 struct mlx5_flow_table *ft; 737 738 fs_get_obj(ft, it); 739 mutex_unlock(&prio->base.lock); 740 return level + ft->level + 1; 741 } 742 } 743 744 fs_get_parent(ns, prio); 745 mutex_unlock(&prio->base.lock); 746 return __alloc_new_level(ns, prio) + level; 747} 748 749static unsigned int alloc_new_level(struct fs_prio *prio) 750{ 751 return _alloc_new_level(prio, NULL); 752} 753 754static int update_root_ft_create(struct mlx5_flow_root_namespace *root, 755 struct mlx5_flow_table *ft) 756{ 757 int err = 0; 758 int min_level = INT_MAX; 759 760 if (root->root_ft) 761 min_level = root->root_ft->level; 762 763 if (ft->level < min_level) 764 err = mlx5_cmd_update_root_ft(root->dev, ft->type, 765 ft->id); 766 else 767 return err; 768 769 if (err) 770 mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n", 771 ft->id); 772 else 773 root->root_ft = ft; 774 775 return err; 776} 777 778static struct mlx5_flow_table *_create_ft_common(struct mlx5_flow_namespace *ns, 779 u16 vport, 780 struct fs_prio *fs_prio, 781 int max_fte, 782 const char *name) 783{ 784 struct mlx5_flow_table *ft; 785 int err; 786 int log_table_sz; 787 int ft_size; 788 char gen_name[20]; 789 struct mlx5_flow_root_namespace *root = find_root(&ns->base); 790 struct mlx5_core_dev *dev = fs_get_dev(&ns->base); 791 792 if (!root) { 793 mlx5_core_err(dev, 794 "flow steering failed to find root of namespace %s", 795 ns->base.name); 796 return ERR_PTR(-ENODEV); 797 } 798 799 if (fs_prio->num_ft == fs_prio->max_ft) 800 return ERR_PTR(-ENOSPC); 801 802 ft = kzalloc(sizeof(*ft), GFP_KERNEL); 803 if (!ft) 804 return ERR_PTR(-ENOMEM); 805 806 fs_init_node(&ft->base, 1); 807 INIT_LIST_HEAD(&ft->fgs); 808 809 /* Temporarily WA until we expose the level set in the API */ 810 if (root->table_type == FS_FT_ESW_EGRESS_ACL || 811 root->table_type == FS_FT_ESW_INGRESS_ACL) 812 ft->level = 0; 813 else 814 ft->level = alloc_new_level(fs_prio); 815 816 ft->base.type = FS_TYPE_FLOW_TABLE; 817 ft->vport = vport; 818 ft->type = root->table_type; 819 /*Two entries are reserved for star rules*/ 820 ft_size = roundup_pow_of_two(max_fte + 2); 821 /*User isn't aware to those rules*/ 822 ft->max_fte = ft_size - 2; 823 log_table_sz = ilog2(ft_size); 824 825 if (name == NULL || name[0] == '\0') { 826 snprintf(gen_name, sizeof(gen_name), "flow_table_%u", ft->id); 827 name = gen_name; 828 } 829 830 err = mlx5_cmd_fs_create_ft(root->dev, ft->vport, ft->type, 831 ft->level, log_table_sz, name, &ft->id); 832 if (err) 833 goto free_ft; 834 835 err = create_star_rule(ft, fs_prio); 836 if (err) 837 goto del_ft; 838 839 if ((root->table_type == FS_FT_NIC_RX) && MLX5_CAP_FLOWTABLE(root->dev, 840 flow_table_properties_nic_receive.modify_root)) { 841 err = update_root_ft_create(root, ft); 842 if (err) 843 goto destroy_star_rule; 844 } 845 846 _fs_add_node(&ft->base, name, &fs_prio->base); 847 848 list_add_tail(&ft->base.list, &fs_prio->objs); 849 fs_prio->num_ft++; 850 851 return ft; 852 853destroy_star_rule: 854 destroy_star_rule(ft, fs_prio); 855del_ft: 856 mlx5_cmd_fs_destroy_ft(root->dev, ft->vport, ft->type, ft->id); 857free_ft: 858 kfree(ft); 859 return ERR_PTR(err); 860} 861 862static struct mlx5_flow_table *create_ft_common(struct mlx5_flow_namespace *ns, 863 u16 vport, 864 unsigned int prio, 865 int max_fte, 866 const char *name) 867{ 868 struct fs_prio *fs_prio = NULL; 869 fs_prio = find_prio(ns, prio); 870 if (!fs_prio) 871 return ERR_PTR(-EINVAL); 872 873 return _create_ft_common(ns, vport, fs_prio, max_fte, name); 874} 875 876 877static struct mlx5_flow_table *find_first_ft_in_ns(struct mlx5_flow_namespace *ns, 878 struct list_head *start); 879 880static struct mlx5_flow_table *find_first_ft_in_prio(struct fs_prio *prio, 881 struct list_head *start); 882 883static struct mlx5_flow_table *mlx5_create_autogrouped_shared_flow_table(struct fs_prio *fs_prio) 884{ 885 struct mlx5_flow_table *ft; 886 887 ft = find_first_ft_in_prio(fs_prio, &fs_prio->objs); 888 if (ft) { 889 ft->shared_refcount++; 890 return ft; 891 } 892 893 return NULL; 894} 895 896struct mlx5_flow_table *mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns, 897 int prio, 898 const char *name, 899 int num_flow_table_entries, 900 int max_num_groups, 901 int num_reserved_entries) 902{ 903 struct mlx5_flow_table *ft = NULL; 904 struct fs_prio *fs_prio; 905 bool is_shared_prio; 906 907 if (max_num_groups > (num_flow_table_entries - num_reserved_entries)) 908 return ERR_PTR(-EINVAL); 909 if (num_reserved_entries > num_flow_table_entries) 910 return ERR_PTR(-EINVAL); 911 912 fs_prio = find_prio(ns, prio); 913 if (!fs_prio) 914 return ERR_PTR(-EINVAL); 915 916 is_shared_prio = fs_prio->flags & MLX5_CORE_FS_PRIO_SHARED; 917 if (is_shared_prio) { 918 mutex_lock(&fs_prio->shared_lock); 919 ft = mlx5_create_autogrouped_shared_flow_table(fs_prio); 920 } 921 922 if (ft) 923 goto return_ft; 924 925 ft = create_ft_common(ns, 0, prio, num_flow_table_entries, 926 name); 927 if (IS_ERR(ft)) 928 goto return_ft; 929 930 ft->autogroup.active = true; 931 ft->autogroup.max_types = max_num_groups; 932 ft->autogroup.max_fte = num_flow_table_entries - num_reserved_entries; 933 /* We save place for flow groups in addition to max types */ 934 ft->autogroup.group_size = ft->autogroup.max_fte / (max_num_groups + 1); 935 936 if (is_shared_prio) 937 ft->shared_refcount = 1; 938 939return_ft: 940 if (is_shared_prio) 941 mutex_unlock(&fs_prio->shared_lock); 942 return ft; 943} 944EXPORT_SYMBOL(mlx5_create_auto_grouped_flow_table); 945 946struct mlx5_flow_table *mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns, 947 u16 vport, 948 int prio, 949 const char *name, 950 int num_flow_table_entries) 951{ 952 return create_ft_common(ns, vport, prio, num_flow_table_entries, name); 953} 954EXPORT_SYMBOL(mlx5_create_vport_flow_table); 955 956struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns, 957 int prio, 958 const char *name, 959 int num_flow_table_entries) 960{ 961 return create_ft_common(ns, 0, prio, num_flow_table_entries, name); 962} 963EXPORT_SYMBOL(mlx5_create_flow_table); 964 965static void _fs_del_ft(struct mlx5_flow_table *ft) 966{ 967 int err; 968 struct mlx5_core_dev *dev = fs_get_dev(&ft->base); 969 struct fs_prio *prio; 970 971 err = mlx5_cmd_fs_destroy_ft(dev, ft->vport, ft->type, ft->id); 972 if (err) 973 mlx5_core_warn(dev, "flow steering can't destroy ft %s\n", 974 ft->base.name); 975 976 fs_get_parent(prio, ft); 977 prio->num_ft--; 978} 979 980static int update_root_ft_destroy(struct mlx5_flow_root_namespace *root, 981 struct mlx5_flow_table *ft) 982{ 983 int err = 0; 984 struct fs_prio *prio; 985 struct mlx5_flow_table *next_ft = NULL; 986 struct mlx5_flow_table *put_ft = NULL; 987 988 if (root->root_ft != ft) 989 return 0; 990 991 fs_get_parent(prio, ft); 992 /*Assuming objs containis only flow tables and 993 * flow tables are sorted by level. 994 */ 995 if (!list_is_last(&ft->base.list, &prio->objs)) { 996 next_ft = list_next_entry(ft, base.list); 997 } else { 998 next_ft = find_next_ft(prio); 999 put_ft = next_ft; 1000 } 1001 1002 if (next_ft) { 1003 err = mlx5_cmd_update_root_ft(root->dev, next_ft->type, 1004 next_ft->id); 1005 if (err) 1006 mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n", 1007 ft->id); 1008 } 1009 if (!err) 1010 root->root_ft = next_ft; 1011 1012 if (put_ft) 1013 fs_put(&put_ft->base); 1014 1015 return err; 1016} 1017 1018/*Objects in the same prio are destroyed in the reverse order they were createrd*/ 1019int mlx5_destroy_flow_table(struct mlx5_flow_table *ft) 1020{ 1021 int err = 0; 1022 struct fs_prio *prio; 1023 struct mlx5_flow_root_namespace *root; 1024 bool is_shared_prio; 1025 struct mlx5_core_dev *dev; 1026 1027 fs_get_parent(prio, ft); 1028 root = find_root(&prio->base); 1029 dev = fs_get_dev(&prio->base); 1030 1031 if (!root) { 1032 mlx5_core_err(dev, 1033 "flow steering failed to find root of priority %s", 1034 prio->base.name); 1035 return -ENODEV; 1036 } 1037 1038 is_shared_prio = prio->flags & MLX5_CORE_FS_PRIO_SHARED; 1039 if (is_shared_prio) { 1040 mutex_lock(&prio->shared_lock); 1041 if (ft->shared_refcount > 1) { 1042 --ft->shared_refcount; 1043 fs_put(&ft->base); 1044 mutex_unlock(&prio->shared_lock); 1045 return 0; 1046 } 1047 } 1048 1049 mutex_lock(&prio->base.lock); 1050 mutex_lock(&ft->base.lock); 1051 1052 err = update_root_ft_destroy(root, ft); 1053 if (err) 1054 goto unlock_ft; 1055 1056 /* delete two last entries */ 1057 destroy_star_rule(ft, prio); 1058 1059 mutex_unlock(&ft->base.lock); 1060 fs_remove_node_parent_locked(&ft->base); 1061 mutex_unlock(&prio->base.lock); 1062 if (is_shared_prio) 1063 mutex_unlock(&prio->shared_lock); 1064 1065 return err; 1066 1067unlock_ft: 1068 mutex_unlock(&ft->base.lock); 1069 mutex_unlock(&prio->base.lock); 1070 if (is_shared_prio) 1071 mutex_unlock(&prio->shared_lock); 1072 1073 return err; 1074} 1075EXPORT_SYMBOL(mlx5_destroy_flow_table); 1076 1077static struct mlx5_flow_group *fs_create_fg(struct mlx5_core_dev *dev, 1078 struct mlx5_flow_table *ft, 1079 struct list_head *prev, 1080 u32 *fg_in, 1081 int refcount) 1082{ 1083 struct mlx5_flow_group *fg; 1084 unsigned int group_size; 1085 int err; 1086 char name[20]; 1087 1088 fg = fs_alloc_fg(fg_in); 1089 if (IS_ERR(fg)) 1090 return fg; 1091 1092 group_size = MLX5_GET(create_flow_group_in, fg_in, end_flow_index) - 1093 MLX5_GET(create_flow_group_in, fg_in, start_flow_index) + 1; 1094 err = mlx5_cmd_fs_create_fg(dev, fg_in, 1095 ft->vport, ft->type, ft->id, 1096 &fg->id); 1097 if (err) 1098 goto free_fg; 1099 1100 mutex_lock(&ft->base.lock); 1101 1102 if (ft->autogroup.active && group_size == ft->autogroup.group_size) 1103 ft->autogroup.num_types++; 1104 1105 snprintf(name, sizeof(name), "group_%u", fg->id); 1106 /*Add node to tree*/ 1107 fs_add_node(&fg->base, &ft->base, name, refcount); 1108 /*Add node to group list*/ 1109 list_add(&fg->base.list, prev); 1110 mutex_unlock(&ft->base.lock); 1111 1112 return fg; 1113 1114free_fg: 1115 kfree(fg); 1116 return ERR_PTR(err); 1117} 1118 1119struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft, 1120 u32 *in) 1121{ 1122 struct mlx5_flow_group *fg; 1123 struct mlx5_core_dev *dev = fs_get_dev(&ft->base); 1124 unsigned int start_index; 1125 1126 start_index = MLX5_GET(create_flow_group_in, in, start_flow_index); 1127 if (!dev) 1128 return ERR_PTR(-ENODEV); 1129 1130 if (ft->autogroup.active && start_index < ft->autogroup.max_fte) 1131 return ERR_PTR(-EPERM); 1132 1133 fg = fs_create_fg(dev, ft, ft->fgs.prev, in, 1); 1134 1135 return fg; 1136} 1137EXPORT_SYMBOL(mlx5_create_flow_group); 1138 1139/*Group is destoyed when all the rules in the group were removed*/ 1140static void fs_del_fg(struct mlx5_flow_group *fg) 1141{ 1142 struct mlx5_flow_table *parent_ft; 1143 struct mlx5_core_dev *dev; 1144 1145 fs_get_parent(parent_ft, fg); 1146 dev = fs_get_dev(&parent_ft->base); 1147 WARN_ON(!dev); 1148 1149 if (parent_ft->autogroup.active && 1150 fg->max_ftes == parent_ft->autogroup.group_size && 1151 fg->start_index < parent_ft->autogroup.max_fte) 1152 parent_ft->autogroup.num_types--; 1153 1154 if (mlx5_cmd_fs_destroy_fg(dev, parent_ft->vport, 1155 parent_ft->type, 1156 parent_ft->id, fg->id)) 1157 mlx5_core_warn(dev, "flow steering can't destroy fg\n"); 1158} 1159 1160void mlx5_destroy_flow_group(struct mlx5_flow_group *fg) 1161{ 1162 fs_remove_node(&fg->base); 1163} 1164EXPORT_SYMBOL(mlx5_destroy_flow_group); 1165 1166static bool _fs_match_exact_val(void *mask, void *val1, void *val2, size_t size) 1167{ 1168 unsigned int i; 1169 1170 /* TODO: optimize by comparing 64bits when possible */ 1171 for (i = 0; i < size; i++, mask++, val1++, val2++) 1172 if ((*((u8 *)val1) & (*(u8 *)mask)) != 1173 ((*(u8 *)val2) & (*(u8 *)mask))) 1174 return false; 1175 1176 return true; 1177} 1178 1179bool fs_match_exact_val(struct mlx5_core_fs_mask *mask, 1180 void *val1, void *val2) 1181{ 1182 if (mask->match_criteria_enable & 1183 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS) { 1184 void *fte_match1 = MLX5_ADDR_OF(fte_match_param, 1185 val1, outer_headers); 1186 void *fte_match2 = MLX5_ADDR_OF(fte_match_param, 1187 val2, outer_headers); 1188 void *fte_mask = MLX5_ADDR_OF(fte_match_param, 1189 mask->match_criteria, outer_headers); 1190 1191 if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2, 1192 MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4))) 1193 return false; 1194 } 1195 1196 if (mask->match_criteria_enable & 1197 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS) { 1198 void *fte_match1 = MLX5_ADDR_OF(fte_match_param, 1199 val1, misc_parameters); 1200 void *fte_match2 = MLX5_ADDR_OF(fte_match_param, 1201 val2, misc_parameters); 1202 void *fte_mask = MLX5_ADDR_OF(fte_match_param, 1203 mask->match_criteria, misc_parameters); 1204 1205 if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2, 1206 MLX5_ST_SZ_BYTES(fte_match_set_misc))) 1207 return false; 1208 } 1209 if (mask->match_criteria_enable & 1210 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS) { 1211 void *fte_match1 = MLX5_ADDR_OF(fte_match_param, 1212 val1, inner_headers); 1213 void *fte_match2 = MLX5_ADDR_OF(fte_match_param, 1214 val2, inner_headers); 1215 void *fte_mask = MLX5_ADDR_OF(fte_match_param, 1216 mask->match_criteria, inner_headers); 1217 1218 if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2, 1219 MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4))) 1220 return false; 1221 } 1222 return true; 1223} 1224 1225bool fs_match_exact_mask(u8 match_criteria_enable1, 1226 u8 match_criteria_enable2, 1227 void *mask1, void *mask2) 1228{ 1229 return match_criteria_enable1 == match_criteria_enable2 && 1230 !memcmp(mask1, mask2, MLX5_ST_SZ_BYTES(fte_match_param)); 1231} 1232 1233static struct mlx5_flow_table *find_first_ft_in_ns_reverse(struct mlx5_flow_namespace *ns, 1234 struct list_head *start); 1235 1236static struct mlx5_flow_table *_find_first_ft_in_prio_reverse(struct fs_prio *prio, 1237 struct list_head *start) 1238{ 1239 struct fs_base *it = container_of(start, struct fs_base, list); 1240 1241 if (!prio) 1242 return NULL; 1243 1244 fs_for_each_ns_or_ft_continue_reverse(it, prio) { 1245 struct mlx5_flow_namespace *ns; 1246 struct mlx5_flow_table *ft; 1247 1248 if (it->type == FS_TYPE_FLOW_TABLE) { 1249 fs_get_obj(ft, it); 1250 fs_get(&ft->base); 1251 return ft; 1252 } 1253 1254 fs_get_obj(ns, it); 1255 WARN_ON(ns->base.type != FS_TYPE_NAMESPACE); 1256 1257 ft = find_first_ft_in_ns_reverse(ns, &ns->prios); 1258 if (ft) 1259 return ft; 1260 } 1261 1262 return NULL; 1263} 1264 1265static struct mlx5_flow_table *find_first_ft_in_prio_reverse(struct fs_prio *prio, 1266 struct list_head *start) 1267{ 1268 struct mlx5_flow_table *ft; 1269 1270 if (!prio) 1271 return NULL; 1272 1273 mutex_lock(&prio->base.lock); 1274 ft = _find_first_ft_in_prio_reverse(prio, start); 1275 mutex_unlock(&prio->base.lock); 1276 1277 return ft; 1278} 1279 1280static struct mlx5_flow_table *find_first_ft_in_ns_reverse(struct mlx5_flow_namespace *ns, 1281 struct list_head *start) 1282{ 1283 struct fs_prio *prio; 1284 1285 if (!ns) 1286 return NULL; 1287 1288 fs_get_obj(prio, container_of(start, struct fs_base, list)); 1289 mutex_lock(&ns->base.lock); 1290 fs_for_each_prio_continue_reverse(prio, ns) { 1291 struct mlx5_flow_table *ft; 1292 1293 ft = find_first_ft_in_prio_reverse(prio, &prio->objs); 1294 if (ft) { 1295 mutex_unlock(&ns->base.lock); 1296 return ft; 1297 } 1298 } 1299 mutex_unlock(&ns->base.lock); 1300 1301 return NULL; 1302} 1303 1304/* Returned a held ft, assumed curr is protected, assumed curr's parent is 1305 * locked 1306 */ 1307static struct mlx5_flow_table *find_prev_ft(struct mlx5_flow_table *curr, 1308 struct fs_prio *prio) 1309{ 1310 struct mlx5_flow_table *ft = NULL; 1311 struct fs_base *curr_base; 1312 1313 if (!curr) 1314 return NULL; 1315 1316 /* prio has either namespace or flow-tables, but not both */ 1317 if (!list_empty(&prio->objs) && 1318 list_first_entry(&prio->objs, struct mlx5_flow_table, base.list) != 1319 curr) 1320 return NULL; 1321 1322 while (!ft && prio) { 1323 struct mlx5_flow_namespace *ns; 1324 1325 fs_get_parent(ns, prio); 1326 ft = find_first_ft_in_ns_reverse(ns, &prio->base.list); 1327 curr_base = &ns->base; 1328 fs_get_parent(prio, ns); 1329 1330 if (prio && !ft) 1331 ft = find_first_ft_in_prio_reverse(prio, 1332 &curr_base->list); 1333 } 1334 return ft; 1335} 1336 1337static struct mlx5_flow_table *_find_first_ft_in_prio(struct fs_prio *prio, 1338 struct list_head *start) 1339{ 1340 struct fs_base *it = container_of(start, struct fs_base, list); 1341 1342 if (!prio) 1343 return NULL; 1344 1345 fs_for_each_ns_or_ft_continue(it, prio) { 1346 struct mlx5_flow_namespace *ns; 1347 struct mlx5_flow_table *ft; 1348 1349 if (it->type == FS_TYPE_FLOW_TABLE) { 1350 fs_get_obj(ft, it); 1351 fs_get(&ft->base); 1352 return ft; 1353 } 1354 1355 fs_get_obj(ns, it); 1356 WARN_ON(ns->base.type != FS_TYPE_NAMESPACE); 1357 1358 ft = find_first_ft_in_ns(ns, &ns->prios); 1359 if (ft) 1360 return ft; 1361 } 1362 1363 return NULL; 1364} 1365 1366static struct mlx5_flow_table *find_first_ft_in_prio(struct fs_prio *prio, 1367 struct list_head *start) 1368{ 1369 struct mlx5_flow_table *ft; 1370 1371 if (!prio) 1372 return NULL; 1373 1374 mutex_lock(&prio->base.lock); 1375 ft = _find_first_ft_in_prio(prio, start); 1376 mutex_unlock(&prio->base.lock); 1377 1378 return ft; 1379} 1380 1381static struct mlx5_flow_table *find_first_ft_in_ns(struct mlx5_flow_namespace *ns, 1382 struct list_head *start) 1383{ 1384 struct fs_prio *prio; 1385 1386 if (!ns) 1387 return NULL; 1388 1389 fs_get_obj(prio, container_of(start, struct fs_base, list)); 1390 mutex_lock(&ns->base.lock); 1391 fs_for_each_prio_continue(prio, ns) { 1392 struct mlx5_flow_table *ft; 1393 1394 ft = find_first_ft_in_prio(prio, &prio->objs); 1395 if (ft) { 1396 mutex_unlock(&ns->base.lock); 1397 return ft; 1398 } 1399 } 1400 mutex_unlock(&ns->base.lock); 1401 1402 return NULL; 1403} 1404 1405/* returned a held ft, assumed curr is protected, assumed curr's parent is 1406 * locked 1407 */ 1408static struct mlx5_flow_table *find_next_ft(struct fs_prio *prio) 1409{ 1410 struct mlx5_flow_table *ft = NULL; 1411 struct fs_base *curr_base; 1412 1413 while (!ft && prio) { 1414 struct mlx5_flow_namespace *ns; 1415 1416 fs_get_parent(ns, prio); 1417 ft = find_first_ft_in_ns(ns, &prio->base.list); 1418 curr_base = &ns->base; 1419 fs_get_parent(prio, ns); 1420 1421 if (!ft && prio) 1422 ft = _find_first_ft_in_prio(prio, &curr_base->list); 1423 } 1424 return ft; 1425} 1426 1427 1428/* called under ft mutex lock */ 1429static struct mlx5_flow_group *create_autogroup(struct mlx5_flow_table *ft, 1430 u8 match_criteria_enable, 1431 u32 *match_criteria) 1432{ 1433 unsigned int group_size; 1434 unsigned int candidate_index = 0; 1435 struct mlx5_flow_group *g; 1436 struct mlx5_flow_group *ret; 1437 struct list_head *prev = &ft->fgs; 1438 struct mlx5_core_dev *dev; 1439 u32 *in; 1440 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1441 void *match_criteria_addr; 1442 u32 max_fte = ft->autogroup.max_fte; 1443 1444 if (!ft->autogroup.active) 1445 return ERR_PTR(-ENOENT); 1446 1447 dev = fs_get_dev(&ft->base); 1448 if (!dev) 1449 return ERR_PTR(-ENODEV); 1450 1451 in = mlx5_vzalloc(inlen); 1452 if (!in) { 1453 mlx5_core_warn(dev, "failed to allocate inbox\n"); 1454 return ERR_PTR(-ENOMEM); 1455 } 1456 1457 1458 if (ft->autogroup.num_types < ft->autogroup.max_types) 1459 group_size = ft->autogroup.group_size; 1460 else 1461 group_size = 1; 1462 1463 if (group_size == 0) { 1464 mlx5_core_warn(dev, 1465 "flow steering can't create group size of 0\n"); 1466 ret = ERR_PTR(-EINVAL); 1467 goto out; 1468 } 1469 1470 /* sorted by start_index */ 1471 fs_for_each_fg(g, ft) { 1472 if (candidate_index + group_size > g->start_index) 1473 candidate_index = g->start_index + g->max_ftes; 1474 else 1475 break; 1476 prev = &g->base.list; 1477 } 1478 1479 if (candidate_index + group_size > max_fte) { 1480 ret = ERR_PTR(-ENOSPC); 1481 goto out; 1482 } 1483 1484 MLX5_SET(create_flow_group_in, in, match_criteria_enable, 1485 match_criteria_enable); 1486 MLX5_SET(create_flow_group_in, in, start_flow_index, candidate_index); 1487 MLX5_SET(create_flow_group_in, in, end_flow_index, candidate_index + 1488 group_size - 1); 1489 match_criteria_addr = MLX5_ADDR_OF(create_flow_group_in, 1490 in, match_criteria); 1491 memcpy(match_criteria_addr, match_criteria, 1492 MLX5_ST_SZ_BYTES(fte_match_param)); 1493 1494 ret = fs_create_fg(dev, ft, prev, in, 0); 1495out: 1496 kvfree(in); 1497 return ret; 1498} 1499 1500static struct mlx5_flow_namespace *get_ns_with_notifiers(struct fs_base *node) 1501{ 1502 struct mlx5_flow_namespace *ns = NULL; 1503 1504 while (node && (node->type != FS_TYPE_NAMESPACE || 1505 list_empty(&container_of(node, struct 1506 mlx5_flow_namespace, 1507 base)->list_notifiers))) 1508 node = node->parent; 1509 1510 if (node) 1511 fs_get_obj(ns, node); 1512 1513 return ns; 1514} 1515 1516 1517/*Assumption- fte is locked*/ 1518static void call_to_add_rule_notifiers(struct mlx5_flow_rule *dst, 1519 struct fs_fte *fte) 1520{ 1521 struct mlx5_flow_namespace *ns; 1522 struct mlx5_flow_handler *iter_handler; 1523 struct fs_client_priv_data *iter_client; 1524 void *data; 1525 bool is_new_rule = list_first_entry(&fte->dests, 1526 struct mlx5_flow_rule, 1527 base.list) == dst; 1528 int err; 1529 1530 ns = get_ns_with_notifiers(&fte->base); 1531 if (!ns) 1532 return; 1533 1534 down_read(&ns->notifiers_rw_sem); 1535 list_for_each_entry(iter_handler, &ns->list_notifiers, 1536 list) { 1537 if (iter_handler->add_dst_cb) { 1538 data = NULL; 1539 mutex_lock(&dst->clients_lock); 1540 list_for_each_entry( 1541 iter_client, &dst->clients_data, list) { 1542 if (iter_client->fs_handler == iter_handler) { 1543 data = iter_client->client_dst_data; 1544 break; 1545 } 1546 } 1547 mutex_unlock(&dst->clients_lock); 1548 err = iter_handler->add_dst_cb(dst, 1549 is_new_rule, 1550 data, 1551 iter_handler->client_context); 1552 if (err) 1553 break; 1554 } 1555 } 1556 up_read(&ns->notifiers_rw_sem); 1557} 1558 1559static void call_to_del_rule_notifiers(struct mlx5_flow_rule *dst, 1560 struct fs_fte *fte) 1561{ 1562 struct mlx5_flow_namespace *ns; 1563 struct mlx5_flow_handler *iter_handler; 1564 struct fs_client_priv_data *iter_client; 1565 void *data; 1566 bool ctx_changed = (fte->dests_size == 0); 1567 1568 ns = get_ns_with_notifiers(&fte->base); 1569 if (!ns) 1570 return; 1571 down_read(&ns->notifiers_rw_sem); 1572 list_for_each_entry(iter_handler, &ns->list_notifiers, 1573 list) { 1574 data = NULL; 1575 mutex_lock(&dst->clients_lock); 1576 list_for_each_entry(iter_client, &dst->clients_data, list) { 1577 if (iter_client->fs_handler == iter_handler) { 1578 data = iter_client->client_dst_data; 1579 break; 1580 } 1581 } 1582 mutex_unlock(&dst->clients_lock); 1583 if (iter_handler->del_dst_cb) { 1584 iter_handler->del_dst_cb(dst, ctx_changed, data, 1585 iter_handler->client_context); 1586 } 1587 } 1588 up_read(&ns->notifiers_rw_sem); 1589} 1590 1591/* fte should not be deleted while calling this function */ 1592static struct mlx5_flow_rule *_fs_add_dst_fte(struct fs_fte *fte, 1593 struct mlx5_flow_group *fg, 1594 struct mlx5_flow_destination *dest) 1595{ 1596 struct mlx5_flow_table *ft; 1597 struct mlx5_flow_rule *dst; 1598 int err; 1599 1600 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 1601 if (!dst) 1602 return ERR_PTR(-ENOMEM); 1603 1604 memcpy(&dst->dest_attr, dest, sizeof(*dest)); 1605 dst->base.type = FS_TYPE_FLOW_DEST; 1606 INIT_LIST_HEAD(&dst->clients_data); 1607 mutex_init(&dst->clients_lock); 1608 fs_get_parent(ft, fg); 1609 /*Add dest to dests list- added as first element after the head*/ 1610 list_add_tail(&dst->base.list, &fte->dests); 1611 fte->dests_size++; 1612 err = mlx5_cmd_fs_set_fte(fs_get_dev(&ft->base), 1613 ft->vport, 1614 &fte->status, 1615 fte->val, ft->type, 1616 ft->id, fte->index, fg->id, &fte->flow_act, 1617 fte->sw_action, fte->dests_size, &fte->dests); 1618 if (err) 1619 goto free_dst; 1620 1621 list_del(&dst->base.list); 1622 1623 return dst; 1624 1625free_dst: 1626 list_del(&dst->base.list); 1627 kfree(dst); 1628 fte->dests_size--; 1629 return ERR_PTR(err); 1630} 1631 1632static char *get_dest_name(struct mlx5_flow_destination *dest) 1633{ 1634 char *name = kzalloc(sizeof(char) * 20, GFP_KERNEL); 1635 1636 switch (dest->type) { 1637 case MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE: 1638 snprintf(name, 20, "dest_%s_%u", "flow_table", 1639 dest->ft->id); 1640 return name; 1641 case MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT: 1642 snprintf(name, 20, "dest_%s_%u", "vport", 1643 dest->vport_num); 1644 return name; 1645 case MLX5_FLOW_CONTEXT_DEST_TYPE_TIR: 1646 snprintf(name, 20, "dest_%s_%u", "tir", dest->tir_num); 1647 return name; 1648 default: 1649 kfree(name); 1650 return NULL; 1651 } 1652} 1653 1654/* assumed fg is locked */ 1655static unsigned int fs_get_free_fg_index(struct mlx5_flow_group *fg, 1656 struct list_head **prev) 1657{ 1658 struct fs_fte *fte; 1659 unsigned int start = fg->start_index; 1660 1661 if (prev) 1662 *prev = &fg->ftes; 1663 1664 /* assumed list is sorted by index */ 1665 fs_for_each_fte(fte, fg) { 1666 if (fte->index != start) 1667 return start; 1668 start++; 1669 if (prev) 1670 *prev = &fte->base.list; 1671 } 1672 1673 return start; 1674} 1675 1676 1677static struct fs_fte *fs_create_fte(struct mlx5_flow_group *fg, 1678 u32 *match_value, 1679 u32 sw_action, 1680 struct mlx5_flow_act *flow_act, 1681 struct list_head **prev) 1682{ 1683 struct fs_fte *fte; 1684 int index = 0; 1685 1686 index = fs_get_free_fg_index(fg, prev); 1687 fte = fs_alloc_fte(sw_action, flow_act, match_value, index); 1688 if (IS_ERR(fte)) 1689 return fte; 1690 1691 return fte; 1692} 1693 1694static void add_rule_to_tree(struct mlx5_flow_rule *rule, 1695 struct fs_fte *fte) 1696{ 1697 char *dest_name; 1698 1699 dest_name = get_dest_name(&rule->dest_attr); 1700 fs_add_node(&rule->base, &fte->base, dest_name, 1); 1701 /* re-add to list, since fs_add_node reset our list */ 1702 list_add_tail(&rule->base.list, &fte->dests); 1703 kfree(dest_name); 1704 call_to_add_rule_notifiers(rule, fte); 1705} 1706 1707static void fs_del_dst(struct mlx5_flow_rule *dst) 1708{ 1709 struct mlx5_flow_table *ft; 1710 struct mlx5_flow_group *fg; 1711 struct fs_fte *fte; 1712 u32 *match_value; 1713 struct mlx5_core_dev *dev = fs_get_dev(&dst->base); 1714 int match_len = MLX5_ST_SZ_BYTES(fte_match_param); 1715 int err; 1716 1717 WARN_ON(!dev); 1718 1719 match_value = mlx5_vzalloc(match_len); 1720 if (!match_value) { 1721 mlx5_core_warn(dev, "failed to allocate inbox\n"); 1722 return; 1723 } 1724 1725 fs_get_parent(fte, dst); 1726 fs_get_parent(fg, fte); 1727 sx_assert(&fg->base.lock.sx, SX_XLOCKED); 1728 memcpy(match_value, fte->val, sizeof(fte->val)); 1729 /* ft can't be changed as fg is locked */ 1730 fs_get_parent(ft, fg); 1731 list_del(&dst->base.list); 1732 fte->dests_size--; 1733 if (fte->dests_size) { 1734 err = mlx5_cmd_fs_set_fte(dev, ft->vport, 1735 &fte->status, match_value, ft->type, 1736 ft->id, fte->index, fg->id, 1737 &fte->flow_act, fte->sw_action, 1738 fte->dests_size, &fte->dests); 1739 if (err) { 1740 mlx5_core_warn(dev, "%s can't delete dst %s\n", 1741 __func__, dst->base.name); 1742 goto err; 1743 } 1744 } 1745 call_to_del_rule_notifiers(dst, fte); 1746err: 1747 kvfree(match_value); 1748} 1749 1750static void fs_del_fte(struct fs_fte *fte) 1751{ 1752 struct mlx5_flow_table *ft; 1753 struct mlx5_flow_group *fg; 1754 int err; 1755 struct mlx5_core_dev *dev; 1756 1757 fs_get_parent(fg, fte); 1758 fs_get_parent(ft, fg); 1759 1760 dev = fs_get_dev(&ft->base); 1761 WARN_ON(!dev); 1762 1763 err = mlx5_cmd_fs_delete_fte(dev, ft->vport, &fte->status, 1764 ft->type, ft->id, fte->index); 1765 if (err) 1766 mlx5_core_warn(dev, "flow steering can't delete fte %s\n", 1767 fte->base.name); 1768 1769 fg->num_ftes--; 1770} 1771 1772static bool check_conflicting_actions(const struct mlx5_flow_act *act1, 1773 const struct mlx5_flow_act *act2) 1774{ 1775 u32 action1 = act1->actions; 1776 u32 action2 = act2->actions; 1777 u32 xored_actions; 1778 1779 xored_actions = action1 ^ action2; 1780 1781 if (xored_actions & (MLX5_FLOW_ACT_ACTIONS_FLOW_TAG)) 1782 return true; 1783 1784 if (action1 & MLX5_FLOW_ACT_ACTIONS_FLOW_TAG && 1785 act1->flow_tag != act2->flow_tag) 1786 return true; 1787 1788 /* Can even have complex actions in merged rules */ 1789 if (action1 & MLX5_FLOW_ACT_ACTIONS_MODIFY_HDR) 1790 return true; 1791 1792 if (action1 & MLX5_FLOW_ACT_ACTIONS_PACKET_REFORMAT) 1793 return true; 1794 1795 if (action1 & MLX5_FLOW_ACT_ACTIONS_COUNT) 1796 return true; 1797 1798 return false; 1799} 1800 1801/* assuming parent fg is locked */ 1802/* Add dst algorithm */ 1803static struct mlx5_flow_rule *fs_add_dst_fg(struct mlx5_flow_group *fg, 1804 u32 *match_value, 1805 u32 sw_action, 1806 struct mlx5_flow_act *flow_act, 1807 struct mlx5_flow_destination *dest) 1808{ 1809 struct fs_fte *fte; 1810 struct mlx5_flow_rule *dst; 1811 struct mlx5_flow_table *ft; 1812 struct list_head *prev; 1813 char fte_name[20]; 1814 1815 mutex_lock(&fg->base.lock); 1816 if (flow_act->flags & MLX5_FLOW_ACT_NO_APPEND) 1817 goto insert_fte; 1818 1819 fs_for_each_fte(fte, fg) { 1820 /* TODO: Check of size against PRM max size */ 1821 mutex_lock(&fte->base.lock); 1822 if (fs_match_exact_val(&fg->mask, match_value, &fte->val) && 1823 sw_action == fte->sw_action && 1824 !check_conflicting_actions(flow_act, &fte->flow_act)) { 1825 dst = _fs_add_dst_fte(fte, fg, dest); 1826 mutex_unlock(&fte->base.lock); 1827 if (IS_ERR(dst)) 1828 goto unlock_fg; 1829 goto add_rule; 1830 } 1831 mutex_unlock(&fte->base.lock); 1832 } 1833 1834insert_fte: 1835 fs_get_parent(ft, fg); 1836 if (fg->num_ftes == fg->max_ftes) { 1837 dst = ERR_PTR(-ENOSPC); 1838 goto unlock_fg; 1839 } 1840 1841 fte = fs_create_fte(fg, match_value, sw_action, flow_act, &prev); 1842 if (IS_ERR(fte)) { 1843 dst = (void *)fte; 1844 goto unlock_fg; 1845 } 1846 dst = _fs_add_dst_fte(fte, fg, dest); 1847 if (IS_ERR(dst)) { 1848 kfree(fte); 1849 goto unlock_fg; 1850 } 1851 1852 fg->num_ftes++; 1853 1854 snprintf(fte_name, sizeof(fte_name), "fte%u", fte->index); 1855 /* Add node to tree */ 1856 fs_add_node(&fte->base, &fg->base, fte_name, 0); 1857 list_add(&fte->base.list, prev); 1858add_rule: 1859 add_rule_to_tree(dst, fte); 1860unlock_fg: 1861 mutex_unlock(&fg->base.lock); 1862 return dst; 1863} 1864 1865static struct mlx5_flow_rule *fs_add_dst_ft(struct mlx5_flow_table *ft, 1866 u8 match_criteria_enable, 1867 u32 *match_criteria, 1868 u32 *match_value, 1869 u32 sw_action, 1870 struct mlx5_flow_act *flow_act, 1871 struct mlx5_flow_destination *dest) 1872{ 1873 /*? where dst_entry is allocated*/ 1874 struct mlx5_flow_group *g; 1875 struct mlx5_flow_rule *dst; 1876 1877 fs_get(&ft->base); 1878 mutex_lock(&ft->base.lock); 1879 fs_for_each_fg(g, ft) 1880 if (fs_match_exact_mask(g->mask.match_criteria_enable, 1881 match_criteria_enable, 1882 g->mask.match_criteria, 1883 match_criteria)) { 1884 mutex_unlock(&ft->base.lock); 1885 1886 dst = fs_add_dst_fg(g, match_value, sw_action, flow_act, dest); 1887 if (PTR_ERR(dst) && PTR_ERR(dst) != -ENOSPC) 1888 goto unlock; 1889 } 1890 mutex_unlock(&ft->base.lock); 1891 1892 g = create_autogroup(ft, match_criteria_enable, match_criteria); 1893 if (IS_ERR(g)) { 1894 dst = (void *)g; 1895 goto unlock; 1896 } 1897 1898 dst = fs_add_dst_fg(g, match_value, 1899 sw_action, flow_act, dest); 1900 if (IS_ERR(dst)) { 1901 /* Remove assumes refcount > 0 and autogroup creates a group 1902 * with a refcount = 0. 1903 */ 1904 fs_get(&g->base); 1905 fs_remove_node(&g->base); 1906 goto unlock; 1907 } 1908 1909unlock: 1910 fs_put(&ft->base); 1911 return dst; 1912} 1913 1914struct mlx5_flow_rule * 1915mlx5_add_flow_rule(struct mlx5_flow_table *ft, 1916 u8 match_criteria_enable, 1917 u32 *match_criteria, 1918 u32 *match_value, 1919 u32 sw_action, 1920 struct mlx5_flow_act *flow_act, 1921 struct mlx5_flow_destination *dest) 1922{ 1923 struct mlx5_flow_rule *dst; 1924 struct mlx5_flow_namespace *ns; 1925 1926 ns = get_ns_with_notifiers(&ft->base); 1927 if (ns) 1928 down_read(&ns->dests_rw_sem); 1929 dst = fs_add_dst_ft(ft, match_criteria_enable, match_criteria, 1930 match_value, sw_action, flow_act, dest); 1931 if (ns) 1932 up_read(&ns->dests_rw_sem); 1933 1934 return dst; 1935 1936 1937} 1938EXPORT_SYMBOL(mlx5_add_flow_rule); 1939 1940void mlx5_del_flow_rule(struct mlx5_flow_rule **pp) 1941{ 1942 struct mlx5_flow_namespace *ns; 1943 struct mlx5_flow_rule *dst; 1944 1945 dst = *pp; 1946 *pp = NULL; 1947 1948 if (IS_ERR_OR_NULL(dst)) 1949 return; 1950 ns = get_ns_with_notifiers(&dst->base); 1951 if (ns) 1952 down_read(&ns->dests_rw_sem); 1953 fs_remove_node(&dst->base); 1954 if (ns) 1955 up_read(&ns->dests_rw_sem); 1956} 1957EXPORT_SYMBOL(mlx5_del_flow_rule); 1958 1959#define MLX5_CORE_FS_ROOT_NS_NAME "root" 1960#define MLX5_CORE_FS_ESW_EGRESS_ACL "esw_egress_root" 1961#define MLX5_CORE_FS_ESW_INGRESS_ACL "esw_ingress_root" 1962#define MLX5_CORE_FS_FDB_ROOT_NS_NAME "fdb_root" 1963#define MLX5_CORE_FS_SNIFFER_RX_ROOT_NS_NAME "sniffer_rx_root" 1964#define MLX5_CORE_FS_SNIFFER_TX_ROOT_NS_NAME "sniffer_tx_root" 1965#define MLX5_CORE_FS_PRIO_MAX_FT 4 1966#define MLX5_CORE_FS_PRIO_MAX_NS 1 1967 1968static struct fs_prio *fs_create_prio(struct mlx5_flow_namespace *ns, 1969 unsigned prio, int max_ft, 1970 const char *name, u8 flags) 1971{ 1972 struct fs_prio *fs_prio; 1973 1974 fs_prio = kzalloc(sizeof(*fs_prio), GFP_KERNEL); 1975 if (!fs_prio) 1976 return ERR_PTR(-ENOMEM); 1977 1978 fs_prio->base.type = FS_TYPE_PRIO; 1979 fs_add_node(&fs_prio->base, &ns->base, name, 1); 1980 fs_prio->max_ft = max_ft; 1981 fs_prio->max_ns = MLX5_CORE_FS_PRIO_MAX_NS; 1982 fs_prio->prio = prio; 1983 fs_prio->flags = flags; 1984 list_add_tail(&fs_prio->base.list, &ns->prios); 1985 INIT_LIST_HEAD(&fs_prio->objs); 1986 mutex_init(&fs_prio->shared_lock); 1987 1988 return fs_prio; 1989} 1990 1991static void cleanup_root_ns(struct mlx5_core_dev *dev) 1992{ 1993 struct mlx5_flow_root_namespace *root_ns = dev->root_ns; 1994 struct fs_prio *iter_prio; 1995 1996 if (!root_ns) 1997 return; 1998 1999 /* stage 1 */ 2000 fs_for_each_prio(iter_prio, &root_ns->ns) { 2001 struct mlx5_flow_namespace *iter_ns; 2002 2003 fs_for_each_ns(iter_ns, iter_prio) { 2004 while (!list_empty(&iter_ns->prios)) { 2005 struct fs_base *iter_prio2 = 2006 list_first_entry(&iter_ns->prios, 2007 struct fs_base, 2008 list); 2009 2010 fs_remove_node(iter_prio2); 2011 } 2012 } 2013 } 2014 2015 /* stage 2 */ 2016 fs_for_each_prio(iter_prio, &root_ns->ns) { 2017 while (!list_empty(&iter_prio->objs)) { 2018 struct fs_base *iter_ns = 2019 list_first_entry(&iter_prio->objs, 2020 struct fs_base, 2021 list); 2022 2023 fs_remove_node(iter_ns); 2024 } 2025 } 2026 /* stage 3 */ 2027 while (!list_empty(&root_ns->ns.prios)) { 2028 struct fs_base *iter_prio = 2029 list_first_entry(&root_ns->ns.prios, 2030 struct fs_base, 2031 list); 2032 2033 fs_remove_node(iter_prio); 2034 } 2035 2036 fs_remove_node(&root_ns->ns.base); 2037 dev->root_ns = NULL; 2038} 2039 2040static void cleanup_single_prio_root_ns(struct mlx5_core_dev *dev, 2041 struct mlx5_flow_root_namespace *root_ns) 2042{ 2043 struct fs_base *prio; 2044 2045 if (!root_ns) 2046 return; 2047 2048 if (!list_empty(&root_ns->ns.prios)) { 2049 prio = list_first_entry(&root_ns->ns.prios, 2050 struct fs_base, 2051 list); 2052 fs_remove_node(prio); 2053 } 2054 fs_remove_node(&root_ns->ns.base); 2055 root_ns = NULL; 2056} 2057 2058void mlx5_cleanup_fs(struct mlx5_core_dev *dev) 2059{ 2060 mlx5_cleanup_fc_stats(dev); 2061 cleanup_root_ns(dev); 2062 cleanup_single_prio_root_ns(dev, dev->sniffer_rx_root_ns); 2063 cleanup_single_prio_root_ns(dev, dev->sniffer_tx_root_ns); 2064 cleanup_single_prio_root_ns(dev, dev->fdb_root_ns); 2065 cleanup_single_prio_root_ns(dev, dev->esw_egress_root_ns); 2066 cleanup_single_prio_root_ns(dev, dev->esw_ingress_root_ns); 2067} 2068 2069static struct mlx5_flow_namespace *fs_init_namespace(struct mlx5_flow_namespace 2070 *ns) 2071{ 2072 ns->base.type = FS_TYPE_NAMESPACE; 2073 init_rwsem(&ns->dests_rw_sem); 2074 init_rwsem(&ns->notifiers_rw_sem); 2075 INIT_LIST_HEAD(&ns->prios); 2076 INIT_LIST_HEAD(&ns->list_notifiers); 2077 2078 return ns; 2079} 2080 2081static struct mlx5_flow_root_namespace *create_root_ns(struct mlx5_core_dev *dev, 2082 enum fs_ft_type 2083 table_type, 2084 char *name) 2085{ 2086 struct mlx5_flow_root_namespace *root_ns; 2087 struct mlx5_flow_namespace *ns; 2088 2089 /* create the root namespace */ 2090 root_ns = mlx5_vzalloc(sizeof(*root_ns)); 2091 if (!root_ns) 2092 goto err; 2093 2094 root_ns->dev = dev; 2095 root_ns->table_type = table_type; 2096 mutex_init(&root_ns->fs_chain_lock); 2097 2098 ns = &root_ns->ns; 2099 fs_init_namespace(ns); 2100 fs_add_node(&ns->base, NULL, name, 1); 2101 2102 return root_ns; 2103err: 2104 return NULL; 2105} 2106 2107static int init_fdb_root_ns(struct mlx5_core_dev *dev) 2108{ 2109 struct fs_prio *prio; 2110 2111 dev->fdb_root_ns = create_root_ns(dev, FS_FT_FDB, 2112 MLX5_CORE_FS_FDB_ROOT_NS_NAME); 2113 if (!dev->fdb_root_ns) 2114 return -ENOMEM; 2115 2116 /* create 1 prio*/ 2117 prio = fs_create_prio(&dev->fdb_root_ns->ns, 0, 1, "fdb_prio", 0); 2118 if (IS_ERR(prio)) 2119 return PTR_ERR(prio); 2120 else 2121 return 0; 2122} 2123 2124#define MAX_VPORTS 128 2125 2126static int init_egress_acl_root_ns(struct mlx5_core_dev *dev) 2127{ 2128 struct fs_prio *prio; 2129 2130 dev->esw_egress_root_ns = create_root_ns(dev, FS_FT_ESW_EGRESS_ACL, 2131 MLX5_CORE_FS_ESW_EGRESS_ACL); 2132 if (!dev->esw_egress_root_ns) 2133 return -ENOMEM; 2134 2135 /* create 1 prio*/ 2136 prio = fs_create_prio(&dev->esw_egress_root_ns->ns, 0, MAX_VPORTS, 2137 "esw_egress_prio", 0); 2138 if (IS_ERR(prio)) 2139 return PTR_ERR(prio); 2140 else 2141 return 0; 2142} 2143 2144static int init_ingress_acl_root_ns(struct mlx5_core_dev *dev) 2145{ 2146 struct fs_prio *prio; 2147 2148 dev->esw_ingress_root_ns = create_root_ns(dev, FS_FT_ESW_INGRESS_ACL, 2149 MLX5_CORE_FS_ESW_INGRESS_ACL); 2150 if (!dev->esw_ingress_root_ns) 2151 return -ENOMEM; 2152 2153 /* create 1 prio*/ 2154 prio = fs_create_prio(&dev->esw_ingress_root_ns->ns, 0, MAX_VPORTS, 2155 "esw_ingress_prio", 0); 2156 if (IS_ERR(prio)) 2157 return PTR_ERR(prio); 2158 else 2159 return 0; 2160} 2161 2162static int init_sniffer_rx_root_ns(struct mlx5_core_dev *dev) 2163{ 2164 struct fs_prio *prio; 2165 2166 dev->sniffer_rx_root_ns = create_root_ns(dev, FS_FT_SNIFFER_RX, 2167 MLX5_CORE_FS_SNIFFER_RX_ROOT_NS_NAME); 2168 if (!dev->sniffer_rx_root_ns) 2169 return -ENOMEM; 2170 2171 /* create 1 prio*/ 2172 prio = fs_create_prio(&dev->sniffer_rx_root_ns->ns, 0, 1, 2173 "sniffer_prio", 0); 2174 if (IS_ERR(prio)) 2175 return PTR_ERR(prio); 2176 else 2177 return 0; 2178} 2179 2180 2181static int init_sniffer_tx_root_ns(struct mlx5_core_dev *dev) 2182{ 2183 struct fs_prio *prio; 2184 2185 dev->sniffer_tx_root_ns = create_root_ns(dev, FS_FT_SNIFFER_TX, 2186 MLX5_CORE_FS_SNIFFER_TX_ROOT_NS_NAME); 2187 if (!dev->sniffer_tx_root_ns) 2188 return -ENOMEM; 2189 2190 /* create 1 prio*/ 2191 prio = fs_create_prio(&dev->sniffer_tx_root_ns->ns, 0, 1, 2192 "sniffer_prio", 0); 2193 if (IS_ERR(prio)) 2194 return PTR_ERR(prio); 2195 else 2196 return 0; 2197} 2198 2199static struct mlx5_flow_namespace *fs_create_namespace(struct fs_prio *prio, 2200 const char *name) 2201{ 2202 struct mlx5_flow_namespace *ns; 2203 2204 ns = kzalloc(sizeof(*ns), GFP_KERNEL); 2205 if (!ns) 2206 return ERR_PTR(-ENOMEM); 2207 2208 fs_init_namespace(ns); 2209 fs_add_node(&ns->base, &prio->base, name, 1); 2210 list_add_tail(&ns->base.list, &prio->objs); 2211 2212 return ns; 2213} 2214 2215#define FLOW_TABLE_BIT_SZ 1 2216#define GET_FLOW_TABLE_CAP(dev, offset) \ 2217 ((be32_to_cpu(*((__be32 *)(dev->hca_caps_cur[MLX5_CAP_FLOW_TABLE]) + \ 2218 offset / 32)) >> \ 2219 (32 - FLOW_TABLE_BIT_SZ - (offset & 0x1f))) & FLOW_TABLE_BIT_SZ) 2220 2221static bool has_required_caps(struct mlx5_core_dev *dev, struct node_caps *caps) 2222{ 2223 int i; 2224 2225 for (i = 0; i < caps->arr_sz; i++) { 2226 if (!GET_FLOW_TABLE_CAP(dev, caps->caps[i])) 2227 return false; 2228 } 2229 return true; 2230} 2231 2232static int _init_root_tree(struct mlx5_core_dev *dev, int max_ft_level, 2233 struct init_tree_node *node, struct fs_base *base_parent, 2234 struct init_tree_node *tree_parent) 2235{ 2236 struct mlx5_flow_namespace *fs_ns; 2237 struct fs_prio *fs_prio; 2238 int priority; 2239 struct fs_base *base; 2240 int i; 2241 int err = 0; 2242 2243 if (node->type == FS_TYPE_PRIO) { 2244 if ((node->min_ft_level > max_ft_level) || 2245 !has_required_caps(dev, &node->caps)) 2246 goto out; 2247 2248 fs_get_obj(fs_ns, base_parent); 2249 priority = node - tree_parent->children; 2250 fs_prio = fs_create_prio(fs_ns, priority, 2251 node->max_ft, 2252 node->name, node->flags); 2253 if (IS_ERR(fs_prio)) { 2254 err = PTR_ERR(fs_prio); 2255 goto out; 2256 } 2257 base = &fs_prio->base; 2258 } else if (node->type == FS_TYPE_NAMESPACE) { 2259 fs_get_obj(fs_prio, base_parent); 2260 fs_ns = fs_create_namespace(fs_prio, node->name); 2261 if (IS_ERR(fs_ns)) { 2262 err = PTR_ERR(fs_ns); 2263 goto out; 2264 } 2265 base = &fs_ns->base; 2266 } else { 2267 return -EINVAL; 2268 } 2269 for (i = 0; i < node->ar_size; i++) { 2270 err = _init_root_tree(dev, max_ft_level, &node->children[i], base, 2271 node); 2272 if (err) 2273 break; 2274 } 2275out: 2276 return err; 2277} 2278 2279static int init_root_tree(struct mlx5_core_dev *dev, int max_ft_level, 2280 struct init_tree_node *node, struct fs_base *parent) 2281{ 2282 int i; 2283 struct mlx5_flow_namespace *fs_ns; 2284 int err = 0; 2285 2286 fs_get_obj(fs_ns, parent); 2287 for (i = 0; i < node->ar_size; i++) { 2288 err = _init_root_tree(dev, max_ft_level, 2289 &node->children[i], &fs_ns->base, node); 2290 if (err) 2291 break; 2292 } 2293 return err; 2294} 2295 2296static int sum_max_ft_in_prio(struct fs_prio *prio); 2297static int sum_max_ft_in_ns(struct mlx5_flow_namespace *ns) 2298{ 2299 struct fs_prio *prio; 2300 int sum = 0; 2301 2302 fs_for_each_prio(prio, ns) { 2303 sum += sum_max_ft_in_prio(prio); 2304 } 2305 return sum; 2306} 2307 2308static int sum_max_ft_in_prio(struct fs_prio *prio) 2309{ 2310 int sum = 0; 2311 struct fs_base *it; 2312 struct mlx5_flow_namespace *ns; 2313 2314 if (prio->max_ft) 2315 return prio->max_ft; 2316 2317 fs_for_each_ns_or_ft(it, prio) { 2318 if (it->type == FS_TYPE_FLOW_TABLE) 2319 continue; 2320 2321 fs_get_obj(ns, it); 2322 sum += sum_max_ft_in_ns(ns); 2323 } 2324 prio->max_ft = sum; 2325 return sum; 2326} 2327 2328static void set_max_ft(struct mlx5_flow_namespace *ns) 2329{ 2330 struct fs_prio *prio; 2331 2332 if (!ns) 2333 return; 2334 2335 fs_for_each_prio(prio, ns) 2336 sum_max_ft_in_prio(prio); 2337} 2338 2339static int init_root_ns(struct mlx5_core_dev *dev) 2340{ 2341 int max_ft_level = MLX5_CAP_FLOWTABLE(dev, 2342 flow_table_properties_nic_receive. 2343 max_ft_level); 2344 2345 dev->root_ns = create_root_ns(dev, FS_FT_NIC_RX, 2346 MLX5_CORE_FS_ROOT_NS_NAME); 2347 if (IS_ERR_OR_NULL(dev->root_ns)) 2348 goto err; 2349 2350 2351 if (init_root_tree(dev, max_ft_level, &root_fs, &dev->root_ns->ns.base)) 2352 goto err; 2353 2354 set_max_ft(&dev->root_ns->ns); 2355 2356 return 0; 2357err: 2358 return -ENOMEM; 2359} 2360 2361u8 mlx5_get_match_criteria_enable(struct mlx5_flow_rule *rule) 2362{ 2363 struct fs_base *pbase; 2364 struct mlx5_flow_group *fg; 2365 2366 pbase = rule->base.parent; 2367 WARN_ON(!pbase); 2368 pbase = pbase->parent; 2369 WARN_ON(!pbase); 2370 2371 fs_get_obj(fg, pbase); 2372 return fg->mask.match_criteria_enable; 2373} 2374 2375void mlx5_get_match_value(u32 *match_value, 2376 struct mlx5_flow_rule *rule) 2377{ 2378 struct fs_base *pbase; 2379 struct fs_fte *fte; 2380 2381 pbase = rule->base.parent; 2382 WARN_ON(!pbase); 2383 fs_get_obj(fte, pbase); 2384 2385 memcpy(match_value, fte->val, sizeof(fte->val)); 2386} 2387 2388void mlx5_get_match_criteria(u32 *match_criteria, 2389 struct mlx5_flow_rule *rule) 2390{ 2391 struct fs_base *pbase; 2392 struct mlx5_flow_group *fg; 2393 2394 pbase = rule->base.parent; 2395 WARN_ON(!pbase); 2396 pbase = pbase->parent; 2397 WARN_ON(!pbase); 2398 2399 fs_get_obj(fg, pbase); 2400 memcpy(match_criteria, &fg->mask.match_criteria, 2401 sizeof(fg->mask.match_criteria)); 2402} 2403 2404int mlx5_init_fs(struct mlx5_core_dev *dev) 2405{ 2406 int err; 2407 2408 if (MLX5_CAP_GEN(dev, nic_flow_table)) { 2409 err = init_root_ns(dev); 2410 if (err) 2411 goto err; 2412 } 2413 2414 err = init_fdb_root_ns(dev); 2415 if (err) 2416 goto err; 2417 2418 err = init_egress_acl_root_ns(dev); 2419 if (err) 2420 goto err; 2421 2422 err = init_ingress_acl_root_ns(dev); 2423 if (err) 2424 goto err; 2425 2426 err = init_sniffer_tx_root_ns(dev); 2427 if (err) 2428 goto err; 2429 2430 err = init_sniffer_rx_root_ns(dev); 2431 if (err) 2432 goto err; 2433 2434 err = mlx5_init_fc_stats(dev); 2435 if (err) 2436 goto err; 2437 2438 return 0; 2439err: 2440 mlx5_cleanup_fs(dev); 2441 return err; 2442} 2443 2444struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev, 2445 enum mlx5_flow_namespace_type type) 2446{ 2447 struct mlx5_flow_root_namespace *root_ns = dev->root_ns; 2448 int prio; 2449 static struct fs_prio *fs_prio; 2450 struct mlx5_flow_namespace *ns; 2451 2452 switch (type) { 2453 case MLX5_FLOW_NAMESPACE_BYPASS: 2454 prio = 0; 2455 break; 2456 case MLX5_FLOW_NAMESPACE_OFFLOADS: 2457 prio = 1; 2458 break; 2459 case MLX5_FLOW_NAMESPACE_KERNEL: 2460 prio = 2; 2461 break; 2462 case MLX5_FLOW_NAMESPACE_LEFTOVERS: 2463 prio = 3; 2464 break; 2465 case MLX5_FLOW_NAMESPACE_FDB: 2466 if (dev->fdb_root_ns) 2467 return &dev->fdb_root_ns->ns; 2468 else 2469 return NULL; 2470 case MLX5_FLOW_NAMESPACE_ESW_EGRESS: 2471 if (dev->esw_egress_root_ns) 2472 return &dev->esw_egress_root_ns->ns; 2473 else 2474 return NULL; 2475 case MLX5_FLOW_NAMESPACE_ESW_INGRESS: 2476 if (dev->esw_ingress_root_ns) 2477 return &dev->esw_ingress_root_ns->ns; 2478 else 2479 return NULL; 2480 case MLX5_FLOW_NAMESPACE_SNIFFER_RX: 2481 if (dev->sniffer_rx_root_ns) 2482 return &dev->sniffer_rx_root_ns->ns; 2483 else 2484 return NULL; 2485 case MLX5_FLOW_NAMESPACE_SNIFFER_TX: 2486 if (dev->sniffer_tx_root_ns) 2487 return &dev->sniffer_tx_root_ns->ns; 2488 else 2489 return NULL; 2490 default: 2491 return NULL; 2492 } 2493 2494 if (!root_ns) 2495 return NULL; 2496 2497 fs_prio = find_prio(&root_ns->ns, prio); 2498 if (!fs_prio) 2499 return NULL; 2500 2501 ns = list_first_entry(&fs_prio->objs, 2502 typeof(*ns), 2503 base.list); 2504 2505 return ns; 2506} 2507EXPORT_SYMBOL(mlx5_get_flow_namespace); 2508 2509 2510int mlx5_set_rule_private_data(struct mlx5_flow_rule *rule, 2511 struct mlx5_flow_handler *fs_handler, 2512 void *client_data) 2513{ 2514 struct fs_client_priv_data *priv_data; 2515 2516 mutex_lock(&rule->clients_lock); 2517 /*Check that hanlder isn't exists in the list already*/ 2518 list_for_each_entry(priv_data, &rule->clients_data, list) { 2519 if (priv_data->fs_handler == fs_handler) { 2520 priv_data->client_dst_data = client_data; 2521 goto unlock; 2522 } 2523 } 2524 priv_data = kzalloc(sizeof(*priv_data), GFP_KERNEL); 2525 if (!priv_data) { 2526 mutex_unlock(&rule->clients_lock); 2527 return -ENOMEM; 2528 } 2529 2530 priv_data->client_dst_data = client_data; 2531 priv_data->fs_handler = fs_handler; 2532 list_add(&priv_data->list, &rule->clients_data); 2533 2534unlock: 2535 mutex_unlock(&rule->clients_lock); 2536 2537 return 0; 2538} 2539 2540static int remove_from_clients(struct mlx5_flow_rule *rule, 2541 bool ctx_changed, 2542 void *client_data, 2543 void *context) 2544{ 2545 struct fs_client_priv_data *iter_client; 2546 struct fs_client_priv_data *temp_client; 2547 struct mlx5_flow_handler *handler = (struct 2548 mlx5_flow_handler*)context; 2549 2550 mutex_lock(&rule->clients_lock); 2551 list_for_each_entry_safe(iter_client, temp_client, 2552 &rule->clients_data, list) { 2553 if (iter_client->fs_handler == handler) { 2554 list_del(&iter_client->list); 2555 kfree(iter_client); 2556 break; 2557 } 2558 } 2559 mutex_unlock(&rule->clients_lock); 2560 2561 return 0; 2562} 2563 2564struct mlx5_flow_handler *mlx5_register_rule_notifier(struct mlx5_core_dev *dev, 2565 enum mlx5_flow_namespace_type ns_type, 2566 rule_event_fn add_cb, 2567 rule_event_fn del_cb, 2568 void *context) 2569{ 2570 struct mlx5_flow_namespace *ns; 2571 struct mlx5_flow_handler *handler; 2572 2573 ns = mlx5_get_flow_namespace(dev, ns_type); 2574 if (!ns) 2575 return ERR_PTR(-EINVAL); 2576 2577 handler = kzalloc(sizeof(*handler), GFP_KERNEL); 2578 if (!handler) 2579 return ERR_PTR(-ENOMEM); 2580 2581 handler->add_dst_cb = add_cb; 2582 handler->del_dst_cb = del_cb; 2583 handler->client_context = context; 2584 handler->ns = ns; 2585 down_write(&ns->notifiers_rw_sem); 2586 list_add_tail(&handler->list, &ns->list_notifiers); 2587 up_write(&ns->notifiers_rw_sem); 2588 2589 return handler; 2590} 2591 2592static void iterate_rules_in_ns(struct mlx5_flow_namespace *ns, 2593 rule_event_fn add_rule_cb, 2594 void *context); 2595 2596void mlx5_unregister_rule_notifier(struct mlx5_flow_handler *handler) 2597{ 2598 struct mlx5_flow_namespace *ns = handler->ns; 2599 2600 /*Remove from dst's clients*/ 2601 down_write(&ns->dests_rw_sem); 2602 down_write(&ns->notifiers_rw_sem); 2603 iterate_rules_in_ns(ns, remove_from_clients, handler); 2604 list_del(&handler->list); 2605 up_write(&ns->notifiers_rw_sem); 2606 up_write(&ns->dests_rw_sem); 2607 kfree(handler); 2608} 2609 2610static void iterate_rules_in_ft(struct mlx5_flow_table *ft, 2611 rule_event_fn add_rule_cb, 2612 void *context) 2613{ 2614 struct mlx5_flow_group *iter_fg; 2615 struct fs_fte *iter_fte; 2616 struct mlx5_flow_rule *iter_rule; 2617 int err = 0; 2618 bool is_new_rule; 2619 2620 mutex_lock(&ft->base.lock); 2621 fs_for_each_fg(iter_fg, ft) { 2622 mutex_lock(&iter_fg->base.lock); 2623 fs_for_each_fte(iter_fte, iter_fg) { 2624 mutex_lock(&iter_fte->base.lock); 2625 is_new_rule = true; 2626 fs_for_each_dst(iter_rule, iter_fte) { 2627 fs_get(&iter_rule->base); 2628 err = add_rule_cb(iter_rule, 2629 is_new_rule, 2630 NULL, 2631 context); 2632 fs_put_parent_locked(&iter_rule->base); 2633 if (err) 2634 break; 2635 is_new_rule = false; 2636 } 2637 mutex_unlock(&iter_fte->base.lock); 2638 if (err) 2639 break; 2640 } 2641 mutex_unlock(&iter_fg->base.lock); 2642 if (err) 2643 break; 2644 } 2645 mutex_unlock(&ft->base.lock); 2646} 2647 2648static void iterate_rules_in_prio(struct fs_prio *prio, 2649 rule_event_fn add_rule_cb, 2650 void *context) 2651{ 2652 struct fs_base *it; 2653 2654 mutex_lock(&prio->base.lock); 2655 fs_for_each_ns_or_ft(it, prio) { 2656 if (it->type == FS_TYPE_FLOW_TABLE) { 2657 struct mlx5_flow_table *ft; 2658 2659 fs_get_obj(ft, it); 2660 iterate_rules_in_ft(ft, add_rule_cb, context); 2661 } else { 2662 struct mlx5_flow_namespace *ns; 2663 2664 fs_get_obj(ns, it); 2665 iterate_rules_in_ns(ns, add_rule_cb, context); 2666 } 2667 } 2668 mutex_unlock(&prio->base.lock); 2669} 2670 2671static void iterate_rules_in_ns(struct mlx5_flow_namespace *ns, 2672 rule_event_fn add_rule_cb, 2673 void *context) 2674{ 2675 struct fs_prio *iter_prio; 2676 2677 mutex_lock(&ns->base.lock); 2678 fs_for_each_prio(iter_prio, ns) { 2679 iterate_rules_in_prio(iter_prio, add_rule_cb, context); 2680 } 2681 mutex_unlock(&ns->base.lock); 2682} 2683 2684void mlx5_flow_iterate_existing_rules(struct mlx5_flow_namespace *ns, 2685 rule_event_fn add_rule_cb, 2686 void *context) 2687{ 2688 down_write(&ns->dests_rw_sem); 2689 down_read(&ns->notifiers_rw_sem); 2690 iterate_rules_in_ns(ns, add_rule_cb, context); 2691 up_read(&ns->notifiers_rw_sem); 2692 up_write(&ns->dests_rw_sem); 2693} 2694 2695 2696void mlx5_del_flow_rules_list(struct mlx5_flow_rules_list *rules_list) 2697{ 2698 struct mlx5_flow_rule_node *iter_node; 2699 struct mlx5_flow_rule_node *temp_node; 2700 2701 list_for_each_entry_safe(iter_node, temp_node, &rules_list->head, list) { 2702 list_del(&iter_node->list); 2703 kfree(iter_node); 2704 } 2705 2706 kfree(rules_list); 2707} 2708 2709#define ROCEV1_ETHERTYPE 0x8915 2710static int set_rocev1_rules(struct list_head *rules_list) 2711{ 2712 struct mlx5_flow_rule_node *rocev1_rule; 2713 2714 rocev1_rule = kzalloc(sizeof(*rocev1_rule), GFP_KERNEL); 2715 if (!rocev1_rule) 2716 return -ENOMEM; 2717 2718 rocev1_rule->match_criteria_enable = 2719 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS; 2720 MLX5_SET(fte_match_set_lyr_2_4, rocev1_rule->match_criteria, ethertype, 2721 0xffff); 2722 MLX5_SET(fte_match_set_lyr_2_4, rocev1_rule->match_value, ethertype, 2723 ROCEV1_ETHERTYPE); 2724 2725 list_add_tail(&rocev1_rule->list, rules_list); 2726 2727 return 0; 2728} 2729 2730#define ROCEV2_UDP_PORT 4791 2731static int set_rocev2_rules(struct list_head *rules_list) 2732{ 2733 struct mlx5_flow_rule_node *ipv4_rule; 2734 struct mlx5_flow_rule_node *ipv6_rule; 2735 2736 ipv4_rule = kzalloc(sizeof(*ipv4_rule), GFP_KERNEL); 2737 if (!ipv4_rule) 2738 return -ENOMEM; 2739 2740 ipv6_rule = kzalloc(sizeof(*ipv6_rule), GFP_KERNEL); 2741 if (!ipv6_rule) { 2742 kfree(ipv4_rule); 2743 return -ENOMEM; 2744 } 2745 2746 ipv4_rule->match_criteria_enable = 2747 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS; 2748 MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, ethertype, 2749 0xffff); 2750 MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, ethertype, 2751 0x0800); 2752 MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, ip_protocol, 2753 0xff); 2754 MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, ip_protocol, 2755 IPPROTO_UDP); 2756 MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, udp_dport, 2757 0xffff); 2758 MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, udp_dport, 2759 ROCEV2_UDP_PORT); 2760 2761 ipv6_rule->match_criteria_enable = 2762 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS; 2763 MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, ethertype, 2764 0xffff); 2765 MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, ethertype, 2766 0x86dd); 2767 MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, ip_protocol, 2768 0xff); 2769 MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, ip_protocol, 2770 IPPROTO_UDP); 2771 MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, udp_dport, 2772 0xffff); 2773 MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, udp_dport, 2774 ROCEV2_UDP_PORT); 2775 2776 list_add_tail(&ipv4_rule->list, rules_list); 2777 list_add_tail(&ipv6_rule->list, rules_list); 2778 2779 return 0; 2780} 2781 2782 2783struct mlx5_flow_rules_list *get_roce_flow_rules(u8 roce_mode) 2784{ 2785 int err = 0; 2786 struct mlx5_flow_rules_list *rules_list = 2787 kzalloc(sizeof(*rules_list), GFP_KERNEL); 2788 2789 if (!rules_list) 2790 return NULL; 2791 2792 INIT_LIST_HEAD(&rules_list->head); 2793 2794 if (roce_mode & MLX5_ROCE_VERSION_1_CAP) { 2795 err = set_rocev1_rules(&rules_list->head); 2796 if (err) 2797 goto free_list; 2798 } 2799 if (roce_mode & MLX5_ROCE_VERSION_2_CAP) 2800 err = set_rocev2_rules(&rules_list->head); 2801 if (err) 2802 goto free_list; 2803 2804 return rules_list; 2805 2806free_list: 2807 mlx5_del_flow_rules_list(rules_list); 2808 return NULL; 2809} 2810 2811struct mlx5_modify_hdr *mlx5_modify_header_alloc(struct mlx5_core_dev *dev, 2812 enum mlx5_flow_namespace_type ns_type, 2813 u8 num_actions, 2814 void *modify_actions) 2815{ 2816 struct mlx5_modify_hdr *modify_hdr; 2817 int err; 2818 2819 modify_hdr = kzalloc(sizeof(*modify_hdr), GFP_KERNEL); 2820 if (!modify_hdr) 2821 return ERR_PTR(-ENOMEM); 2822 2823 modify_hdr->ns_type = ns_type; 2824 err = mlx5_cmd_modify_header_alloc(dev, ns_type, num_actions, 2825 modify_actions, modify_hdr); 2826 if (err) { 2827 kfree(modify_hdr); 2828 return ERR_PTR(err); 2829 } 2830 2831 return modify_hdr; 2832} 2833EXPORT_SYMBOL(mlx5_modify_header_alloc); 2834 2835void mlx5_modify_header_dealloc(struct mlx5_core_dev *dev, 2836 struct mlx5_modify_hdr *modify_hdr) 2837{ 2838 mlx5_cmd_modify_header_dealloc(dev, modify_hdr); 2839 kfree(modify_hdr); 2840} 2841EXPORT_SYMBOL(mlx5_modify_header_dealloc); 2842 2843struct mlx5_pkt_reformat *mlx5_packet_reformat_alloc(struct mlx5_core_dev *dev, 2844 struct mlx5_pkt_reformat_params *params, 2845 enum mlx5_flow_namespace_type ns_type) 2846{ 2847 struct mlx5_pkt_reformat *pkt_reformat; 2848 int err; 2849 2850 pkt_reformat = kzalloc(sizeof(*pkt_reformat), GFP_KERNEL); 2851 if (!pkt_reformat) 2852 return ERR_PTR(-ENOMEM); 2853 2854 pkt_reformat->ns_type = ns_type; 2855 pkt_reformat->reformat_type = params->type; 2856 err = mlx5_cmd_packet_reformat_alloc(dev, params, ns_type, 2857 pkt_reformat); 2858 if (err) { 2859 kfree(pkt_reformat); 2860 return ERR_PTR(err); 2861 } 2862 2863 return pkt_reformat; 2864} 2865EXPORT_SYMBOL(mlx5_packet_reformat_alloc); 2866 2867void mlx5_packet_reformat_dealloc(struct mlx5_core_dev *dev, 2868 struct mlx5_pkt_reformat *pkt_reformat) 2869{ 2870 mlx5_cmd_packet_reformat_dealloc(dev, pkt_reformat); 2871 kfree(pkt_reformat); 2872} 2873EXPORT_SYMBOL(mlx5_packet_reformat_dealloc); 2874 2875