1/* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#include <linux/etherdevice.h> 34#include <linux/idr.h> 35#include <linux/mlx5/driver.h> 36#include <linux/mlx5/mlx5_ifc.h> 37#include <linux/mlx5/vport.h> 38#include <linux/mlx5/fs.h> 39#include "mlx5_core.h" 40#include "eswitch.h" 41#include "esw/indir_table.h" 42#include "esw/acl/ofld.h" 43#include "rdma.h" 44#include "en.h" 45#include "fs_core.h" 46#include "lib/mlx5.h" 47#include "lib/devcom.h" 48#include "lib/eq.h" 49#include "lib/fs_chains.h" 50#include "en_tc.h" 51#include "en/mapping.h" 52#include "devlink.h" 53#include "lag/lag.h" 54#include "en/tc/post_meter.h" 55 56#define mlx5_esw_for_each_rep(esw, i, rep) \ 57 xa_for_each(&((esw)->offloads.vport_reps), i, rep) 58 59/* There are two match-all miss flows, one for unicast dst mac and 60 * one for multicast. 61 */ 62#define MLX5_ESW_MISS_FLOWS (2) 63#define UPLINK_REP_INDEX 0 64 65#define MLX5_ESW_VPORT_TBL_SIZE 128 66#define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4 67 68#define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1) 69 70static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = { 71 .max_fte = MLX5_ESW_VPORT_TBL_SIZE, 72 .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS, 73 .flags = 0, 74}; 75 76static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw, 77 u16 vport_num) 78{ 79 return xa_load(&esw->offloads.vport_reps, vport_num); 80} 81 82static void 83mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw, 84 struct mlx5_flow_spec *spec, 85 struct mlx5_esw_flow_attr *attr) 86{ 87 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) || !attr || !attr->in_rep) 88 return; 89 90 if (attr->int_port) { 91 spec->flow_context.flow_source = mlx5e_tc_int_port_get_flow_source(attr->int_port); 92 93 return; 94 } 95 96 spec->flow_context.flow_source = (attr->in_rep->vport == MLX5_VPORT_UPLINK) ? 97 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK : 98 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; 99} 100 101/* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits 102 * are not needed as well in the following process. So clear them all for simplicity. 103 */ 104void 105mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec) 106{ 107 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 108 void *misc2; 109 110 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 111 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0); 112 113 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 114 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0); 115 116 if (!memchr_inv(misc2, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2))) 117 spec->match_criteria_enable &= ~MLX5_MATCH_MISC_PARAMETERS_2; 118 } 119} 120 121static void 122mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw, 123 struct mlx5_flow_spec *spec, 124 struct mlx5_flow_attr *attr, 125 struct mlx5_eswitch *src_esw, 126 u16 vport) 127{ 128 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 129 u32 metadata; 130 void *misc2; 131 void *misc; 132 133 /* Use metadata matching because vport is not represented by single 134 * VHCA in dual-port RoCE mode, and matching on source vport may fail. 135 */ 136 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 137 if (mlx5_esw_indir_table_decap_vport(attr)) 138 vport = mlx5_esw_indir_table_decap_vport(attr); 139 140 if (!attr->chain && esw_attr && esw_attr->int_port) 141 metadata = 142 mlx5e_tc_int_port_get_metadata_for_match(esw_attr->int_port); 143 else 144 metadata = 145 mlx5_eswitch_get_vport_metadata_for_match(src_esw, vport); 146 147 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 148 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, metadata); 149 150 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 151 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 152 mlx5_eswitch_get_vport_metadata_mask()); 153 154 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 155 } else { 156 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 157 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 158 159 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 160 MLX5_SET(fte_match_set_misc, misc, 161 source_eswitch_owner_vhca_id, 162 MLX5_CAP_GEN(src_esw->dev, vhca_id)); 163 164 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 165 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 166 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 167 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 168 source_eswitch_owner_vhca_id); 169 170 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 171 } 172} 173 174static int 175esw_setup_decap_indir(struct mlx5_eswitch *esw, 176 struct mlx5_flow_attr *attr) 177{ 178 struct mlx5_flow_table *ft; 179 180 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 181 return -EOPNOTSUPP; 182 183 ft = mlx5_esw_indir_table_get(esw, attr, 184 mlx5_esw_indir_table_decap_vport(attr), true); 185 return PTR_ERR_OR_ZERO(ft); 186} 187 188static void 189esw_cleanup_decap_indir(struct mlx5_eswitch *esw, 190 struct mlx5_flow_attr *attr) 191{ 192 if (mlx5_esw_indir_table_decap_vport(attr)) 193 mlx5_esw_indir_table_put(esw, 194 mlx5_esw_indir_table_decap_vport(attr), 195 true); 196} 197 198static int 199esw_setup_mtu_dest(struct mlx5_flow_destination *dest, 200 struct mlx5e_meter_attr *meter, 201 int i) 202{ 203 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_RANGE; 204 dest[i].range.field = MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN; 205 dest[i].range.min = 0; 206 dest[i].range.max = meter->params.mtu; 207 dest[i].range.hit_ft = mlx5e_post_meter_get_mtu_true_ft(meter->post_meter); 208 dest[i].range.miss_ft = mlx5e_post_meter_get_mtu_false_ft(meter->post_meter); 209 210 return 0; 211} 212 213static int 214esw_setup_sampler_dest(struct mlx5_flow_destination *dest, 215 struct mlx5_flow_act *flow_act, 216 u32 sampler_id, 217 int i) 218{ 219 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 220 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER; 221 dest[i].sampler_id = sampler_id; 222 223 return 0; 224} 225 226static int 227esw_setup_ft_dest(struct mlx5_flow_destination *dest, 228 struct mlx5_flow_act *flow_act, 229 struct mlx5_eswitch *esw, 230 struct mlx5_flow_attr *attr, 231 int i) 232{ 233 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 234 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 235 dest[i].ft = attr->dest_ft; 236 237 if (mlx5_esw_indir_table_decap_vport(attr)) 238 return esw_setup_decap_indir(esw, attr); 239 return 0; 240} 241 242static void 243esw_setup_accept_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 244 struct mlx5_fs_chains *chains, int i) 245{ 246 if (mlx5_chains_ignore_flow_level_supported(chains)) 247 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 248 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 249 dest[i].ft = mlx5_chains_get_tc_end_ft(chains); 250} 251 252static void 253esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 254 struct mlx5_eswitch *esw, int i) 255{ 256 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level)) 257 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 258 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 259 dest[i].ft = mlx5_eswitch_get_slow_fdb(esw); 260} 261 262static int 263esw_setup_chain_dest(struct mlx5_flow_destination *dest, 264 struct mlx5_flow_act *flow_act, 265 struct mlx5_fs_chains *chains, 266 u32 chain, u32 prio, u32 level, 267 int i) 268{ 269 struct mlx5_flow_table *ft; 270 271 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 272 ft = mlx5_chains_get_table(chains, chain, prio, level); 273 if (IS_ERR(ft)) 274 return PTR_ERR(ft); 275 276 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 277 dest[i].ft = ft; 278 return 0; 279} 280 281static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr, 282 int from, int to) 283{ 284 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 285 struct mlx5_fs_chains *chains = esw_chains(esw); 286 int i; 287 288 for (i = from; i < to; i++) 289 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 290 mlx5_chains_put_table(chains, 0, 1, 0); 291 else if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].vport, 292 esw_attr->dests[i].mdev)) 293 mlx5_esw_indir_table_put(esw, esw_attr->dests[i].vport, false); 294} 295 296static bool 297esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr) 298{ 299 int i; 300 301 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) 302 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 303 return true; 304 return false; 305} 306 307static int 308esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest, 309 struct mlx5_flow_act *flow_act, 310 struct mlx5_eswitch *esw, 311 struct mlx5_fs_chains *chains, 312 struct mlx5_flow_attr *attr, 313 int *i) 314{ 315 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 316 int err; 317 318 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 319 return -EOPNOTSUPP; 320 321 /* flow steering cannot handle more than one dest with the same ft 322 * in a single flow 323 */ 324 if (esw_attr->out_count - esw_attr->split_count > 1) 325 return -EOPNOTSUPP; 326 327 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i); 328 if (err) 329 return err; 330 331 if (esw_attr->dests[esw_attr->split_count].pkt_reformat) { 332 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 333 flow_act->pkt_reformat = esw_attr->dests[esw_attr->split_count].pkt_reformat; 334 } 335 (*i)++; 336 337 return 0; 338} 339 340static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw, 341 struct mlx5_flow_attr *attr) 342{ 343 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 344 345 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count); 346} 347 348static bool 349esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) 350{ 351 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 352 bool result = false; 353 int i; 354 355 /* Indirect table is supported only for flows with in_port uplink 356 * and the destination is vport on the same eswitch as the uplink, 357 * return false in case at least one of destinations doesn't meet 358 * this criteria. 359 */ 360 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) { 361 if (esw_attr->dests[i].vport_valid && 362 mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].vport, 363 esw_attr->dests[i].mdev)) { 364 result = true; 365 } else { 366 result = false; 367 break; 368 } 369 } 370 return result; 371} 372 373static int 374esw_setup_indir_table(struct mlx5_flow_destination *dest, 375 struct mlx5_flow_act *flow_act, 376 struct mlx5_eswitch *esw, 377 struct mlx5_flow_attr *attr, 378 int *i) 379{ 380 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 381 int j, err; 382 383 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 384 return -EOPNOTSUPP; 385 386 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) { 387 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 388 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 389 390 dest[*i].ft = mlx5_esw_indir_table_get(esw, attr, 391 esw_attr->dests[j].vport, false); 392 if (IS_ERR(dest[*i].ft)) { 393 err = PTR_ERR(dest[*i].ft); 394 goto err_indir_tbl_get; 395 } 396 } 397 398 if (mlx5_esw_indir_table_decap_vport(attr)) { 399 err = esw_setup_decap_indir(esw, attr); 400 if (err) 401 goto err_indir_tbl_get; 402 } 403 404 return 0; 405 406err_indir_tbl_get: 407 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j); 408 return err; 409} 410 411static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) 412{ 413 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 414 415 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count); 416 esw_cleanup_decap_indir(esw, attr); 417} 418 419static void 420esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level) 421{ 422 mlx5_chains_put_table(chains, chain, prio, level); 423} 424 425static bool esw_same_vhca_id(struct mlx5_core_dev *mdev1, struct mlx5_core_dev *mdev2) 426{ 427 return MLX5_CAP_GEN(mdev1, vhca_id) == MLX5_CAP_GEN(mdev2, vhca_id); 428} 429 430static bool esw_setup_uplink_fwd_ipsec_needed(struct mlx5_eswitch *esw, 431 struct mlx5_esw_flow_attr *esw_attr, 432 int attr_idx) 433{ 434 if (esw->offloads.ft_ipsec_tx_pol && 435 esw_attr->dests[attr_idx].vport_valid && 436 esw_attr->dests[attr_idx].vport == MLX5_VPORT_UPLINK && 437 /* To be aligned with software, encryption is needed only for tunnel device */ 438 (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) && 439 esw_attr->dests[attr_idx].vport != esw_attr->in_rep->vport && 440 esw_same_vhca_id(esw_attr->dests[attr_idx].mdev, esw->dev)) 441 return true; 442 443 return false; 444} 445 446static bool esw_flow_dests_fwd_ipsec_check(struct mlx5_eswitch *esw, 447 struct mlx5_esw_flow_attr *esw_attr) 448{ 449 int i; 450 451 if (!esw->offloads.ft_ipsec_tx_pol) 452 return true; 453 454 for (i = 0; i < esw_attr->split_count; i++) 455 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i)) 456 return false; 457 458 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) 459 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i) && 460 (esw_attr->out_count - esw_attr->split_count > 1)) 461 return false; 462 463 return true; 464} 465 466static void 467esw_setup_dest_fwd_vport(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 468 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 469 int attr_idx, int dest_idx, bool pkt_reformat) 470{ 471 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 472 dest[dest_idx].vport.num = esw_attr->dests[attr_idx].vport; 473 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) { 474 dest[dest_idx].vport.vhca_id = 475 MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id); 476 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 477 if (dest[dest_idx].vport.num == MLX5_VPORT_UPLINK && 478 mlx5_lag_is_mpesw(esw->dev)) 479 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_UPLINK; 480 } 481 if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) { 482 if (pkt_reformat) { 483 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 484 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 485 } 486 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID; 487 dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 488 } 489} 490 491static void 492esw_setup_dest_fwd_ipsec(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 493 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 494 int attr_idx, int dest_idx, bool pkt_reformat) 495{ 496 dest[dest_idx].ft = esw->offloads.ft_ipsec_tx_pol; 497 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 498 if (pkt_reformat && 499 esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) { 500 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 501 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 502 } 503} 504 505static void 506esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 507 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 508 int attr_idx, int dest_idx, bool pkt_reformat) 509{ 510 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx)) 511 esw_setup_dest_fwd_ipsec(dest, flow_act, esw, esw_attr, 512 attr_idx, dest_idx, pkt_reformat); 513 else 514 esw_setup_dest_fwd_vport(dest, flow_act, esw, esw_attr, 515 attr_idx, dest_idx, pkt_reformat); 516} 517 518static int 519esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 520 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 521 int i) 522{ 523 int j; 524 525 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++) 526 esw_setup_vport_dest(dest, flow_act, esw, esw_attr, j, i, true); 527 return i; 528} 529 530static bool 531esw_src_port_rewrite_supported(struct mlx5_eswitch *esw) 532{ 533 return MLX5_CAP_GEN(esw->dev, reg_c_preserve) && 534 mlx5_eswitch_vport_match_metadata_enabled(esw) && 535 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level); 536} 537 538static bool 539esw_dests_to_int_external(struct mlx5_flow_destination *dests, int max_dest) 540{ 541 bool internal_dest = false, external_dest = false; 542 int i; 543 544 for (i = 0; i < max_dest; i++) { 545 if (dests[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT && 546 dests[i].type != MLX5_FLOW_DESTINATION_TYPE_UPLINK) 547 continue; 548 549 /* Uplink dest is external, but considered as internal 550 * if there is reformat because firmware uses LB+hairpin to support it. 551 */ 552 if (dests[i].vport.num == MLX5_VPORT_UPLINK && 553 !(dests[i].vport.flags & MLX5_FLOW_DEST_VPORT_REFORMAT_ID)) 554 external_dest = true; 555 else 556 internal_dest = true; 557 558 if (internal_dest && external_dest) 559 return true; 560 } 561 562 return false; 563} 564 565static int 566esw_setup_dests(struct mlx5_flow_destination *dest, 567 struct mlx5_flow_act *flow_act, 568 struct mlx5_eswitch *esw, 569 struct mlx5_flow_attr *attr, 570 struct mlx5_flow_spec *spec, 571 int *i) 572{ 573 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 574 struct mlx5_fs_chains *chains = esw_chains(esw); 575 int err = 0; 576 577 if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) && 578 esw_src_port_rewrite_supported(esw)) 579 attr->flags |= MLX5_ATTR_FLAG_SRC_REWRITE; 580 581 if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) { 582 esw_setup_slow_path_dest(dest, flow_act, esw, *i); 583 (*i)++; 584 goto out; 585 } 586 587 if (attr->flags & MLX5_ATTR_FLAG_SAMPLE) { 588 esw_setup_sampler_dest(dest, flow_act, attr->sample_attr.sampler_id, *i); 589 (*i)++; 590 } else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) { 591 esw_setup_accept_dest(dest, flow_act, chains, *i); 592 (*i)++; 593 } else if (attr->flags & MLX5_ATTR_FLAG_MTU) { 594 err = esw_setup_mtu_dest(dest, &attr->meter_attr, *i); 595 (*i)++; 596 } else if (esw_is_indir_table(esw, attr)) { 597 err = esw_setup_indir_table(dest, flow_act, esw, attr, i); 598 } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) { 599 err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i); 600 } else { 601 *i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i); 602 603 if (attr->dest_ft) { 604 err = esw_setup_ft_dest(dest, flow_act, esw, attr, *i); 605 (*i)++; 606 } else if (attr->dest_chain) { 607 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 608 1, 0, *i); 609 (*i)++; 610 } 611 } 612 613out: 614 return err; 615} 616 617static void 618esw_cleanup_dests(struct mlx5_eswitch *esw, 619 struct mlx5_flow_attr *attr) 620{ 621 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 622 struct mlx5_fs_chains *chains = esw_chains(esw); 623 624 if (attr->dest_ft) { 625 esw_cleanup_decap_indir(esw, attr); 626 } else if (!mlx5e_tc_attr_flags_skip(attr->flags)) { 627 if (attr->dest_chain) 628 esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0); 629 else if (esw_is_indir_table(esw, attr)) 630 esw_cleanup_indir_table(esw, attr); 631 else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) 632 esw_cleanup_chain_src_port_rewrite(esw, attr); 633 } 634} 635 636static void 637esw_setup_meter(struct mlx5_flow_attr *attr, struct mlx5_flow_act *flow_act) 638{ 639 struct mlx5e_flow_meter_handle *meter; 640 641 meter = attr->meter_attr.meter; 642 flow_act->exe_aso.type = attr->exe_aso_type; 643 flow_act->exe_aso.object_id = meter->obj_id; 644 flow_act->exe_aso.flow_meter.meter_idx = meter->idx; 645 flow_act->exe_aso.flow_meter.init_color = MLX5_FLOW_METER_COLOR_GREEN; 646 /* use metadata reg 5 for packet color */ 647 flow_act->exe_aso.return_reg_id = 5; 648} 649 650struct mlx5_flow_handle * 651mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, 652 struct mlx5_flow_spec *spec, 653 struct mlx5_flow_attr *attr) 654{ 655 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 656 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 657 struct mlx5_fs_chains *chains = esw_chains(esw); 658 bool split = !!(esw_attr->split_count); 659 struct mlx5_vport_tbl_attr fwd_attr; 660 struct mlx5_flow_destination *dest; 661 struct mlx5_flow_handle *rule; 662 struct mlx5_flow_table *fdb; 663 int i = 0; 664 665 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 666 return ERR_PTR(-EOPNOTSUPP); 667 668 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 669 return ERR_PTR(-EOPNOTSUPP); 670 671 if (!esw_flow_dests_fwd_ipsec_check(esw, esw_attr)) 672 return ERR_PTR(-EOPNOTSUPP); 673 674 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL); 675 if (!dest) 676 return ERR_PTR(-ENOMEM); 677 678 flow_act.action = attr->action; 679 680 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) { 681 flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]); 682 flow_act.vlan[0].vid = esw_attr->vlan_vid[0]; 683 flow_act.vlan[0].prio = esw_attr->vlan_prio[0]; 684 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) { 685 flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]); 686 flow_act.vlan[1].vid = esw_attr->vlan_vid[1]; 687 flow_act.vlan[1].prio = esw_attr->vlan_prio[1]; 688 } 689 } 690 691 mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr); 692 693 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) { 694 int err; 695 696 err = esw_setup_dests(dest, &flow_act, esw, attr, spec, &i); 697 if (err) { 698 rule = ERR_PTR(err); 699 goto err_create_goto_table; 700 } 701 702 /* Header rewrite with combined wire+loopback in FDB is not allowed */ 703 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) && 704 esw_dests_to_int_external(dest, i)) { 705 esw_warn(esw->dev, 706 "FDB: Header rewrite with forwarding to both internal and external dests is not allowed\n"); 707 rule = ERR_PTR(-EINVAL); 708 goto err_esw_get; 709 } 710 } 711 712 if (esw_attr->decap_pkt_reformat) 713 flow_act.pkt_reformat = esw_attr->decap_pkt_reformat; 714 715 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 716 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 717 dest[i].counter_id = mlx5_fc_id(attr->counter); 718 i++; 719 } 720 721 if (attr->outer_match_level != MLX5_MATCH_NONE) 722 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 723 if (attr->inner_match_level != MLX5_MATCH_NONE) 724 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS; 725 726 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 727 flow_act.modify_hdr = attr->modify_hdr; 728 729 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) && 730 attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER) 731 esw_setup_meter(attr, &flow_act); 732 733 if (split) { 734 fwd_attr.chain = attr->chain; 735 fwd_attr.prio = attr->prio; 736 fwd_attr.vport = esw_attr->in_rep->vport; 737 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 738 739 fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr); 740 } else { 741 if (attr->chain || attr->prio) 742 fdb = mlx5_chains_get_table(chains, attr->chain, 743 attr->prio, 0); 744 else 745 fdb = attr->ft; 746 747 if (!(attr->flags & MLX5_ATTR_FLAG_NO_IN_PORT)) 748 mlx5_eswitch_set_rule_source_port(esw, spec, attr, 749 esw_attr->in_mdev->priv.eswitch, 750 esw_attr->in_rep->vport); 751 } 752 if (IS_ERR(fdb)) { 753 rule = ERR_CAST(fdb); 754 goto err_esw_get; 755 } 756 757 if (!i) { 758 kfree(dest); 759 dest = NULL; 760 } 761 762 if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec)) 763 rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr, 764 &flow_act, dest, i); 765 else 766 rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i); 767 if (IS_ERR(rule)) 768 goto err_add_rule; 769 else 770 atomic64_inc(&esw->offloads.num_flows); 771 772 kfree(dest); 773 return rule; 774 775err_add_rule: 776 if (split) 777 mlx5_esw_vporttbl_put(esw, &fwd_attr); 778 else if (attr->chain || attr->prio) 779 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 780err_esw_get: 781 esw_cleanup_dests(esw, attr); 782err_create_goto_table: 783 kfree(dest); 784 return rule; 785} 786 787struct mlx5_flow_handle * 788mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, 789 struct mlx5_flow_spec *spec, 790 struct mlx5_flow_attr *attr) 791{ 792 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 793 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 794 struct mlx5_fs_chains *chains = esw_chains(esw); 795 struct mlx5_vport_tbl_attr fwd_attr; 796 struct mlx5_flow_destination *dest; 797 struct mlx5_flow_table *fast_fdb; 798 struct mlx5_flow_table *fwd_fdb; 799 struct mlx5_flow_handle *rule; 800 int i, err = 0; 801 802 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL); 803 if (!dest) 804 return ERR_PTR(-ENOMEM); 805 806 fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0); 807 if (IS_ERR(fast_fdb)) { 808 rule = ERR_CAST(fast_fdb); 809 goto err_get_fast; 810 } 811 812 fwd_attr.chain = attr->chain; 813 fwd_attr.prio = attr->prio; 814 fwd_attr.vport = esw_attr->in_rep->vport; 815 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 816 fwd_fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr); 817 if (IS_ERR(fwd_fdb)) { 818 rule = ERR_CAST(fwd_fdb); 819 goto err_get_fwd; 820 } 821 822 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 823 for (i = 0; i < esw_attr->split_count; i++) { 824 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 825 /* Source port rewrite (forward to ovs internal port or statck device) isn't 826 * supported in the rule of split action. 827 */ 828 err = -EOPNOTSUPP; 829 else 830 esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false); 831 832 if (err) { 833 rule = ERR_PTR(err); 834 goto err_chain_src_rewrite; 835 } 836 } 837 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 838 dest[i].ft = fwd_fdb; 839 i++; 840 841 mlx5_eswitch_set_rule_source_port(esw, spec, attr, 842 esw_attr->in_mdev->priv.eswitch, 843 esw_attr->in_rep->vport); 844 845 if (attr->outer_match_level != MLX5_MATCH_NONE) 846 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 847 848 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 849 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i); 850 851 if (IS_ERR(rule)) { 852 i = esw_attr->split_count; 853 goto err_chain_src_rewrite; 854 } 855 856 atomic64_inc(&esw->offloads.num_flows); 857 858 kfree(dest); 859 return rule; 860err_chain_src_rewrite: 861 mlx5_esw_vporttbl_put(esw, &fwd_attr); 862err_get_fwd: 863 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 864err_get_fast: 865 kfree(dest); 866 return rule; 867} 868 869static void 870__mlx5_eswitch_del_rule(struct mlx5_eswitch *esw, 871 struct mlx5_flow_handle *rule, 872 struct mlx5_flow_attr *attr, 873 bool fwd_rule) 874{ 875 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 876 struct mlx5_fs_chains *chains = esw_chains(esw); 877 bool split = (esw_attr->split_count > 0); 878 struct mlx5_vport_tbl_attr fwd_attr; 879 int i; 880 881 mlx5_del_flow_rules(rule); 882 883 if (!mlx5e_tc_attr_flags_skip(attr->flags)) { 884 /* unref the term table */ 885 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) { 886 if (esw_attr->dests[i].termtbl) 887 mlx5_eswitch_termtbl_put(esw, esw_attr->dests[i].termtbl); 888 } 889 } 890 891 atomic64_dec(&esw->offloads.num_flows); 892 893 if (fwd_rule || split) { 894 fwd_attr.chain = attr->chain; 895 fwd_attr.prio = attr->prio; 896 fwd_attr.vport = esw_attr->in_rep->vport; 897 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 898 } 899 900 if (fwd_rule) { 901 mlx5_esw_vporttbl_put(esw, &fwd_attr); 902 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 903 } else { 904 if (split) 905 mlx5_esw_vporttbl_put(esw, &fwd_attr); 906 else if (attr->chain || attr->prio) 907 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 908 esw_cleanup_dests(esw, attr); 909 } 910} 911 912void 913mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw, 914 struct mlx5_flow_handle *rule, 915 struct mlx5_flow_attr *attr) 916{ 917 __mlx5_eswitch_del_rule(esw, rule, attr, false); 918} 919 920void 921mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw, 922 struct mlx5_flow_handle *rule, 923 struct mlx5_flow_attr *attr) 924{ 925 __mlx5_eswitch_del_rule(esw, rule, attr, true); 926} 927 928struct mlx5_flow_handle * 929mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw, 930 struct mlx5_eswitch *from_esw, 931 struct mlx5_eswitch_rep *rep, 932 u32 sqn) 933{ 934 struct mlx5_flow_act flow_act = {0}; 935 struct mlx5_flow_destination dest = {}; 936 struct mlx5_flow_handle *flow_rule; 937 struct mlx5_flow_spec *spec; 938 void *misc; 939 u16 vport; 940 941 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 942 if (!spec) { 943 flow_rule = ERR_PTR(-ENOMEM); 944 goto out; 945 } 946 947 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 948 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn); 949 950 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 951 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn); 952 953 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 954 955 /* source vport is the esw manager */ 956 vport = from_esw->manager_vport; 957 958 if (mlx5_eswitch_vport_match_metadata_enabled(on_esw)) { 959 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 960 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 961 mlx5_eswitch_get_vport_metadata_for_match(from_esw, vport)); 962 963 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 964 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 965 mlx5_eswitch_get_vport_metadata_mask()); 966 967 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 968 } else { 969 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 970 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 971 972 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch)) 973 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 974 MLX5_CAP_GEN(from_esw->dev, vhca_id)); 975 976 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 977 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 978 979 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch)) 980 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 981 source_eswitch_owner_vhca_id); 982 983 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 984 } 985 986 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 987 dest.vport.num = rep->vport; 988 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id); 989 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 990 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 991 992 if (rep->vport == MLX5_VPORT_UPLINK && 993 on_esw == from_esw && on_esw->offloads.ft_ipsec_tx_pol) { 994 dest.ft = on_esw->offloads.ft_ipsec_tx_pol; 995 flow_act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL; 996 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 997 } else { 998 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 999 dest.vport.num = rep->vport; 1000 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id); 1001 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 1002 } 1003 1004 if (MLX5_CAP_ESW_FLOWTABLE(on_esw->dev, flow_source) && 1005 rep->vport == MLX5_VPORT_UPLINK) 1006 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; 1007 1008 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(on_esw), 1009 spec, &flow_act, &dest, 1); 1010 if (IS_ERR(flow_rule)) 1011 esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %ld\n", 1012 PTR_ERR(flow_rule)); 1013out: 1014 kvfree(spec); 1015 return flow_rule; 1016} 1017EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule); 1018 1019void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule) 1020{ 1021 mlx5_del_flow_rules(rule); 1022} 1023 1024void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule) 1025{ 1026 if (rule) 1027 mlx5_del_flow_rules(rule); 1028} 1029 1030struct mlx5_flow_handle * 1031mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num) 1032{ 1033 struct mlx5_flow_destination dest = {}; 1034 struct mlx5_flow_act flow_act = {0}; 1035 struct mlx5_flow_handle *flow_rule; 1036 struct mlx5_flow_spec *spec; 1037 1038 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1039 if (!spec) 1040 return ERR_PTR(-ENOMEM); 1041 1042 MLX5_SET(fte_match_param, spec->match_criteria, 1043 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); 1044 MLX5_SET(fte_match_param, spec->match_criteria, 1045 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); 1046 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1, 1047 ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK); 1048 1049 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1050 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1051 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1052 1053 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0, 1054 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num)); 1055 dest.vport.num = vport_num; 1056 1057 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1058 spec, &flow_act, &dest, 1); 1059 if (IS_ERR(flow_rule)) 1060 esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %ld\n", 1061 vport_num, PTR_ERR(flow_rule)); 1062 1063 kvfree(spec); 1064 return flow_rule; 1065} 1066 1067static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw) 1068{ 1069 return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 1070 MLX5_FDB_TO_VPORT_REG_C_1; 1071} 1072 1073static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable) 1074{ 1075 u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {}; 1076 u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {}; 1077 u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {}; 1078 u8 curr, wanted; 1079 int err; 1080 1081 if (!mlx5_eswitch_reg_c1_loopback_supported(esw) && 1082 !mlx5_eswitch_vport_match_metadata_enabled(esw)) 1083 return 0; 1084 1085 MLX5_SET(query_esw_vport_context_in, in, opcode, 1086 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT); 1087 err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out); 1088 if (err) 1089 return err; 1090 1091 curr = MLX5_GET(query_esw_vport_context_out, out, 1092 esw_vport_context.fdb_to_vport_reg_c_id); 1093 wanted = MLX5_FDB_TO_VPORT_REG_C_0; 1094 if (mlx5_eswitch_reg_c1_loopback_supported(esw)) 1095 wanted |= MLX5_FDB_TO_VPORT_REG_C_1; 1096 1097 if (enable) 1098 curr |= wanted; 1099 else 1100 curr &= ~wanted; 1101 1102 MLX5_SET(modify_esw_vport_context_in, min, 1103 esw_vport_context.fdb_to_vport_reg_c_id, curr); 1104 MLX5_SET(modify_esw_vport_context_in, min, 1105 field_select.fdb_to_vport_reg_c_id, 1); 1106 1107 err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min); 1108 if (!err) { 1109 if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1)) 1110 esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 1111 else 1112 esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 1113 } 1114 1115 return err; 1116} 1117 1118static void peer_miss_rules_setup(struct mlx5_eswitch *esw, 1119 struct mlx5_core_dev *peer_dev, 1120 struct mlx5_flow_spec *spec, 1121 struct mlx5_flow_destination *dest) 1122{ 1123 void *misc; 1124 1125 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1126 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1127 misc_parameters_2); 1128 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1129 mlx5_eswitch_get_vport_metadata_mask()); 1130 1131 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1132 } else { 1133 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1134 misc_parameters); 1135 1136 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 1137 MLX5_CAP_GEN(peer_dev, vhca_id)); 1138 1139 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 1140 1141 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1142 misc_parameters); 1143 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1144 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 1145 source_eswitch_owner_vhca_id); 1146 } 1147 1148 dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1149 dest->vport.num = peer_dev->priv.eswitch->manager_vport; 1150 dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id); 1151 dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 1152} 1153 1154static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw, 1155 struct mlx5_eswitch *peer_esw, 1156 struct mlx5_flow_spec *spec, 1157 u16 vport) 1158{ 1159 void *misc; 1160 1161 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1162 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1163 misc_parameters_2); 1164 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1165 mlx5_eswitch_get_vport_metadata_for_match(peer_esw, 1166 vport)); 1167 } else { 1168 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1169 misc_parameters); 1170 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 1171 } 1172} 1173 1174static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, 1175 struct mlx5_core_dev *peer_dev) 1176{ 1177 struct mlx5_flow_destination dest = {}; 1178 struct mlx5_flow_act flow_act = {0}; 1179 struct mlx5_flow_handle **flows; 1180 /* total vports is the same for both e-switches */ 1181 int nvports = esw->total_vports; 1182 struct mlx5_flow_handle *flow; 1183 struct mlx5_flow_spec *spec; 1184 struct mlx5_vport *vport; 1185 int err, pfindex; 1186 unsigned long i; 1187 void *misc; 1188 1189 if (!MLX5_VPORT_MANAGER(esw->dev) && !mlx5_core_is_ecpf_esw_manager(esw->dev)) 1190 return 0; 1191 1192 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1193 if (!spec) 1194 return -ENOMEM; 1195 1196 peer_miss_rules_setup(esw, peer_dev, spec, &dest); 1197 1198 flows = kvcalloc(nvports, sizeof(*flows), GFP_KERNEL); 1199 if (!flows) { 1200 err = -ENOMEM; 1201 goto alloc_flows_err; 1202 } 1203 1204 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1205 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1206 misc_parameters); 1207 1208 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1209 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); 1210 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch, 1211 spec, MLX5_VPORT_PF); 1212 1213 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1214 spec, &flow_act, &dest, 1); 1215 if (IS_ERR(flow)) { 1216 err = PTR_ERR(flow); 1217 goto add_pf_flow_err; 1218 } 1219 flows[vport->index] = flow; 1220 } 1221 1222 if (mlx5_ecpf_vport_exists(esw->dev)) { 1223 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); 1224 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF); 1225 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1226 spec, &flow_act, &dest, 1); 1227 if (IS_ERR(flow)) { 1228 err = PTR_ERR(flow); 1229 goto add_ecpf_flow_err; 1230 } 1231 flows[vport->index] = flow; 1232 } 1233 1234 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) { 1235 esw_set_peer_miss_rule_source_port(esw, 1236 peer_dev->priv.eswitch, 1237 spec, vport->vport); 1238 1239 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1240 spec, &flow_act, &dest, 1); 1241 if (IS_ERR(flow)) { 1242 err = PTR_ERR(flow); 1243 goto add_vf_flow_err; 1244 } 1245 flows[vport->index] = flow; 1246 } 1247 1248 if (mlx5_core_ec_sriov_enabled(esw->dev)) { 1249 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) { 1250 if (i >= mlx5_core_max_ec_vfs(peer_dev)) 1251 break; 1252 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch, 1253 spec, vport->vport); 1254 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1255 spec, &flow_act, &dest, 1); 1256 if (IS_ERR(flow)) { 1257 err = PTR_ERR(flow); 1258 goto add_ec_vf_flow_err; 1259 } 1260 flows[vport->index] = flow; 1261 } 1262 } 1263 1264 pfindex = mlx5_get_dev_index(peer_dev); 1265 if (pfindex >= MLX5_MAX_PORTS) { 1266 esw_warn(esw->dev, "Peer dev index(%d) is over the max num defined(%d)\n", 1267 pfindex, MLX5_MAX_PORTS); 1268 err = -EINVAL; 1269 goto add_ec_vf_flow_err; 1270 } 1271 esw->fdb_table.offloads.peer_miss_rules[pfindex] = flows; 1272 1273 kvfree(spec); 1274 return 0; 1275 1276add_ec_vf_flow_err: 1277 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) { 1278 if (!flows[vport->index]) 1279 continue; 1280 mlx5_del_flow_rules(flows[vport->index]); 1281 } 1282add_vf_flow_err: 1283 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) { 1284 if (!flows[vport->index]) 1285 continue; 1286 mlx5_del_flow_rules(flows[vport->index]); 1287 } 1288 if (mlx5_ecpf_vport_exists(esw->dev)) { 1289 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); 1290 mlx5_del_flow_rules(flows[vport->index]); 1291 } 1292add_ecpf_flow_err: 1293 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1294 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); 1295 mlx5_del_flow_rules(flows[vport->index]); 1296 } 1297add_pf_flow_err: 1298 esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err); 1299 kvfree(flows); 1300alloc_flows_err: 1301 kvfree(spec); 1302 return err; 1303} 1304 1305static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw, 1306 struct mlx5_core_dev *peer_dev) 1307{ 1308 u16 peer_index = mlx5_get_dev_index(peer_dev); 1309 struct mlx5_flow_handle **flows; 1310 struct mlx5_vport *vport; 1311 unsigned long i; 1312 1313 flows = esw->fdb_table.offloads.peer_miss_rules[peer_index]; 1314 if (!flows) 1315 return; 1316 1317 if (mlx5_core_ec_sriov_enabled(esw->dev)) { 1318 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) { 1319 /* The flow for a particular vport could be NULL if the other ECPF 1320 * has fewer or no VFs enabled 1321 */ 1322 if (!flows[vport->index]) 1323 continue; 1324 mlx5_del_flow_rules(flows[vport->index]); 1325 } 1326 } 1327 1328 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) 1329 mlx5_del_flow_rules(flows[vport->index]); 1330 1331 if (mlx5_ecpf_vport_exists(esw->dev)) { 1332 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); 1333 mlx5_del_flow_rules(flows[vport->index]); 1334 } 1335 1336 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1337 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); 1338 mlx5_del_flow_rules(flows[vport->index]); 1339 } 1340 1341 kvfree(flows); 1342 esw->fdb_table.offloads.peer_miss_rules[peer_index] = NULL; 1343} 1344 1345static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw) 1346{ 1347 struct mlx5_flow_act flow_act = {0}; 1348 struct mlx5_flow_destination dest = {}; 1349 struct mlx5_flow_handle *flow_rule = NULL; 1350 struct mlx5_flow_spec *spec; 1351 void *headers_c; 1352 void *headers_v; 1353 int err = 0; 1354 u8 *dmac_c; 1355 u8 *dmac_v; 1356 1357 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1358 if (!spec) { 1359 err = -ENOMEM; 1360 goto out; 1361 } 1362 1363 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1364 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1365 outer_headers); 1366 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, 1367 outer_headers.dmac_47_16); 1368 dmac_c[0] = 0x01; 1369 1370 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1371 dest.vport.num = esw->manager_vport; 1372 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1373 1374 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1375 spec, &flow_act, &dest, 1); 1376 if (IS_ERR(flow_rule)) { 1377 err = PTR_ERR(flow_rule); 1378 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err); 1379 goto out; 1380 } 1381 1382 esw->fdb_table.offloads.miss_rule_uni = flow_rule; 1383 1384 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1385 outer_headers); 1386 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, 1387 outer_headers.dmac_47_16); 1388 dmac_v[0] = 0x01; 1389 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1390 spec, &flow_act, &dest, 1); 1391 if (IS_ERR(flow_rule)) { 1392 err = PTR_ERR(flow_rule); 1393 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err); 1394 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1395 goto out; 1396 } 1397 1398 esw->fdb_table.offloads.miss_rule_multi = flow_rule; 1399 1400out: 1401 kvfree(spec); 1402 return err; 1403} 1404 1405struct mlx5_flow_handle * 1406esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag) 1407{ 1408 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 1409 struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore; 1410 struct mlx5_flow_context *flow_context; 1411 struct mlx5_flow_handle *flow_rule; 1412 struct mlx5_flow_destination dest; 1413 struct mlx5_flow_spec *spec; 1414 void *misc; 1415 1416 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 1417 return ERR_PTR(-EOPNOTSUPP); 1418 1419 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1420 if (!spec) 1421 return ERR_PTR(-ENOMEM); 1422 1423 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1424 misc_parameters_2); 1425 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1426 ESW_REG_C0_USER_DATA_METADATA_MASK); 1427 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1428 misc_parameters_2); 1429 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag); 1430 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1431 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 1432 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 1433 flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id; 1434 1435 flow_context = &spec->flow_context; 1436 flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 1437 flow_context->flow_tag = tag; 1438 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1439 dest.ft = esw->offloads.ft_offloads; 1440 1441 flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 1442 kvfree(spec); 1443 1444 if (IS_ERR(flow_rule)) 1445 esw_warn(esw->dev, 1446 "Failed to create restore rule for tag: %d, err(%d)\n", 1447 tag, (int)PTR_ERR(flow_rule)); 1448 1449 return flow_rule; 1450} 1451 1452#define MAX_PF_SQ 256 1453#define MAX_SQ_NVPORTS 32 1454 1455void 1456mlx5_esw_set_flow_group_source_port(struct mlx5_eswitch *esw, 1457 u32 *flow_group_in, 1458 int match_params) 1459{ 1460 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1461 flow_group_in, 1462 match_criteria); 1463 1464 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1465 MLX5_SET(create_flow_group_in, flow_group_in, 1466 match_criteria_enable, 1467 MLX5_MATCH_MISC_PARAMETERS_2 | match_params); 1468 1469 MLX5_SET(fte_match_param, match_criteria, 1470 misc_parameters_2.metadata_reg_c_0, 1471 mlx5_eswitch_get_vport_metadata_mask()); 1472 } else { 1473 MLX5_SET(create_flow_group_in, flow_group_in, 1474 match_criteria_enable, 1475 MLX5_MATCH_MISC_PARAMETERS | match_params); 1476 1477 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1478 misc_parameters.source_port); 1479 } 1480} 1481 1482#if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 1483static void esw_vport_tbl_put(struct mlx5_eswitch *esw) 1484{ 1485 struct mlx5_vport_tbl_attr attr; 1486 struct mlx5_vport *vport; 1487 unsigned long i; 1488 1489 attr.chain = 0; 1490 attr.prio = 1; 1491 mlx5_esw_for_each_vport(esw, i, vport) { 1492 attr.vport = vport->vport; 1493 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 1494 mlx5_esw_vporttbl_put(esw, &attr); 1495 } 1496} 1497 1498static int esw_vport_tbl_get(struct mlx5_eswitch *esw) 1499{ 1500 struct mlx5_vport_tbl_attr attr; 1501 struct mlx5_flow_table *fdb; 1502 struct mlx5_vport *vport; 1503 unsigned long i; 1504 1505 attr.chain = 0; 1506 attr.prio = 1; 1507 mlx5_esw_for_each_vport(esw, i, vport) { 1508 attr.vport = vport->vport; 1509 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 1510 fdb = mlx5_esw_vporttbl_get(esw, &attr); 1511 if (IS_ERR(fdb)) 1512 goto out; 1513 } 1514 return 0; 1515 1516out: 1517 esw_vport_tbl_put(esw); 1518 return PTR_ERR(fdb); 1519} 1520 1521#define fdb_modify_header_fwd_to_table_supported(esw) \ 1522 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table)) 1523static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags) 1524{ 1525 struct mlx5_core_dev *dev = esw->dev; 1526 1527 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level)) 1528 *flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED; 1529 1530 if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) && 1531 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) { 1532 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1533 esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n"); 1534 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) { 1535 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1536 esw_warn(dev, "Tc chains and priorities offload aren't supported\n"); 1537 } else if (!fdb_modify_header_fwd_to_table_supported(esw)) { 1538 /* Disabled when ttl workaround is needed, e.g 1539 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig 1540 */ 1541 esw_warn(dev, 1542 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n"); 1543 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1544 } else { 1545 *flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1546 esw_info(dev, "Supported tc chains and prios offload\n"); 1547 } 1548 1549 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1550 *flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED; 1551} 1552 1553static int 1554esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1555{ 1556 struct mlx5_core_dev *dev = esw->dev; 1557 struct mlx5_flow_table *nf_ft, *ft; 1558 struct mlx5_chains_attr attr = {}; 1559 struct mlx5_fs_chains *chains; 1560 int err; 1561 1562 esw_init_chains_offload_flags(esw, &attr.flags); 1563 attr.ns = MLX5_FLOW_NAMESPACE_FDB; 1564 attr.max_grp_num = esw->params.large_group_num; 1565 attr.default_ft = miss_fdb; 1566 attr.mapping = esw->offloads.reg_c0_obj_pool; 1567 1568 chains = mlx5_chains_create(dev, &attr); 1569 if (IS_ERR(chains)) { 1570 err = PTR_ERR(chains); 1571 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err); 1572 return err; 1573 } 1574 mlx5_chains_print_info(chains); 1575 1576 esw->fdb_table.offloads.esw_chains_priv = chains; 1577 1578 /* Create tc_end_ft which is the always created ft chain */ 1579 nf_ft = mlx5_chains_get_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1580 1, 0); 1581 if (IS_ERR(nf_ft)) { 1582 err = PTR_ERR(nf_ft); 1583 goto nf_ft_err; 1584 } 1585 1586 /* Always open the root for fast path */ 1587 ft = mlx5_chains_get_table(chains, 0, 1, 0); 1588 if (IS_ERR(ft)) { 1589 err = PTR_ERR(ft); 1590 goto level_0_err; 1591 } 1592 1593 /* Open level 1 for split fdb rules now if prios isn't supported */ 1594 if (!mlx5_chains_prios_supported(chains)) { 1595 err = esw_vport_tbl_get(esw); 1596 if (err) 1597 goto level_1_err; 1598 } 1599 1600 mlx5_chains_set_end_ft(chains, nf_ft); 1601 1602 return 0; 1603 1604level_1_err: 1605 mlx5_chains_put_table(chains, 0, 1, 0); 1606level_0_err: 1607 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1608nf_ft_err: 1609 mlx5_chains_destroy(chains); 1610 esw->fdb_table.offloads.esw_chains_priv = NULL; 1611 1612 return err; 1613} 1614 1615static void 1616esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1617{ 1618 if (!mlx5_chains_prios_supported(chains)) 1619 esw_vport_tbl_put(esw); 1620 mlx5_chains_put_table(chains, 0, 1, 0); 1621 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1622 mlx5_chains_destroy(chains); 1623} 1624 1625#else /* CONFIG_MLX5_CLS_ACT */ 1626 1627static int 1628esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1629{ return 0; } 1630 1631static void 1632esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1633{} 1634 1635#endif 1636 1637static int 1638esw_create_send_to_vport_group(struct mlx5_eswitch *esw, 1639 struct mlx5_flow_table *fdb, 1640 u32 *flow_group_in, 1641 int *ix) 1642{ 1643 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1644 struct mlx5_flow_group *g; 1645 void *match_criteria; 1646 int count, err = 0; 1647 1648 memset(flow_group_in, 0, inlen); 1649 1650 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, MLX5_MATCH_MISC_PARAMETERS); 1651 1652 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1653 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn); 1654 1655 if (!mlx5_eswitch_vport_match_metadata_enabled(esw) && 1656 MLX5_CAP_ESW(esw->dev, merged_eswitch)) { 1657 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1658 misc_parameters.source_eswitch_owner_vhca_id); 1659 MLX5_SET(create_flow_group_in, flow_group_in, 1660 source_eswitch_owner_vhca_id_valid, 1); 1661 } 1662 1663 /* See comment at table_size calculation */ 1664 count = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ); 1665 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1666 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, *ix + count - 1); 1667 *ix += count; 1668 1669 g = mlx5_create_flow_group(fdb, flow_group_in); 1670 if (IS_ERR(g)) { 1671 err = PTR_ERR(g); 1672 esw_warn(esw->dev, "Failed to create send-to-vport flow group err(%d)\n", err); 1673 goto out; 1674 } 1675 esw->fdb_table.offloads.send_to_vport_grp = g; 1676 1677out: 1678 return err; 1679} 1680 1681static int 1682esw_create_meta_send_to_vport_group(struct mlx5_eswitch *esw, 1683 struct mlx5_flow_table *fdb, 1684 u32 *flow_group_in, 1685 int *ix) 1686{ 1687 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1688 struct mlx5_flow_group *g; 1689 void *match_criteria; 1690 int err = 0; 1691 1692 if (!esw_src_port_rewrite_supported(esw)) 1693 return 0; 1694 1695 memset(flow_group_in, 0, inlen); 1696 1697 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1698 MLX5_MATCH_MISC_PARAMETERS_2); 1699 1700 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1701 1702 MLX5_SET(fte_match_param, match_criteria, 1703 misc_parameters_2.metadata_reg_c_0, 1704 mlx5_eswitch_get_vport_metadata_mask()); 1705 MLX5_SET(fte_match_param, match_criteria, 1706 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); 1707 1708 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1709 MLX5_SET(create_flow_group_in, flow_group_in, 1710 end_flow_index, *ix + esw->total_vports - 1); 1711 *ix += esw->total_vports; 1712 1713 g = mlx5_create_flow_group(fdb, flow_group_in); 1714 if (IS_ERR(g)) { 1715 err = PTR_ERR(g); 1716 esw_warn(esw->dev, 1717 "Failed to create send-to-vport meta flow group err(%d)\n", err); 1718 goto send_vport_meta_err; 1719 } 1720 esw->fdb_table.offloads.send_to_vport_meta_grp = g; 1721 1722 return 0; 1723 1724send_vport_meta_err: 1725 return err; 1726} 1727 1728static int 1729esw_create_peer_esw_miss_group(struct mlx5_eswitch *esw, 1730 struct mlx5_flow_table *fdb, 1731 u32 *flow_group_in, 1732 int *ix) 1733{ 1734 int max_peer_ports = (esw->total_vports - 1) * (MLX5_MAX_PORTS - 1); 1735 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1736 struct mlx5_flow_group *g; 1737 void *match_criteria; 1738 int err = 0; 1739 1740 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1741 return 0; 1742 1743 memset(flow_group_in, 0, inlen); 1744 1745 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, 0); 1746 1747 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1748 match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1749 flow_group_in, 1750 match_criteria); 1751 1752 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1753 misc_parameters.source_eswitch_owner_vhca_id); 1754 1755 MLX5_SET(create_flow_group_in, flow_group_in, 1756 source_eswitch_owner_vhca_id_valid, 1); 1757 } 1758 1759 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1760 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1761 *ix + max_peer_ports); 1762 *ix += max_peer_ports + 1; 1763 1764 g = mlx5_create_flow_group(fdb, flow_group_in); 1765 if (IS_ERR(g)) { 1766 err = PTR_ERR(g); 1767 esw_warn(esw->dev, "Failed to create peer miss flow group err(%d)\n", err); 1768 goto out; 1769 } 1770 esw->fdb_table.offloads.peer_miss_grp = g; 1771 1772out: 1773 return err; 1774} 1775 1776static int 1777esw_create_miss_group(struct mlx5_eswitch *esw, 1778 struct mlx5_flow_table *fdb, 1779 u32 *flow_group_in, 1780 int *ix) 1781{ 1782 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1783 struct mlx5_flow_group *g; 1784 void *match_criteria; 1785 int err = 0; 1786 u8 *dmac; 1787 1788 memset(flow_group_in, 0, inlen); 1789 1790 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1791 MLX5_MATCH_OUTER_HEADERS); 1792 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 1793 match_criteria); 1794 dmac = MLX5_ADDR_OF(fte_match_param, match_criteria, 1795 outer_headers.dmac_47_16); 1796 dmac[0] = 0x01; 1797 1798 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1799 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1800 *ix + MLX5_ESW_MISS_FLOWS); 1801 1802 g = mlx5_create_flow_group(fdb, flow_group_in); 1803 if (IS_ERR(g)) { 1804 err = PTR_ERR(g); 1805 esw_warn(esw->dev, "Failed to create miss flow group err(%d)\n", err); 1806 goto miss_err; 1807 } 1808 esw->fdb_table.offloads.miss_grp = g; 1809 1810 err = esw_add_fdb_miss_rule(esw); 1811 if (err) 1812 goto miss_rule_err; 1813 1814 return 0; 1815 1816miss_rule_err: 1817 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1818miss_err: 1819 return err; 1820} 1821 1822static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw) 1823{ 1824 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1825 struct mlx5_flow_table_attr ft_attr = {}; 1826 struct mlx5_core_dev *dev = esw->dev; 1827 struct mlx5_flow_namespace *root_ns; 1828 struct mlx5_flow_table *fdb = NULL; 1829 int table_size, ix = 0, err = 0; 1830 u32 flags = 0, *flow_group_in; 1831 1832 esw_debug(esw->dev, "Create offloads FDB Tables\n"); 1833 1834 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1835 if (!flow_group_in) 1836 return -ENOMEM; 1837 1838 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); 1839 if (!root_ns) { 1840 esw_warn(dev, "Failed to get FDB flow namespace\n"); 1841 err = -EOPNOTSUPP; 1842 goto ns_err; 1843 } 1844 esw->fdb_table.offloads.ns = root_ns; 1845 err = mlx5_flow_namespace_set_mode(root_ns, 1846 esw->dev->priv.steering->mode); 1847 if (err) { 1848 esw_warn(dev, "Failed to set FDB namespace steering mode\n"); 1849 goto ns_err; 1850 } 1851 1852 /* To be strictly correct: 1853 * MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) 1854 * should be: 1855 * esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ + 1856 * peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ 1857 * but as the peer device might not be in switchdev mode it's not 1858 * possible. We use the fact that by default FW sets max vfs and max sfs 1859 * to the same value on both devices. If it needs to be changed in the future note 1860 * the peer miss group should also be created based on the number of 1861 * total vports of the peer (currently is also uses esw->total_vports). 1862 */ 1863 table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) + 1864 esw->total_vports * MLX5_MAX_PORTS + MLX5_ESW_MISS_FLOWS; 1865 1866 /* create the slow path fdb with encap set, so further table instances 1867 * can be created at run time while VFs are probed if the FW allows that. 1868 */ 1869 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1870 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT | 1871 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); 1872 1873 ft_attr.flags = flags; 1874 ft_attr.max_fte = table_size; 1875 ft_attr.prio = FDB_SLOW_PATH; 1876 1877 fdb = mlx5_create_flow_table(root_ns, &ft_attr); 1878 if (IS_ERR(fdb)) { 1879 err = PTR_ERR(fdb); 1880 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err); 1881 goto slow_fdb_err; 1882 } 1883 esw->fdb_table.offloads.slow_fdb = fdb; 1884 1885 /* Create empty TC-miss managed table. This allows plugging in following 1886 * priorities without directly exposing their level 0 table to 1887 * eswitch_offloads and passing it as miss_fdb to following call to 1888 * esw_chains_create(). 1889 */ 1890 memset(&ft_attr, 0, sizeof(ft_attr)); 1891 ft_attr.prio = FDB_TC_MISS; 1892 esw->fdb_table.offloads.tc_miss_table = mlx5_create_flow_table(root_ns, &ft_attr); 1893 if (IS_ERR(esw->fdb_table.offloads.tc_miss_table)) { 1894 err = PTR_ERR(esw->fdb_table.offloads.tc_miss_table); 1895 esw_warn(dev, "Failed to create TC miss FDB Table err %d\n", err); 1896 goto tc_miss_table_err; 1897 } 1898 1899 err = esw_chains_create(esw, esw->fdb_table.offloads.tc_miss_table); 1900 if (err) { 1901 esw_warn(dev, "Failed to open fdb chains err(%d)\n", err); 1902 goto fdb_chains_err; 1903 } 1904 1905 err = esw_create_send_to_vport_group(esw, fdb, flow_group_in, &ix); 1906 if (err) 1907 goto send_vport_err; 1908 1909 err = esw_create_meta_send_to_vport_group(esw, fdb, flow_group_in, &ix); 1910 if (err) 1911 goto send_vport_meta_err; 1912 1913 err = esw_create_peer_esw_miss_group(esw, fdb, flow_group_in, &ix); 1914 if (err) 1915 goto peer_miss_err; 1916 1917 err = esw_create_miss_group(esw, fdb, flow_group_in, &ix); 1918 if (err) 1919 goto miss_err; 1920 1921 kvfree(flow_group_in); 1922 return 0; 1923 1924miss_err: 1925 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1926 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 1927peer_miss_err: 1928 if (esw->fdb_table.offloads.send_to_vport_meta_grp) 1929 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); 1930send_vport_meta_err: 1931 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 1932send_vport_err: 1933 esw_chains_destroy(esw, esw_chains(esw)); 1934fdb_chains_err: 1935 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); 1936tc_miss_table_err: 1937 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw)); 1938slow_fdb_err: 1939 /* Holds true only as long as DMFS is the default */ 1940 mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS); 1941ns_err: 1942 kvfree(flow_group_in); 1943 return err; 1944} 1945 1946static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) 1947{ 1948 if (!mlx5_eswitch_get_slow_fdb(esw)) 1949 return; 1950 1951 esw_debug(esw->dev, "Destroy offloads FDB Tables\n"); 1952 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi); 1953 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1954 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 1955 if (esw->fdb_table.offloads.send_to_vport_meta_grp) 1956 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); 1957 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1958 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 1959 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1960 1961 esw_chains_destroy(esw, esw_chains(esw)); 1962 1963 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); 1964 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw)); 1965 /* Holds true only as long as DMFS is the default */ 1966 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns, 1967 MLX5_FLOW_STEERING_MODE_DMFS); 1968 atomic64_set(&esw->user_count, 0); 1969} 1970 1971static int esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch *esw) 1972{ 1973 int nvports; 1974 1975 nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS; 1976 if (mlx5e_tc_int_port_supported(esw)) 1977 nvports += MLX5E_TC_MAX_INT_PORT_NUM; 1978 1979 return nvports; 1980} 1981 1982static int esw_create_offloads_table(struct mlx5_eswitch *esw) 1983{ 1984 struct mlx5_flow_table_attr ft_attr = {}; 1985 struct mlx5_core_dev *dev = esw->dev; 1986 struct mlx5_flow_table *ft_offloads; 1987 struct mlx5_flow_namespace *ns; 1988 int err = 0; 1989 1990 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 1991 if (!ns) { 1992 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 1993 return -EOPNOTSUPP; 1994 } 1995 1996 ft_attr.max_fte = esw_get_nr_ft_offloads_steering_src_ports(esw) + 1997 MLX5_ESW_FT_OFFLOADS_DROP_RULE; 1998 ft_attr.prio = 1; 1999 2000 ft_offloads = mlx5_create_flow_table(ns, &ft_attr); 2001 if (IS_ERR(ft_offloads)) { 2002 err = PTR_ERR(ft_offloads); 2003 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err); 2004 return err; 2005 } 2006 2007 esw->offloads.ft_offloads = ft_offloads; 2008 return 0; 2009} 2010 2011static void esw_destroy_offloads_table(struct mlx5_eswitch *esw) 2012{ 2013 struct mlx5_esw_offload *offloads = &esw->offloads; 2014 2015 mlx5_destroy_flow_table(offloads->ft_offloads); 2016} 2017 2018static int esw_create_vport_rx_group(struct mlx5_eswitch *esw) 2019{ 2020 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2021 struct mlx5_flow_group *g; 2022 u32 *flow_group_in; 2023 int nvports; 2024 int err = 0; 2025 2026 nvports = esw_get_nr_ft_offloads_steering_src_ports(esw); 2027 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2028 if (!flow_group_in) 2029 return -ENOMEM; 2030 2031 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, 0); 2032 2033 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2034 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1); 2035 2036 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); 2037 2038 if (IS_ERR(g)) { 2039 err = PTR_ERR(g); 2040 mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err); 2041 goto out; 2042 } 2043 2044 esw->offloads.vport_rx_group = g; 2045out: 2046 kvfree(flow_group_in); 2047 return err; 2048} 2049 2050static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw) 2051{ 2052 mlx5_destroy_flow_group(esw->offloads.vport_rx_group); 2053} 2054 2055static int esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch *esw) 2056{ 2057 /* ft_offloads table is enlarged by MLX5_ESW_FT_OFFLOADS_DROP_RULE (1) 2058 * for the drop rule, which is placed at the end of the table. 2059 * So return the total of vport and int_port as rule index. 2060 */ 2061 return esw_get_nr_ft_offloads_steering_src_ports(esw); 2062} 2063 2064static int esw_create_vport_rx_drop_group(struct mlx5_eswitch *esw) 2065{ 2066 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2067 struct mlx5_flow_group *g; 2068 u32 *flow_group_in; 2069 int flow_index; 2070 int err = 0; 2071 2072 flow_index = esw_create_vport_rx_drop_rule_index(esw); 2073 2074 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2075 if (!flow_group_in) 2076 return -ENOMEM; 2077 2078 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_index); 2079 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_index); 2080 2081 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); 2082 2083 if (IS_ERR(g)) { 2084 err = PTR_ERR(g); 2085 mlx5_core_warn(esw->dev, "Failed to create vport rx drop group err %d\n", err); 2086 goto out; 2087 } 2088 2089 esw->offloads.vport_rx_drop_group = g; 2090out: 2091 kvfree(flow_group_in); 2092 return err; 2093} 2094 2095static void esw_destroy_vport_rx_drop_group(struct mlx5_eswitch *esw) 2096{ 2097 if (esw->offloads.vport_rx_drop_group) 2098 mlx5_destroy_flow_group(esw->offloads.vport_rx_drop_group); 2099} 2100 2101void 2102mlx5_esw_set_spec_source_port(struct mlx5_eswitch *esw, 2103 u16 vport, 2104 struct mlx5_flow_spec *spec) 2105{ 2106 void *misc; 2107 2108 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 2109 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 2110 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2111 mlx5_eswitch_get_vport_metadata_for_match(esw, vport)); 2112 2113 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 2114 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2115 mlx5_eswitch_get_vport_metadata_mask()); 2116 2117 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 2118 } else { 2119 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 2120 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 2121 2122 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 2123 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 2124 2125 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 2126 } 2127} 2128 2129struct mlx5_flow_handle * 2130mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport, 2131 struct mlx5_flow_destination *dest) 2132{ 2133 struct mlx5_flow_act flow_act = {0}; 2134 struct mlx5_flow_handle *flow_rule; 2135 struct mlx5_flow_spec *spec; 2136 2137 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2138 if (!spec) { 2139 flow_rule = ERR_PTR(-ENOMEM); 2140 goto out; 2141 } 2142 2143 mlx5_esw_set_spec_source_port(esw, vport, spec); 2144 2145 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2146 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec, 2147 &flow_act, dest, 1); 2148 if (IS_ERR(flow_rule)) { 2149 esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule)); 2150 goto out; 2151 } 2152 2153out: 2154 kvfree(spec); 2155 return flow_rule; 2156} 2157 2158static int esw_create_vport_rx_drop_rule(struct mlx5_eswitch *esw) 2159{ 2160 struct mlx5_flow_act flow_act = {}; 2161 struct mlx5_flow_handle *flow_rule; 2162 2163 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; 2164 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, NULL, 2165 &flow_act, NULL, 0); 2166 if (IS_ERR(flow_rule)) { 2167 esw_warn(esw->dev, 2168 "fs offloads: Failed to add vport rx drop rule err %ld\n", 2169 PTR_ERR(flow_rule)); 2170 return PTR_ERR(flow_rule); 2171 } 2172 2173 esw->offloads.vport_rx_drop_rule = flow_rule; 2174 2175 return 0; 2176} 2177 2178static void esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch *esw) 2179{ 2180 if (esw->offloads.vport_rx_drop_rule) 2181 mlx5_del_flow_rules(esw->offloads.vport_rx_drop_rule); 2182} 2183 2184static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode) 2185{ 2186 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2; 2187 struct mlx5_core_dev *dev = esw->dev; 2188 struct mlx5_vport *vport; 2189 unsigned long i; 2190 2191 if (!MLX5_CAP_GEN(dev, vport_group_manager)) 2192 return -EOPNOTSUPP; 2193 2194 if (!mlx5_esw_is_fdb_created(esw)) 2195 return -EOPNOTSUPP; 2196 2197 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 2198 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 2199 mlx5_mode = MLX5_INLINE_MODE_NONE; 2200 goto out; 2201 case MLX5_CAP_INLINE_MODE_L2: 2202 mlx5_mode = MLX5_INLINE_MODE_L2; 2203 goto out; 2204 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 2205 goto query_vports; 2206 } 2207 2208query_vports: 2209 mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode); 2210 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 2211 mlx5_query_nic_vport_min_inline(dev, vport->vport, &mlx5_mode); 2212 if (prev_mlx5_mode != mlx5_mode) 2213 return -EINVAL; 2214 prev_mlx5_mode = mlx5_mode; 2215 } 2216 2217out: 2218 *mode = mlx5_mode; 2219 return 0; 2220} 2221 2222static void esw_destroy_restore_table(struct mlx5_eswitch *esw) 2223{ 2224 struct mlx5_esw_offload *offloads = &esw->offloads; 2225 2226 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 2227 return; 2228 2229 mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id); 2230 mlx5_destroy_flow_group(offloads->restore_group); 2231 mlx5_destroy_flow_table(offloads->ft_offloads_restore); 2232} 2233 2234static int esw_create_restore_table(struct mlx5_eswitch *esw) 2235{ 2236 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 2237 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2238 struct mlx5_flow_table_attr ft_attr = {}; 2239 struct mlx5_core_dev *dev = esw->dev; 2240 struct mlx5_flow_namespace *ns; 2241 struct mlx5_modify_hdr *mod_hdr; 2242 void *match_criteria, *misc; 2243 struct mlx5_flow_table *ft; 2244 struct mlx5_flow_group *g; 2245 u32 *flow_group_in; 2246 int err = 0; 2247 2248 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 2249 return 0; 2250 2251 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 2252 if (!ns) { 2253 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 2254 return -EOPNOTSUPP; 2255 } 2256 2257 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2258 if (!flow_group_in) { 2259 err = -ENOMEM; 2260 goto out_free; 2261 } 2262 2263 ft_attr.max_fte = 1 << ESW_REG_C0_USER_DATA_METADATA_BITS; 2264 ft = mlx5_create_flow_table(ns, &ft_attr); 2265 if (IS_ERR(ft)) { 2266 err = PTR_ERR(ft); 2267 esw_warn(esw->dev, "Failed to create restore table, err %d\n", 2268 err); 2269 goto out_free; 2270 } 2271 2272 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 2273 match_criteria); 2274 misc = MLX5_ADDR_OF(fte_match_param, match_criteria, 2275 misc_parameters_2); 2276 2277 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2278 ESW_REG_C0_USER_DATA_METADATA_MASK); 2279 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2280 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 2281 ft_attr.max_fte - 1); 2282 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 2283 MLX5_MATCH_MISC_PARAMETERS_2); 2284 g = mlx5_create_flow_group(ft, flow_group_in); 2285 if (IS_ERR(g)) { 2286 err = PTR_ERR(g); 2287 esw_warn(dev, "Failed to create restore flow group, err: %d\n", 2288 err); 2289 goto err_group; 2290 } 2291 2292 MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY); 2293 MLX5_SET(copy_action_in, modact, src_field, 2294 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1); 2295 MLX5_SET(copy_action_in, modact, dst_field, 2296 MLX5_ACTION_IN_FIELD_METADATA_REG_B); 2297 mod_hdr = mlx5_modify_header_alloc(esw->dev, 2298 MLX5_FLOW_NAMESPACE_KERNEL, 1, 2299 modact); 2300 if (IS_ERR(mod_hdr)) { 2301 err = PTR_ERR(mod_hdr); 2302 esw_warn(dev, "Failed to create restore mod header, err: %d\n", 2303 err); 2304 goto err_mod_hdr; 2305 } 2306 2307 esw->offloads.ft_offloads_restore = ft; 2308 esw->offloads.restore_group = g; 2309 esw->offloads.restore_copy_hdr_id = mod_hdr; 2310 2311 kvfree(flow_group_in); 2312 2313 return 0; 2314 2315err_mod_hdr: 2316 mlx5_destroy_flow_group(g); 2317err_group: 2318 mlx5_destroy_flow_table(ft); 2319out_free: 2320 kvfree(flow_group_in); 2321 2322 return err; 2323} 2324 2325static int esw_offloads_start(struct mlx5_eswitch *esw, 2326 struct netlink_ext_ack *extack) 2327{ 2328 int err; 2329 2330 esw->mode = MLX5_ESWITCH_OFFLOADS; 2331 err = mlx5_eswitch_enable_locked(esw, esw->dev->priv.sriov.num_vfs); 2332 if (err) { 2333 NL_SET_ERR_MSG_MOD(extack, 2334 "Failed setting eswitch to offloads"); 2335 esw->mode = MLX5_ESWITCH_LEGACY; 2336 mlx5_rescan_drivers(esw->dev); 2337 return err; 2338 } 2339 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) { 2340 if (mlx5_eswitch_inline_mode_get(esw, 2341 &esw->offloads.inline_mode)) { 2342 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2; 2343 NL_SET_ERR_MSG_MOD(extack, 2344 "Inline mode is different between vports"); 2345 } 2346 } 2347 return 0; 2348} 2349 2350static int mlx5_esw_offloads_rep_init(struct mlx5_eswitch *esw, const struct mlx5_vport *vport) 2351{ 2352 struct mlx5_eswitch_rep *rep; 2353 int rep_type; 2354 int err; 2355 2356 rep = kzalloc(sizeof(*rep), GFP_KERNEL); 2357 if (!rep) 2358 return -ENOMEM; 2359 2360 rep->vport = vport->vport; 2361 rep->vport_index = vport->index; 2362 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) 2363 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED); 2364 2365 err = xa_insert(&esw->offloads.vport_reps, rep->vport, rep, GFP_KERNEL); 2366 if (err) 2367 goto insert_err; 2368 2369 return 0; 2370 2371insert_err: 2372 kfree(rep); 2373 return err; 2374} 2375 2376static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch *esw, 2377 struct mlx5_eswitch_rep *rep) 2378{ 2379 xa_erase(&esw->offloads.vport_reps, rep->vport); 2380 kfree(rep); 2381} 2382 2383static void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw) 2384{ 2385 struct mlx5_eswitch_rep *rep; 2386 unsigned long i; 2387 2388 mlx5_esw_for_each_rep(esw, i, rep) 2389 mlx5_esw_offloads_rep_cleanup(esw, rep); 2390 xa_destroy(&esw->offloads.vport_reps); 2391} 2392 2393static int esw_offloads_init_reps(struct mlx5_eswitch *esw) 2394{ 2395 struct mlx5_vport *vport; 2396 unsigned long i; 2397 int err; 2398 2399 xa_init(&esw->offloads.vport_reps); 2400 2401 mlx5_esw_for_each_vport(esw, i, vport) { 2402 err = mlx5_esw_offloads_rep_init(esw, vport); 2403 if (err) 2404 goto err; 2405 } 2406 return 0; 2407 2408err: 2409 esw_offloads_cleanup_reps(esw); 2410 return err; 2411} 2412 2413static int esw_port_metadata_set(struct devlink *devlink, u32 id, 2414 struct devlink_param_gset_ctx *ctx) 2415{ 2416 struct mlx5_core_dev *dev = devlink_priv(devlink); 2417 struct mlx5_eswitch *esw = dev->priv.eswitch; 2418 int err = 0; 2419 2420 down_write(&esw->mode_lock); 2421 if (mlx5_esw_is_fdb_created(esw)) { 2422 err = -EBUSY; 2423 goto done; 2424 } 2425 if (!mlx5_esw_vport_match_metadata_supported(esw)) { 2426 err = -EOPNOTSUPP; 2427 goto done; 2428 } 2429 if (ctx->val.vbool) 2430 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA; 2431 else 2432 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA; 2433done: 2434 up_write(&esw->mode_lock); 2435 return err; 2436} 2437 2438static int esw_port_metadata_get(struct devlink *devlink, u32 id, 2439 struct devlink_param_gset_ctx *ctx) 2440{ 2441 struct mlx5_core_dev *dev = devlink_priv(devlink); 2442 2443 ctx->val.vbool = mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch); 2444 return 0; 2445} 2446 2447static int esw_port_metadata_validate(struct devlink *devlink, u32 id, 2448 union devlink_param_value val, 2449 struct netlink_ext_ack *extack) 2450{ 2451 struct mlx5_core_dev *dev = devlink_priv(devlink); 2452 u8 esw_mode; 2453 2454 esw_mode = mlx5_eswitch_mode(dev); 2455 if (esw_mode == MLX5_ESWITCH_OFFLOADS) { 2456 NL_SET_ERR_MSG_MOD(extack, 2457 "E-Switch must either disabled or non switchdev mode"); 2458 return -EBUSY; 2459 } 2460 return 0; 2461} 2462 2463static const struct devlink_param esw_devlink_params[] = { 2464 DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_ESW_PORT_METADATA, 2465 "esw_port_metadata", DEVLINK_PARAM_TYPE_BOOL, 2466 BIT(DEVLINK_PARAM_CMODE_RUNTIME), 2467 esw_port_metadata_get, 2468 esw_port_metadata_set, 2469 esw_port_metadata_validate), 2470}; 2471 2472int esw_offloads_init(struct mlx5_eswitch *esw) 2473{ 2474 int err; 2475 2476 err = esw_offloads_init_reps(esw); 2477 if (err) 2478 return err; 2479 2480 if (MLX5_ESWITCH_MANAGER(esw->dev) && 2481 mlx5_esw_vport_match_metadata_supported(esw)) 2482 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA; 2483 2484 err = devl_params_register(priv_to_devlink(esw->dev), 2485 esw_devlink_params, 2486 ARRAY_SIZE(esw_devlink_params)); 2487 if (err) 2488 goto err_params; 2489 2490 return 0; 2491 2492err_params: 2493 esw_offloads_cleanup_reps(esw); 2494 return err; 2495} 2496 2497void esw_offloads_cleanup(struct mlx5_eswitch *esw) 2498{ 2499 devl_params_unregister(priv_to_devlink(esw->dev), 2500 esw_devlink_params, 2501 ARRAY_SIZE(esw_devlink_params)); 2502 esw_offloads_cleanup_reps(esw); 2503} 2504 2505static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw, 2506 struct mlx5_eswitch_rep *rep, u8 rep_type) 2507{ 2508 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 2509 REP_LOADED, REP_REGISTERED) == REP_LOADED) 2510 esw->offloads.rep_ops[rep_type]->unload(rep); 2511} 2512 2513static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type) 2514{ 2515 struct mlx5_eswitch_rep *rep; 2516 unsigned long i; 2517 2518 mlx5_esw_for_each_rep(esw, i, rep) 2519 __esw_offloads_unload_rep(esw, rep, rep_type); 2520} 2521 2522static int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num) 2523{ 2524 struct mlx5_eswitch_rep *rep; 2525 int rep_type; 2526 int err; 2527 2528 rep = mlx5_eswitch_get_rep(esw, vport_num); 2529 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) 2530 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 2531 REP_REGISTERED, REP_LOADED) == REP_REGISTERED) { 2532 err = esw->offloads.rep_ops[rep_type]->load(esw->dev, rep); 2533 if (err) 2534 goto err_reps; 2535 } 2536 2537 return 0; 2538 2539err_reps: 2540 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED); 2541 for (--rep_type; rep_type >= 0; rep_type--) 2542 __esw_offloads_unload_rep(esw, rep, rep_type); 2543 return err; 2544} 2545 2546static void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num) 2547{ 2548 struct mlx5_eswitch_rep *rep; 2549 int rep_type; 2550 2551 rep = mlx5_eswitch_get_rep(esw, vport_num); 2552 for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--) 2553 __esw_offloads_unload_rep(esw, rep, rep_type); 2554} 2555 2556int mlx5_esw_offloads_init_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2557{ 2558 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2559 return 0; 2560 2561 return mlx5_esw_offloads_pf_vf_devlink_port_init(esw, vport); 2562} 2563 2564void mlx5_esw_offloads_cleanup_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2565{ 2566 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2567 return; 2568 2569 mlx5_esw_offloads_pf_vf_devlink_port_cleanup(esw, vport); 2570} 2571 2572int mlx5_esw_offloads_init_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport, 2573 struct mlx5_devlink_port *dl_port, 2574 u32 controller, u32 sfnum) 2575{ 2576 return mlx5_esw_offloads_sf_devlink_port_init(esw, vport, dl_port, controller, sfnum); 2577} 2578 2579void mlx5_esw_offloads_cleanup_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2580{ 2581 mlx5_esw_offloads_sf_devlink_port_cleanup(esw, vport); 2582} 2583 2584int mlx5_esw_offloads_load_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2585{ 2586 int err; 2587 2588 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2589 return 0; 2590 2591 err = mlx5_esw_offloads_devlink_port_register(esw, vport); 2592 if (err) 2593 return err; 2594 2595 err = mlx5_esw_offloads_rep_load(esw, vport->vport); 2596 if (err) 2597 goto load_err; 2598 return err; 2599 2600load_err: 2601 mlx5_esw_offloads_devlink_port_unregister(esw, vport); 2602 return err; 2603} 2604 2605void mlx5_esw_offloads_unload_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2606{ 2607 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2608 return; 2609 2610 mlx5_esw_offloads_rep_unload(esw, vport->vport); 2611 2612 mlx5_esw_offloads_devlink_port_unregister(esw, vport); 2613} 2614 2615static int esw_set_slave_root_fdb(struct mlx5_core_dev *master, 2616 struct mlx5_core_dev *slave) 2617{ 2618 u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {}; 2619 u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {}; 2620 struct mlx5_flow_root_namespace *root; 2621 struct mlx5_flow_namespace *ns; 2622 int err; 2623 2624 MLX5_SET(set_flow_table_root_in, in, opcode, 2625 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT); 2626 MLX5_SET(set_flow_table_root_in, in, table_type, 2627 FS_FT_FDB); 2628 2629 if (master) { 2630 ns = mlx5_get_flow_namespace(master, 2631 MLX5_FLOW_NAMESPACE_FDB); 2632 root = find_root(&ns->node); 2633 mutex_lock(&root->chain_lock); 2634 MLX5_SET(set_flow_table_root_in, in, 2635 table_eswitch_owner_vhca_id_valid, 1); 2636 MLX5_SET(set_flow_table_root_in, in, 2637 table_eswitch_owner_vhca_id, 2638 MLX5_CAP_GEN(master, vhca_id)); 2639 MLX5_SET(set_flow_table_root_in, in, table_id, 2640 root->root_ft->id); 2641 } else { 2642 ns = mlx5_get_flow_namespace(slave, 2643 MLX5_FLOW_NAMESPACE_FDB); 2644 root = find_root(&ns->node); 2645 mutex_lock(&root->chain_lock); 2646 MLX5_SET(set_flow_table_root_in, in, table_id, 2647 root->root_ft->id); 2648 } 2649 2650 err = mlx5_cmd_exec(slave, in, sizeof(in), out, sizeof(out)); 2651 mutex_unlock(&root->chain_lock); 2652 2653 return err; 2654} 2655 2656static int __esw_set_master_egress_rule(struct mlx5_core_dev *master, 2657 struct mlx5_core_dev *slave, 2658 struct mlx5_vport *vport, 2659 struct mlx5_flow_table *acl) 2660{ 2661 u16 slave_index = MLX5_CAP_GEN(slave, vhca_id); 2662 struct mlx5_flow_handle *flow_rule = NULL; 2663 struct mlx5_flow_destination dest = {}; 2664 struct mlx5_flow_act flow_act = {}; 2665 struct mlx5_flow_spec *spec; 2666 int err = 0; 2667 void *misc; 2668 2669 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2670 if (!spec) 2671 return -ENOMEM; 2672 2673 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 2674 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 2675 misc_parameters); 2676 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_UPLINK); 2677 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, slave_index); 2678 2679 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 2680 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 2681 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 2682 source_eswitch_owner_vhca_id); 2683 2684 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2685 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 2686 dest.vport.num = slave->priv.eswitch->manager_vport; 2687 dest.vport.vhca_id = MLX5_CAP_GEN(slave, vhca_id); 2688 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 2689 2690 flow_rule = mlx5_add_flow_rules(acl, spec, &flow_act, 2691 &dest, 1); 2692 if (IS_ERR(flow_rule)) { 2693 err = PTR_ERR(flow_rule); 2694 } else { 2695 err = xa_insert(&vport->egress.offloads.bounce_rules, 2696 slave_index, flow_rule, GFP_KERNEL); 2697 if (err) 2698 mlx5_del_flow_rules(flow_rule); 2699 } 2700 2701 kvfree(spec); 2702 return err; 2703} 2704 2705static int esw_master_egress_create_resources(struct mlx5_eswitch *esw, 2706 struct mlx5_flow_namespace *egress_ns, 2707 struct mlx5_vport *vport, size_t count) 2708{ 2709 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2710 struct mlx5_flow_table_attr ft_attr = { 2711 .max_fte = count, .prio = 0, .level = 0, 2712 }; 2713 struct mlx5_flow_table *acl; 2714 struct mlx5_flow_group *g; 2715 void *match_criteria; 2716 u32 *flow_group_in; 2717 int err; 2718 2719 if (vport->egress.acl) 2720 return 0; 2721 2722 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2723 if (!flow_group_in) 2724 return -ENOMEM; 2725 2726 if (vport->vport || mlx5_core_is_ecpf(esw->dev)) 2727 ft_attr.flags = MLX5_FLOW_TABLE_OTHER_VPORT; 2728 2729 acl = mlx5_create_vport_flow_table(egress_ns, &ft_attr, vport->vport); 2730 if (IS_ERR(acl)) { 2731 err = PTR_ERR(acl); 2732 goto out; 2733 } 2734 2735 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 2736 match_criteria); 2737 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 2738 misc_parameters.source_port); 2739 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 2740 misc_parameters.source_eswitch_owner_vhca_id); 2741 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 2742 MLX5_MATCH_MISC_PARAMETERS); 2743 2744 MLX5_SET(create_flow_group_in, flow_group_in, 2745 source_eswitch_owner_vhca_id_valid, 1); 2746 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2747 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, count); 2748 2749 g = mlx5_create_flow_group(acl, flow_group_in); 2750 if (IS_ERR(g)) { 2751 err = PTR_ERR(g); 2752 goto err_group; 2753 } 2754 2755 vport->egress.acl = acl; 2756 vport->egress.offloads.bounce_grp = g; 2757 vport->egress.type = VPORT_EGRESS_ACL_TYPE_SHARED_FDB; 2758 xa_init_flags(&vport->egress.offloads.bounce_rules, XA_FLAGS_ALLOC); 2759 2760 kvfree(flow_group_in); 2761 2762 return 0; 2763 2764err_group: 2765 mlx5_destroy_flow_table(acl); 2766out: 2767 kvfree(flow_group_in); 2768 return err; 2769} 2770 2771static void esw_master_egress_destroy_resources(struct mlx5_vport *vport) 2772{ 2773 if (!xa_empty(&vport->egress.offloads.bounce_rules)) 2774 return; 2775 mlx5_destroy_flow_group(vport->egress.offloads.bounce_grp); 2776 vport->egress.offloads.bounce_grp = NULL; 2777 mlx5_destroy_flow_table(vport->egress.acl); 2778 vport->egress.acl = NULL; 2779} 2780 2781static int esw_set_master_egress_rule(struct mlx5_core_dev *master, 2782 struct mlx5_core_dev *slave, size_t count) 2783{ 2784 struct mlx5_eswitch *esw = master->priv.eswitch; 2785 u16 slave_index = MLX5_CAP_GEN(slave, vhca_id); 2786 struct mlx5_flow_namespace *egress_ns; 2787 struct mlx5_vport *vport; 2788 int err; 2789 2790 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport); 2791 if (IS_ERR(vport)) 2792 return PTR_ERR(vport); 2793 2794 egress_ns = mlx5_get_flow_vport_acl_namespace(master, 2795 MLX5_FLOW_NAMESPACE_ESW_EGRESS, 2796 vport->index); 2797 if (!egress_ns) 2798 return -EINVAL; 2799 2800 if (vport->egress.acl && vport->egress.type != VPORT_EGRESS_ACL_TYPE_SHARED_FDB) 2801 return 0; 2802 2803 err = esw_master_egress_create_resources(esw, egress_ns, vport, count); 2804 if (err) 2805 return err; 2806 2807 if (xa_load(&vport->egress.offloads.bounce_rules, slave_index)) 2808 return -EINVAL; 2809 2810 err = __esw_set_master_egress_rule(master, slave, vport, vport->egress.acl); 2811 if (err) 2812 goto err_rule; 2813 2814 return 0; 2815 2816err_rule: 2817 esw_master_egress_destroy_resources(vport); 2818 return err; 2819} 2820 2821static void esw_unset_master_egress_rule(struct mlx5_core_dev *dev, 2822 struct mlx5_core_dev *slave_dev) 2823{ 2824 struct mlx5_vport *vport; 2825 2826 vport = mlx5_eswitch_get_vport(dev->priv.eswitch, 2827 dev->priv.eswitch->manager_vport); 2828 2829 esw_acl_egress_ofld_bounce_rule_destroy(vport, MLX5_CAP_GEN(slave_dev, vhca_id)); 2830 2831 if (xa_empty(&vport->egress.offloads.bounce_rules)) { 2832 esw_acl_egress_ofld_cleanup(vport); 2833 xa_destroy(&vport->egress.offloads.bounce_rules); 2834 } 2835} 2836 2837int mlx5_eswitch_offloads_single_fdb_add_one(struct mlx5_eswitch *master_esw, 2838 struct mlx5_eswitch *slave_esw, int max_slaves) 2839{ 2840 int err; 2841 2842 err = esw_set_slave_root_fdb(master_esw->dev, 2843 slave_esw->dev); 2844 if (err) 2845 return err; 2846 2847 err = esw_set_master_egress_rule(master_esw->dev, 2848 slave_esw->dev, max_slaves); 2849 if (err) 2850 goto err_acl; 2851 2852 return err; 2853 2854err_acl: 2855 esw_set_slave_root_fdb(NULL, slave_esw->dev); 2856 return err; 2857} 2858 2859void mlx5_eswitch_offloads_single_fdb_del_one(struct mlx5_eswitch *master_esw, 2860 struct mlx5_eswitch *slave_esw) 2861{ 2862 esw_set_slave_root_fdb(NULL, slave_esw->dev); 2863 esw_unset_master_egress_rule(master_esw->dev, slave_esw->dev); 2864} 2865 2866#define ESW_OFFLOADS_DEVCOM_PAIR (0) 2867#define ESW_OFFLOADS_DEVCOM_UNPAIR (1) 2868 2869static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch *esw, 2870 struct mlx5_eswitch *peer_esw) 2871{ 2872 const struct mlx5_eswitch_rep_ops *ops; 2873 struct mlx5_eswitch_rep *rep; 2874 unsigned long i; 2875 u8 rep_type; 2876 2877 mlx5_esw_for_each_rep(esw, i, rep) { 2878 rep_type = NUM_REP_TYPES; 2879 while (rep_type--) { 2880 ops = esw->offloads.rep_ops[rep_type]; 2881 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 2882 ops->event) 2883 ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_UNPAIR, peer_esw); 2884 } 2885 } 2886} 2887 2888static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw, 2889 struct mlx5_eswitch *peer_esw) 2890{ 2891#if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 2892 mlx5e_tc_clean_fdb_peer_flows(esw); 2893#endif 2894 mlx5_esw_offloads_rep_event_unpair(esw, peer_esw); 2895 esw_del_fdb_peer_miss_rules(esw, peer_esw->dev); 2896} 2897 2898static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw, 2899 struct mlx5_eswitch *peer_esw) 2900{ 2901 const struct mlx5_eswitch_rep_ops *ops; 2902 struct mlx5_eswitch_rep *rep; 2903 unsigned long i; 2904 u8 rep_type; 2905 int err; 2906 2907 err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev); 2908 if (err) 2909 return err; 2910 2911 mlx5_esw_for_each_rep(esw, i, rep) { 2912 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) { 2913 ops = esw->offloads.rep_ops[rep_type]; 2914 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 2915 ops->event) { 2916 err = ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_PAIR, peer_esw); 2917 if (err) 2918 goto err_out; 2919 } 2920 } 2921 } 2922 2923 return 0; 2924 2925err_out: 2926 mlx5_esw_offloads_unpair(esw, peer_esw); 2927 return err; 2928} 2929 2930static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw, 2931 struct mlx5_eswitch *peer_esw, 2932 bool pair) 2933{ 2934 u16 peer_vhca_id = MLX5_CAP_GEN(peer_esw->dev, vhca_id); 2935 u16 vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id); 2936 struct mlx5_flow_root_namespace *peer_ns; 2937 struct mlx5_flow_root_namespace *ns; 2938 int err; 2939 2940 peer_ns = peer_esw->dev->priv.steering->fdb_root_ns; 2941 ns = esw->dev->priv.steering->fdb_root_ns; 2942 2943 if (pair) { 2944 err = mlx5_flow_namespace_set_peer(ns, peer_ns, peer_vhca_id); 2945 if (err) 2946 return err; 2947 2948 err = mlx5_flow_namespace_set_peer(peer_ns, ns, vhca_id); 2949 if (err) { 2950 mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id); 2951 return err; 2952 } 2953 } else { 2954 mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id); 2955 mlx5_flow_namespace_set_peer(peer_ns, NULL, vhca_id); 2956 } 2957 2958 return 0; 2959} 2960 2961static int mlx5_esw_offloads_devcom_event(int event, 2962 void *my_data, 2963 void *event_data) 2964{ 2965 struct mlx5_eswitch *esw = my_data; 2966 struct mlx5_eswitch *peer_esw = event_data; 2967 u16 esw_i, peer_esw_i; 2968 bool esw_paired; 2969 int err; 2970 2971 peer_esw_i = MLX5_CAP_GEN(peer_esw->dev, vhca_id); 2972 esw_i = MLX5_CAP_GEN(esw->dev, vhca_id); 2973 esw_paired = !!xa_load(&esw->paired, peer_esw_i); 2974 2975 switch (event) { 2976 case ESW_OFFLOADS_DEVCOM_PAIR: 2977 if (mlx5_eswitch_vport_match_metadata_enabled(esw) != 2978 mlx5_eswitch_vport_match_metadata_enabled(peer_esw)) 2979 break; 2980 2981 if (esw_paired) 2982 break; 2983 2984 err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true); 2985 if (err) 2986 goto err_out; 2987 2988 err = mlx5_esw_offloads_pair(esw, peer_esw); 2989 if (err) 2990 goto err_peer; 2991 2992 err = mlx5_esw_offloads_pair(peer_esw, esw); 2993 if (err) 2994 goto err_pair; 2995 2996 err = xa_insert(&esw->paired, peer_esw_i, peer_esw, GFP_KERNEL); 2997 if (err) 2998 goto err_xa; 2999 3000 err = xa_insert(&peer_esw->paired, esw_i, esw, GFP_KERNEL); 3001 if (err) 3002 goto err_peer_xa; 3003 3004 esw->num_peers++; 3005 peer_esw->num_peers++; 3006 mlx5_devcom_comp_set_ready(esw->devcom, true); 3007 break; 3008 3009 case ESW_OFFLOADS_DEVCOM_UNPAIR: 3010 if (!esw_paired) 3011 break; 3012 3013 peer_esw->num_peers--; 3014 esw->num_peers--; 3015 if (!esw->num_peers && !peer_esw->num_peers) 3016 mlx5_devcom_comp_set_ready(esw->devcom, false); 3017 xa_erase(&peer_esw->paired, esw_i); 3018 xa_erase(&esw->paired, peer_esw_i); 3019 mlx5_esw_offloads_unpair(peer_esw, esw); 3020 mlx5_esw_offloads_unpair(esw, peer_esw); 3021 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 3022 break; 3023 } 3024 3025 return 0; 3026 3027err_peer_xa: 3028 xa_erase(&esw->paired, peer_esw_i); 3029err_xa: 3030 mlx5_esw_offloads_unpair(peer_esw, esw); 3031err_pair: 3032 mlx5_esw_offloads_unpair(esw, peer_esw); 3033err_peer: 3034 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 3035err_out: 3036 mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d", 3037 event, err); 3038 return err; 3039} 3040 3041void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, u64 key) 3042{ 3043 int i; 3044 3045 for (i = 0; i < MLX5_MAX_PORTS; i++) 3046 INIT_LIST_HEAD(&esw->offloads.peer_flows[i]); 3047 mutex_init(&esw->offloads.peer_mutex); 3048 3049 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 3050 return; 3051 3052 if ((MLX5_VPORT_MANAGER(esw->dev) || mlx5_core_is_ecpf_esw_manager(esw->dev)) && 3053 !mlx5_lag_is_supported(esw->dev)) 3054 return; 3055 3056 xa_init(&esw->paired); 3057 esw->num_peers = 0; 3058 esw->devcom = mlx5_devcom_register_component(esw->dev->priv.devc, 3059 MLX5_DEVCOM_ESW_OFFLOADS, 3060 key, 3061 mlx5_esw_offloads_devcom_event, 3062 esw); 3063 if (IS_ERR(esw->devcom)) 3064 return; 3065 3066 mlx5_devcom_send_event(esw->devcom, 3067 ESW_OFFLOADS_DEVCOM_PAIR, 3068 ESW_OFFLOADS_DEVCOM_UNPAIR, 3069 esw); 3070} 3071 3072void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw) 3073{ 3074 if (IS_ERR_OR_NULL(esw->devcom)) 3075 return; 3076 3077 mlx5_devcom_send_event(esw->devcom, 3078 ESW_OFFLOADS_DEVCOM_UNPAIR, 3079 ESW_OFFLOADS_DEVCOM_UNPAIR, 3080 esw); 3081 3082 mlx5_devcom_unregister_component(esw->devcom); 3083 xa_destroy(&esw->paired); 3084 esw->devcom = NULL; 3085} 3086 3087bool mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch *esw) 3088{ 3089 return mlx5_devcom_comp_is_ready(esw->devcom); 3090} 3091 3092bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw) 3093{ 3094 if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl)) 3095 return false; 3096 3097 if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 3098 MLX5_FDB_TO_VPORT_REG_C_0)) 3099 return false; 3100 3101 return true; 3102} 3103 3104#define MLX5_ESW_METADATA_RSVD_UPLINK 1 3105 3106/* Share the same metadata for uplink's. This is fine because: 3107 * (a) In shared FDB mode (LAG) both uplink's are treated the 3108 * same and tagged with the same metadata. 3109 * (b) In non shared FDB mode, packets from physical port0 3110 * cannot hit eswitch of PF1 and vice versa. 3111 */ 3112static u32 mlx5_esw_match_metadata_reserved(struct mlx5_eswitch *esw) 3113{ 3114 return MLX5_ESW_METADATA_RSVD_UPLINK; 3115} 3116 3117u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw) 3118{ 3119 u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1; 3120 /* Reserve 0xf for internal port offload */ 3121 u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 2; 3122 u32 pf_num; 3123 int id; 3124 3125 /* Only 4 bits of pf_num */ 3126 pf_num = mlx5_get_dev_index(esw->dev); 3127 if (pf_num > max_pf_num) 3128 return 0; 3129 3130 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */ 3131 /* Use only non-zero vport_id (2-4095) for all PF's */ 3132 id = ida_alloc_range(&esw->offloads.vport_metadata_ida, 3133 MLX5_ESW_METADATA_RSVD_UPLINK + 1, 3134 vport_end_ida, GFP_KERNEL); 3135 if (id < 0) 3136 return 0; 3137 id = (pf_num << ESW_VPORT_BITS) | id; 3138 return id; 3139} 3140 3141void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata) 3142{ 3143 u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1; 3144 3145 /* Metadata contains only 12 bits of actual ida id */ 3146 ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask); 3147} 3148 3149static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw, 3150 struct mlx5_vport *vport) 3151{ 3152 if (vport->vport == MLX5_VPORT_UPLINK) 3153 vport->default_metadata = mlx5_esw_match_metadata_reserved(esw); 3154 else 3155 vport->default_metadata = mlx5_esw_match_metadata_alloc(esw); 3156 3157 vport->metadata = vport->default_metadata; 3158 return vport->metadata ? 0 : -ENOSPC; 3159} 3160 3161static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw, 3162 struct mlx5_vport *vport) 3163{ 3164 if (!vport->default_metadata) 3165 return; 3166 3167 if (vport->vport == MLX5_VPORT_UPLINK) 3168 return; 3169 3170 WARN_ON(vport->metadata != vport->default_metadata); 3171 mlx5_esw_match_metadata_free(esw, vport->default_metadata); 3172} 3173 3174static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw) 3175{ 3176 struct mlx5_vport *vport; 3177 unsigned long i; 3178 3179 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 3180 return; 3181 3182 mlx5_esw_for_each_vport(esw, i, vport) 3183 esw_offloads_vport_metadata_cleanup(esw, vport); 3184} 3185 3186static int esw_offloads_metadata_init(struct mlx5_eswitch *esw) 3187{ 3188 struct mlx5_vport *vport; 3189 unsigned long i; 3190 int err; 3191 3192 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 3193 return 0; 3194 3195 mlx5_esw_for_each_vport(esw, i, vport) { 3196 err = esw_offloads_vport_metadata_setup(esw, vport); 3197 if (err) 3198 goto metadata_err; 3199 } 3200 3201 return 0; 3202 3203metadata_err: 3204 esw_offloads_metadata_uninit(esw); 3205 return err; 3206} 3207 3208int 3209esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw, 3210 struct mlx5_vport *vport) 3211{ 3212 int err; 3213 3214 err = esw_acl_ingress_ofld_setup(esw, vport); 3215 if (err) 3216 return err; 3217 3218 err = esw_acl_egress_ofld_setup(esw, vport); 3219 if (err) 3220 goto egress_err; 3221 3222 return 0; 3223 3224egress_err: 3225 esw_acl_ingress_ofld_cleanup(esw, vport); 3226 return err; 3227} 3228 3229void 3230esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw, 3231 struct mlx5_vport *vport) 3232{ 3233 esw_acl_egress_ofld_cleanup(vport); 3234 esw_acl_ingress_ofld_cleanup(esw, vport); 3235} 3236 3237static int esw_create_offloads_acl_tables(struct mlx5_eswitch *esw) 3238{ 3239 struct mlx5_vport *uplink, *manager; 3240 int ret; 3241 3242 uplink = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 3243 if (IS_ERR(uplink)) 3244 return PTR_ERR(uplink); 3245 3246 ret = esw_vport_create_offloads_acl_tables(esw, uplink); 3247 if (ret) 3248 return ret; 3249 3250 manager = mlx5_eswitch_get_vport(esw, esw->manager_vport); 3251 if (IS_ERR(manager)) { 3252 ret = PTR_ERR(manager); 3253 goto err_manager; 3254 } 3255 3256 ret = esw_vport_create_offloads_acl_tables(esw, manager); 3257 if (ret) 3258 goto err_manager; 3259 3260 return 0; 3261 3262err_manager: 3263 esw_vport_destroy_offloads_acl_tables(esw, uplink); 3264 return ret; 3265} 3266 3267static void esw_destroy_offloads_acl_tables(struct mlx5_eswitch *esw) 3268{ 3269 struct mlx5_vport *vport; 3270 3271 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport); 3272 if (!IS_ERR(vport)) 3273 esw_vport_destroy_offloads_acl_tables(esw, vport); 3274 3275 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 3276 if (!IS_ERR(vport)) 3277 esw_vport_destroy_offloads_acl_tables(esw, vport); 3278} 3279 3280int mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw) 3281{ 3282 struct mlx5_eswitch_rep *rep; 3283 unsigned long i; 3284 int ret; 3285 3286 if (!esw || esw->mode != MLX5_ESWITCH_OFFLOADS) 3287 return 0; 3288 3289 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 3290 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED) 3291 return 0; 3292 3293 ret = mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK); 3294 if (ret) 3295 return ret; 3296 3297 mlx5_esw_for_each_rep(esw, i, rep) { 3298 if (atomic_read(&rep->rep_data[REP_ETH].state) == REP_LOADED) 3299 mlx5_esw_offloads_rep_load(esw, rep->vport); 3300 } 3301 3302 return 0; 3303} 3304 3305static int esw_offloads_steering_init(struct mlx5_eswitch *esw) 3306{ 3307 struct mlx5_esw_indir_table *indir; 3308 int err; 3309 3310 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb)); 3311 mutex_init(&esw->fdb_table.offloads.vports.lock); 3312 hash_init(esw->fdb_table.offloads.vports.table); 3313 atomic64_set(&esw->user_count, 0); 3314 3315 indir = mlx5_esw_indir_table_init(); 3316 if (IS_ERR(indir)) { 3317 err = PTR_ERR(indir); 3318 goto create_indir_err; 3319 } 3320 esw->fdb_table.offloads.indir = indir; 3321 3322 err = esw_create_offloads_acl_tables(esw); 3323 if (err) 3324 goto create_acl_err; 3325 3326 err = esw_create_offloads_table(esw); 3327 if (err) 3328 goto create_offloads_err; 3329 3330 err = esw_create_restore_table(esw); 3331 if (err) 3332 goto create_restore_err; 3333 3334 err = esw_create_offloads_fdb_tables(esw); 3335 if (err) 3336 goto create_fdb_err; 3337 3338 err = esw_create_vport_rx_group(esw); 3339 if (err) 3340 goto create_fg_err; 3341 3342 err = esw_create_vport_rx_drop_group(esw); 3343 if (err) 3344 goto create_rx_drop_fg_err; 3345 3346 err = esw_create_vport_rx_drop_rule(esw); 3347 if (err) 3348 goto create_rx_drop_rule_err; 3349 3350 return 0; 3351 3352create_rx_drop_rule_err: 3353 esw_destroy_vport_rx_drop_group(esw); 3354create_rx_drop_fg_err: 3355 esw_destroy_vport_rx_group(esw); 3356create_fg_err: 3357 esw_destroy_offloads_fdb_tables(esw); 3358create_fdb_err: 3359 esw_destroy_restore_table(esw); 3360create_restore_err: 3361 esw_destroy_offloads_table(esw); 3362create_offloads_err: 3363 esw_destroy_offloads_acl_tables(esw); 3364create_acl_err: 3365 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir); 3366create_indir_err: 3367 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 3368 return err; 3369} 3370 3371static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw) 3372{ 3373 esw_destroy_vport_rx_drop_rule(esw); 3374 esw_destroy_vport_rx_drop_group(esw); 3375 esw_destroy_vport_rx_group(esw); 3376 esw_destroy_offloads_fdb_tables(esw); 3377 esw_destroy_restore_table(esw); 3378 esw_destroy_offloads_table(esw); 3379 esw_destroy_offloads_acl_tables(esw); 3380 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir); 3381 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 3382} 3383 3384static void 3385esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) 3386{ 3387 struct devlink *devlink; 3388 bool host_pf_disabled; 3389 u16 new_num_vfs; 3390 3391 new_num_vfs = MLX5_GET(query_esw_functions_out, out, 3392 host_params_context.host_num_of_vfs); 3393 host_pf_disabled = MLX5_GET(query_esw_functions_out, out, 3394 host_params_context.host_pf_disabled); 3395 3396 if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled) 3397 return; 3398 3399 devlink = priv_to_devlink(esw->dev); 3400 devl_lock(devlink); 3401 /* Number of VFs can only change from "0 to x" or "x to 0". */ 3402 if (esw->esw_funcs.num_vfs > 0) { 3403 mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); 3404 } else { 3405 int err; 3406 3407 err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs, 3408 MLX5_VPORT_UC_ADDR_CHANGE); 3409 if (err) { 3410 devl_unlock(devlink); 3411 return; 3412 } 3413 } 3414 esw->esw_funcs.num_vfs = new_num_vfs; 3415 devl_unlock(devlink); 3416} 3417 3418static void esw_functions_changed_event_handler(struct work_struct *work) 3419{ 3420 struct mlx5_host_work *host_work; 3421 struct mlx5_eswitch *esw; 3422 const u32 *out; 3423 3424 host_work = container_of(work, struct mlx5_host_work, work); 3425 esw = host_work->esw; 3426 3427 out = mlx5_esw_query_functions(esw->dev); 3428 if (IS_ERR(out)) 3429 goto out; 3430 3431 esw_vfs_changed_event_handler(esw, out); 3432 kvfree(out); 3433out: 3434 kfree(host_work); 3435} 3436 3437int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data) 3438{ 3439 struct mlx5_esw_functions *esw_funcs; 3440 struct mlx5_host_work *host_work; 3441 struct mlx5_eswitch *esw; 3442 3443 host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC); 3444 if (!host_work) 3445 return NOTIFY_DONE; 3446 3447 esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb); 3448 esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs); 3449 3450 host_work->esw = esw; 3451 3452 INIT_WORK(&host_work->work, esw_functions_changed_event_handler); 3453 queue_work(esw->work_queue, &host_work->work); 3454 3455 return NOTIFY_OK; 3456} 3457 3458static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw) 3459{ 3460 const u32 *query_host_out; 3461 3462 if (!mlx5_core_is_ecpf_esw_manager(esw->dev)) 3463 return 0; 3464 3465 query_host_out = mlx5_esw_query_functions(esw->dev); 3466 if (IS_ERR(query_host_out)) 3467 return PTR_ERR(query_host_out); 3468 3469 /* Mark non local controller with non zero controller number. */ 3470 esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out, 3471 host_params_context.host_number); 3472 kvfree(query_host_out); 3473 return 0; 3474} 3475 3476bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch *esw, u32 controller) 3477{ 3478 /* Local controller is always valid */ 3479 if (controller == 0) 3480 return true; 3481 3482 if (!mlx5_core_is_ecpf_esw_manager(esw->dev)) 3483 return false; 3484 3485 /* External host number starts with zero in device */ 3486 return (controller == esw->offloads.host_number + 1); 3487} 3488 3489int esw_offloads_enable(struct mlx5_eswitch *esw) 3490{ 3491 struct mapping_ctx *reg_c0_obj_pool; 3492 struct mlx5_vport *vport; 3493 unsigned long i; 3494 u64 mapping_id; 3495 int err; 3496 3497 mutex_init(&esw->offloads.termtbl_mutex); 3498 mlx5_rdma_enable_roce(esw->dev); 3499 3500 err = mlx5_esw_host_number_init(esw); 3501 if (err) 3502 goto err_metadata; 3503 3504 err = esw_offloads_metadata_init(esw); 3505 if (err) 3506 goto err_metadata; 3507 3508 err = esw_set_passing_vport_metadata(esw, true); 3509 if (err) 3510 goto err_vport_metadata; 3511 3512 mapping_id = mlx5_query_nic_system_image_guid(esw->dev); 3513 3514 reg_c0_obj_pool = mapping_create_for_id(mapping_id, MAPPING_TYPE_CHAIN, 3515 sizeof(struct mlx5_mapped_obj), 3516 ESW_REG_C0_USER_DATA_METADATA_MASK, 3517 true); 3518 3519 if (IS_ERR(reg_c0_obj_pool)) { 3520 err = PTR_ERR(reg_c0_obj_pool); 3521 goto err_pool; 3522 } 3523 esw->offloads.reg_c0_obj_pool = reg_c0_obj_pool; 3524 3525 err = esw_offloads_steering_init(esw); 3526 if (err) 3527 goto err_steering_init; 3528 3529 /* Representor will control the vport link state */ 3530 mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs) 3531 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN; 3532 if (mlx5_core_ec_sriov_enabled(esw->dev)) 3533 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) 3534 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN; 3535 3536 /* Uplink vport rep must load first. */ 3537 err = mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK); 3538 if (err) 3539 goto err_uplink; 3540 3541 err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE); 3542 if (err) 3543 goto err_vports; 3544 3545 return 0; 3546 3547err_vports: 3548 mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK); 3549err_uplink: 3550 esw_offloads_steering_cleanup(esw); 3551err_steering_init: 3552 mapping_destroy(reg_c0_obj_pool); 3553err_pool: 3554 esw_set_passing_vport_metadata(esw, false); 3555err_vport_metadata: 3556 esw_offloads_metadata_uninit(esw); 3557err_metadata: 3558 mlx5_rdma_disable_roce(esw->dev); 3559 mutex_destroy(&esw->offloads.termtbl_mutex); 3560 return err; 3561} 3562 3563static int esw_offloads_stop(struct mlx5_eswitch *esw, 3564 struct netlink_ext_ack *extack) 3565{ 3566 int err; 3567 3568 esw->mode = MLX5_ESWITCH_LEGACY; 3569 3570 /* If changing from switchdev to legacy mode without sriov enabled, 3571 * no need to create legacy fdb. 3572 */ 3573 if (!mlx5_core_is_pf(esw->dev) || !mlx5_sriov_is_enabled(esw->dev)) 3574 return 0; 3575 3576 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS); 3577 if (err) 3578 NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy"); 3579 3580 return err; 3581} 3582 3583void esw_offloads_disable(struct mlx5_eswitch *esw) 3584{ 3585 mlx5_eswitch_disable_pf_vf_vports(esw); 3586 mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK); 3587 esw_set_passing_vport_metadata(esw, false); 3588 esw_offloads_steering_cleanup(esw); 3589 mapping_destroy(esw->offloads.reg_c0_obj_pool); 3590 esw_offloads_metadata_uninit(esw); 3591 mlx5_rdma_disable_roce(esw->dev); 3592 mutex_destroy(&esw->offloads.termtbl_mutex); 3593} 3594 3595static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode) 3596{ 3597 switch (mode) { 3598 case DEVLINK_ESWITCH_MODE_LEGACY: 3599 *mlx5_mode = MLX5_ESWITCH_LEGACY; 3600 break; 3601 case DEVLINK_ESWITCH_MODE_SWITCHDEV: 3602 *mlx5_mode = MLX5_ESWITCH_OFFLOADS; 3603 break; 3604 default: 3605 return -EINVAL; 3606 } 3607 3608 return 0; 3609} 3610 3611static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode) 3612{ 3613 switch (mlx5_mode) { 3614 case MLX5_ESWITCH_LEGACY: 3615 *mode = DEVLINK_ESWITCH_MODE_LEGACY; 3616 break; 3617 case MLX5_ESWITCH_OFFLOADS: 3618 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; 3619 break; 3620 default: 3621 return -EINVAL; 3622 } 3623 3624 return 0; 3625} 3626 3627static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode) 3628{ 3629 switch (mode) { 3630 case DEVLINK_ESWITCH_INLINE_MODE_NONE: 3631 *mlx5_mode = MLX5_INLINE_MODE_NONE; 3632 break; 3633 case DEVLINK_ESWITCH_INLINE_MODE_LINK: 3634 *mlx5_mode = MLX5_INLINE_MODE_L2; 3635 break; 3636 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK: 3637 *mlx5_mode = MLX5_INLINE_MODE_IP; 3638 break; 3639 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT: 3640 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP; 3641 break; 3642 default: 3643 return -EINVAL; 3644 } 3645 3646 return 0; 3647} 3648 3649static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode) 3650{ 3651 switch (mlx5_mode) { 3652 case MLX5_INLINE_MODE_NONE: 3653 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE; 3654 break; 3655 case MLX5_INLINE_MODE_L2: 3656 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK; 3657 break; 3658 case MLX5_INLINE_MODE_IP: 3659 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK; 3660 break; 3661 case MLX5_INLINE_MODE_TCP_UDP: 3662 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT; 3663 break; 3664 default: 3665 return -EINVAL; 3666 } 3667 3668 return 0; 3669} 3670 3671int mlx5_eswitch_block_mode(struct mlx5_core_dev *dev) 3672{ 3673 struct mlx5_eswitch *esw = dev->priv.eswitch; 3674 int err; 3675 3676 if (!mlx5_esw_allowed(esw)) 3677 return 0; 3678 3679 /* Take TC into account */ 3680 err = mlx5_esw_try_lock(esw); 3681 if (err < 0) 3682 return err; 3683 3684 esw->offloads.num_block_mode++; 3685 mlx5_esw_unlock(esw); 3686 return 0; 3687} 3688 3689void mlx5_eswitch_unblock_mode(struct mlx5_core_dev *dev) 3690{ 3691 struct mlx5_eswitch *esw = dev->priv.eswitch; 3692 3693 if (!mlx5_esw_allowed(esw)) 3694 return; 3695 3696 down_write(&esw->mode_lock); 3697 esw->offloads.num_block_mode--; 3698 up_write(&esw->mode_lock); 3699} 3700 3701int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, 3702 struct netlink_ext_ack *extack) 3703{ 3704 u16 cur_mlx5_mode, mlx5_mode = 0; 3705 struct mlx5_eswitch *esw; 3706 int err = 0; 3707 3708 esw = mlx5_devlink_eswitch_get(devlink); 3709 if (IS_ERR(esw)) 3710 return PTR_ERR(esw); 3711 3712 if (esw_mode_from_devlink(mode, &mlx5_mode)) 3713 return -EINVAL; 3714 3715 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && mlx5_get_sd(esw->dev)) { 3716 NL_SET_ERR_MSG_MOD(extack, 3717 "Can't change E-Switch mode to switchdev when multi-PF netdev (Socket Direct) is configured."); 3718 return -EPERM; 3719 } 3720 3721 mlx5_lag_disable_change(esw->dev); 3722 err = mlx5_esw_try_lock(esw); 3723 if (err < 0) { 3724 NL_SET_ERR_MSG_MOD(extack, "Can't change mode, E-Switch is busy"); 3725 goto enable_lag; 3726 } 3727 cur_mlx5_mode = err; 3728 err = 0; 3729 3730 if (cur_mlx5_mode == mlx5_mode) 3731 goto unlock; 3732 3733 if (esw->offloads.num_block_mode) { 3734 NL_SET_ERR_MSG_MOD(extack, 3735 "Can't change eswitch mode when IPsec SA and/or policies are configured"); 3736 err = -EOPNOTSUPP; 3737 goto unlock; 3738 } 3739 3740 esw->eswitch_operation_in_progress = true; 3741 up_write(&esw->mode_lock); 3742 3743 mlx5_eswitch_disable_locked(esw); 3744 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) { 3745 if (mlx5_devlink_trap_get_num_active(esw->dev)) { 3746 NL_SET_ERR_MSG_MOD(extack, 3747 "Can't change mode while devlink traps are active"); 3748 err = -EOPNOTSUPP; 3749 goto skip; 3750 } 3751 err = esw_offloads_start(esw, extack); 3752 } else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) { 3753 err = esw_offloads_stop(esw, extack); 3754 mlx5_rescan_drivers(esw->dev); 3755 } else { 3756 err = -EINVAL; 3757 } 3758 3759skip: 3760 down_write(&esw->mode_lock); 3761 esw->eswitch_operation_in_progress = false; 3762unlock: 3763 mlx5_esw_unlock(esw); 3764enable_lag: 3765 mlx5_lag_enable_change(esw->dev); 3766 return err; 3767} 3768 3769int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) 3770{ 3771 struct mlx5_eswitch *esw; 3772 3773 esw = mlx5_devlink_eswitch_get(devlink); 3774 if (IS_ERR(esw)) 3775 return PTR_ERR(esw); 3776 3777 return esw_mode_to_devlink(esw->mode, mode); 3778} 3779 3780static int mlx5_esw_vports_inline_set(struct mlx5_eswitch *esw, u8 mlx5_mode, 3781 struct netlink_ext_ack *extack) 3782{ 3783 struct mlx5_core_dev *dev = esw->dev; 3784 struct mlx5_vport *vport; 3785 u16 err_vport_num = 0; 3786 unsigned long i; 3787 int err = 0; 3788 3789 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 3790 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode); 3791 if (err) { 3792 err_vport_num = vport->vport; 3793 NL_SET_ERR_MSG_MOD(extack, 3794 "Failed to set min inline on vport"); 3795 goto revert_inline_mode; 3796 } 3797 } 3798 if (mlx5_core_ec_sriov_enabled(esw->dev)) { 3799 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) { 3800 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode); 3801 if (err) { 3802 err_vport_num = vport->vport; 3803 NL_SET_ERR_MSG_MOD(extack, 3804 "Failed to set min inline on vport"); 3805 goto revert_ec_vf_inline_mode; 3806 } 3807 } 3808 } 3809 return 0; 3810 3811revert_ec_vf_inline_mode: 3812 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) { 3813 if (vport->vport == err_vport_num) 3814 break; 3815 mlx5_modify_nic_vport_min_inline(dev, 3816 vport->vport, 3817 esw->offloads.inline_mode); 3818 } 3819revert_inline_mode: 3820 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 3821 if (vport->vport == err_vport_num) 3822 break; 3823 mlx5_modify_nic_vport_min_inline(dev, 3824 vport->vport, 3825 esw->offloads.inline_mode); 3826 } 3827 return err; 3828} 3829 3830int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode, 3831 struct netlink_ext_ack *extack) 3832{ 3833 struct mlx5_core_dev *dev = devlink_priv(devlink); 3834 struct mlx5_eswitch *esw; 3835 u8 mlx5_mode; 3836 int err; 3837 3838 esw = mlx5_devlink_eswitch_get(devlink); 3839 if (IS_ERR(esw)) 3840 return PTR_ERR(esw); 3841 3842 down_write(&esw->mode_lock); 3843 3844 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 3845 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 3846 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) { 3847 err = 0; 3848 goto out; 3849 } 3850 3851 fallthrough; 3852 case MLX5_CAP_INLINE_MODE_L2: 3853 NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set"); 3854 err = -EOPNOTSUPP; 3855 goto out; 3856 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 3857 break; 3858 } 3859 3860 if (atomic64_read(&esw->offloads.num_flows) > 0) { 3861 NL_SET_ERR_MSG_MOD(extack, 3862 "Can't set inline mode when flows are configured"); 3863 err = -EOPNOTSUPP; 3864 goto out; 3865 } 3866 3867 err = esw_inline_mode_from_devlink(mode, &mlx5_mode); 3868 if (err) 3869 goto out; 3870 3871 esw->eswitch_operation_in_progress = true; 3872 up_write(&esw->mode_lock); 3873 3874 err = mlx5_esw_vports_inline_set(esw, mlx5_mode, extack); 3875 if (!err) 3876 esw->offloads.inline_mode = mlx5_mode; 3877 3878 down_write(&esw->mode_lock); 3879 esw->eswitch_operation_in_progress = false; 3880 up_write(&esw->mode_lock); 3881 return 0; 3882 3883out: 3884 up_write(&esw->mode_lock); 3885 return err; 3886} 3887 3888int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode) 3889{ 3890 struct mlx5_eswitch *esw; 3891 3892 esw = mlx5_devlink_eswitch_get(devlink); 3893 if (IS_ERR(esw)) 3894 return PTR_ERR(esw); 3895 3896 return esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode); 3897} 3898 3899bool mlx5_eswitch_block_encap(struct mlx5_core_dev *dev) 3900{ 3901 struct mlx5_eswitch *esw = dev->priv.eswitch; 3902 3903 if (!mlx5_esw_allowed(esw)) 3904 return true; 3905 3906 down_write(&esw->mode_lock); 3907 if (esw->mode != MLX5_ESWITCH_LEGACY && 3908 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) { 3909 up_write(&esw->mode_lock); 3910 return false; 3911 } 3912 3913 esw->offloads.num_block_encap++; 3914 up_write(&esw->mode_lock); 3915 return true; 3916} 3917 3918void mlx5_eswitch_unblock_encap(struct mlx5_core_dev *dev) 3919{ 3920 struct mlx5_eswitch *esw = dev->priv.eswitch; 3921 3922 if (!mlx5_esw_allowed(esw)) 3923 return; 3924 3925 down_write(&esw->mode_lock); 3926 esw->offloads.num_block_encap--; 3927 up_write(&esw->mode_lock); 3928} 3929 3930int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, 3931 enum devlink_eswitch_encap_mode encap, 3932 struct netlink_ext_ack *extack) 3933{ 3934 struct mlx5_core_dev *dev = devlink_priv(devlink); 3935 struct mlx5_eswitch *esw; 3936 int err = 0; 3937 3938 esw = mlx5_devlink_eswitch_get(devlink); 3939 if (IS_ERR(esw)) 3940 return PTR_ERR(esw); 3941 3942 down_write(&esw->mode_lock); 3943 3944 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE && 3945 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) || 3946 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) { 3947 err = -EOPNOTSUPP; 3948 goto unlock; 3949 } 3950 3951 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) { 3952 err = -EOPNOTSUPP; 3953 goto unlock; 3954 } 3955 3956 if (esw->mode == MLX5_ESWITCH_LEGACY) { 3957 esw->offloads.encap = encap; 3958 goto unlock; 3959 } 3960 3961 if (esw->offloads.encap == encap) 3962 goto unlock; 3963 3964 if (atomic64_read(&esw->offloads.num_flows) > 0) { 3965 NL_SET_ERR_MSG_MOD(extack, 3966 "Can't set encapsulation when flows are configured"); 3967 err = -EOPNOTSUPP; 3968 goto unlock; 3969 } 3970 3971 if (esw->offloads.num_block_encap) { 3972 NL_SET_ERR_MSG_MOD(extack, 3973 "Can't set encapsulation when IPsec SA and/or policies are configured"); 3974 err = -EOPNOTSUPP; 3975 goto unlock; 3976 } 3977 3978 esw->eswitch_operation_in_progress = true; 3979 up_write(&esw->mode_lock); 3980 3981 esw_destroy_offloads_fdb_tables(esw); 3982 3983 esw->offloads.encap = encap; 3984 3985 err = esw_create_offloads_fdb_tables(esw); 3986 3987 if (err) { 3988 NL_SET_ERR_MSG_MOD(extack, 3989 "Failed re-creating fast FDB table"); 3990 esw->offloads.encap = !encap; 3991 (void)esw_create_offloads_fdb_tables(esw); 3992 } 3993 3994 down_write(&esw->mode_lock); 3995 esw->eswitch_operation_in_progress = false; 3996 3997unlock: 3998 up_write(&esw->mode_lock); 3999 return err; 4000} 4001 4002int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, 4003 enum devlink_eswitch_encap_mode *encap) 4004{ 4005 struct mlx5_eswitch *esw; 4006 4007 esw = mlx5_devlink_eswitch_get(devlink); 4008 if (IS_ERR(esw)) 4009 return PTR_ERR(esw); 4010 4011 *encap = esw->offloads.encap; 4012 return 0; 4013} 4014 4015static bool 4016mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num) 4017{ 4018 /* Currently, only ECPF based device has representor for host PF. */ 4019 if (vport_num == MLX5_VPORT_PF && 4020 !mlx5_core_is_ecpf_esw_manager(esw->dev)) 4021 return false; 4022 4023 if (vport_num == MLX5_VPORT_ECPF && 4024 !mlx5_ecpf_vport_exists(esw->dev)) 4025 return false; 4026 4027 return true; 4028} 4029 4030void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw, 4031 const struct mlx5_eswitch_rep_ops *ops, 4032 u8 rep_type) 4033{ 4034 struct mlx5_eswitch_rep_data *rep_data; 4035 struct mlx5_eswitch_rep *rep; 4036 unsigned long i; 4037 4038 esw->offloads.rep_ops[rep_type] = ops; 4039 mlx5_esw_for_each_rep(esw, i, rep) { 4040 if (likely(mlx5_eswitch_vport_has_rep(esw, rep->vport))) { 4041 rep->esw = esw; 4042 rep_data = &rep->rep_data[rep_type]; 4043 atomic_set(&rep_data->state, REP_REGISTERED); 4044 } 4045 } 4046} 4047EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps); 4048 4049void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type) 4050{ 4051 struct mlx5_eswitch_rep *rep; 4052 unsigned long i; 4053 4054 if (esw->mode == MLX5_ESWITCH_OFFLOADS) 4055 __unload_reps_all_vport(esw, rep_type); 4056 4057 mlx5_esw_for_each_rep(esw, i, rep) 4058 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED); 4059} 4060EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps); 4061 4062void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type) 4063{ 4064 struct mlx5_eswitch_rep *rep; 4065 4066 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 4067 return rep->rep_data[rep_type].priv; 4068} 4069 4070void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw, 4071 u16 vport, 4072 u8 rep_type) 4073{ 4074 struct mlx5_eswitch_rep *rep; 4075 4076 rep = mlx5_eswitch_get_rep(esw, vport); 4077 4078 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 4079 esw->offloads.rep_ops[rep_type]->get_proto_dev) 4080 return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep); 4081 return NULL; 4082} 4083EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev); 4084 4085void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type) 4086{ 4087 return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type); 4088} 4089EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev); 4090 4091struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw, 4092 u16 vport) 4093{ 4094 return mlx5_eswitch_get_rep(esw, vport); 4095} 4096EXPORT_SYMBOL(mlx5_eswitch_vport_rep); 4097 4098bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw) 4099{ 4100 return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED); 4101} 4102EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled); 4103 4104bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw) 4105{ 4106 return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA); 4107} 4108EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled); 4109 4110u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw, 4111 u16 vport_num) 4112{ 4113 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 4114 4115 if (WARN_ON_ONCE(IS_ERR(vport))) 4116 return 0; 4117 4118 return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS); 4119} 4120EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match); 4121 4122static int mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch *esw, u16 vport_num, u16 *vhca_id) 4123{ 4124 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 4125 void *query_ctx; 4126 void *hca_caps; 4127 int err; 4128 4129 *vhca_id = 0; 4130 4131 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 4132 if (!query_ctx) 4133 return -ENOMEM; 4134 4135 err = mlx5_vport_get_other_func_general_cap(esw->dev, vport_num, query_ctx); 4136 if (err) 4137 goto out_free; 4138 4139 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 4140 *vhca_id = MLX5_GET(cmd_hca_cap, hca_caps, vhca_id); 4141 4142out_free: 4143 kfree(query_ctx); 4144 return err; 4145} 4146 4147int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch *esw, u16 vport_num) 4148{ 4149 u16 *old_entry, *vhca_map_entry, vhca_id; 4150 int err; 4151 4152 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id); 4153 if (err) { 4154 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%u,err=%d)\n", 4155 vport_num, err); 4156 return err; 4157 } 4158 4159 vhca_map_entry = kmalloc(sizeof(*vhca_map_entry), GFP_KERNEL); 4160 if (!vhca_map_entry) 4161 return -ENOMEM; 4162 4163 *vhca_map_entry = vport_num; 4164 old_entry = xa_store(&esw->offloads.vhca_map, vhca_id, vhca_map_entry, GFP_KERNEL); 4165 if (xa_is_err(old_entry)) { 4166 kfree(vhca_map_entry); 4167 return xa_err(old_entry); 4168 } 4169 kfree(old_entry); 4170 return 0; 4171} 4172 4173void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch *esw, u16 vport_num) 4174{ 4175 u16 *vhca_map_entry, vhca_id; 4176 int err; 4177 4178 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id); 4179 if (err) 4180 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%hu,err=%d)\n", 4181 vport_num, err); 4182 4183 vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vhca_id); 4184 kfree(vhca_map_entry); 4185} 4186 4187int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num) 4188{ 4189 u16 *res = xa_load(&esw->offloads.vhca_map, vhca_id); 4190 4191 if (!res) 4192 return -ENOENT; 4193 4194 *vport_num = *res; 4195 return 0; 4196} 4197 4198u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw, 4199 u16 vport_num) 4200{ 4201 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 4202 4203 if (WARN_ON_ONCE(IS_ERR(vport))) 4204 return 0; 4205 4206 return vport->metadata; 4207} 4208EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set); 4209 4210int mlx5_devlink_port_fn_hw_addr_get(struct devlink_port *port, 4211 u8 *hw_addr, int *hw_addr_len, 4212 struct netlink_ext_ack *extack) 4213{ 4214 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4215 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4216 4217 mutex_lock(&esw->state_lock); 4218 ether_addr_copy(hw_addr, vport->info.mac); 4219 *hw_addr_len = ETH_ALEN; 4220 mutex_unlock(&esw->state_lock); 4221 return 0; 4222} 4223 4224int mlx5_devlink_port_fn_hw_addr_set(struct devlink_port *port, 4225 const u8 *hw_addr, int hw_addr_len, 4226 struct netlink_ext_ack *extack) 4227{ 4228 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4229 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4230 4231 return mlx5_eswitch_set_vport_mac(esw, vport->vport, hw_addr); 4232} 4233 4234int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enabled, 4235 struct netlink_ext_ack *extack) 4236{ 4237 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4238 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4239 4240 if (!MLX5_CAP_GEN(esw->dev, migration)) { 4241 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration"); 4242 return -EOPNOTSUPP; 4243 } 4244 4245 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 4246 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); 4247 return -EOPNOTSUPP; 4248 } 4249 4250 mutex_lock(&esw->state_lock); 4251 *is_enabled = vport->info.mig_enabled; 4252 mutex_unlock(&esw->state_lock); 4253 return 0; 4254} 4255 4256int mlx5_devlink_port_fn_migratable_set(struct devlink_port *port, bool enable, 4257 struct netlink_ext_ack *extack) 4258{ 4259 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4260 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4261 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 4262 void *query_ctx; 4263 void *hca_caps; 4264 int err; 4265 4266 if (!MLX5_CAP_GEN(esw->dev, migration)) { 4267 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration"); 4268 return -EOPNOTSUPP; 4269 } 4270 4271 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 4272 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); 4273 return -EOPNOTSUPP; 4274 } 4275 4276 mutex_lock(&esw->state_lock); 4277 4278 if (vport->info.mig_enabled == enable) { 4279 err = 0; 4280 goto out; 4281 } 4282 4283 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 4284 if (!query_ctx) { 4285 err = -ENOMEM; 4286 goto out; 4287 } 4288 4289 err = mlx5_vport_get_other_func_cap(esw->dev, vport->vport, query_ctx, 4290 MLX5_CAP_GENERAL_2); 4291 if (err) { 4292 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 4293 goto out_free; 4294 } 4295 4296 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 4297 MLX5_SET(cmd_hca_cap_2, hca_caps, migratable, enable); 4298 4299 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport->vport, 4300 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2); 4301 if (err) { 4302 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA migratable cap"); 4303 goto out_free; 4304 } 4305 4306 vport->info.mig_enabled = enable; 4307 4308out_free: 4309 kfree(query_ctx); 4310out: 4311 mutex_unlock(&esw->state_lock); 4312 return err; 4313} 4314 4315int mlx5_devlink_port_fn_roce_get(struct devlink_port *port, bool *is_enabled, 4316 struct netlink_ext_ack *extack) 4317{ 4318 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4319 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4320 4321 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 4322 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); 4323 return -EOPNOTSUPP; 4324 } 4325 4326 mutex_lock(&esw->state_lock); 4327 *is_enabled = vport->info.roce_enabled; 4328 mutex_unlock(&esw->state_lock); 4329 return 0; 4330} 4331 4332int mlx5_devlink_port_fn_roce_set(struct devlink_port *port, bool enable, 4333 struct netlink_ext_ack *extack) 4334{ 4335 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4336 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4337 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 4338 u16 vport_num = vport->vport; 4339 void *query_ctx; 4340 void *hca_caps; 4341 int err; 4342 4343 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 4344 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); 4345 return -EOPNOTSUPP; 4346 } 4347 4348 mutex_lock(&esw->state_lock); 4349 4350 if (vport->info.roce_enabled == enable) { 4351 err = 0; 4352 goto out; 4353 } 4354 4355 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 4356 if (!query_ctx) { 4357 err = -ENOMEM; 4358 goto out; 4359 } 4360 4361 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx, 4362 MLX5_CAP_GENERAL); 4363 if (err) { 4364 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 4365 goto out_free; 4366 } 4367 4368 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 4369 MLX5_SET(cmd_hca_cap, hca_caps, roce, enable); 4370 4371 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport_num, 4372 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); 4373 if (err) { 4374 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA roce cap"); 4375 goto out_free; 4376 } 4377 4378 vport->info.roce_enabled = enable; 4379 4380out_free: 4381 kfree(query_ctx); 4382out: 4383 mutex_unlock(&esw->state_lock); 4384 return err; 4385} 4386 4387int 4388mlx5_eswitch_restore_ipsec_rule(struct mlx5_eswitch *esw, struct mlx5_flow_handle *rule, 4389 struct mlx5_esw_flow_attr *esw_attr, int attr_idx) 4390{ 4391 struct mlx5_flow_destination new_dest = {}; 4392 struct mlx5_flow_destination old_dest = {}; 4393 4394 if (!esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx)) 4395 return 0; 4396 4397 esw_setup_dest_fwd_ipsec(&old_dest, NULL, esw, esw_attr, attr_idx, 0, false); 4398 esw_setup_dest_fwd_vport(&new_dest, NULL, esw, esw_attr, attr_idx, 0, false); 4399 4400 return mlx5_modify_rule_destination(rule, &new_dest, &old_dest); 4401} 4402 4403#ifdef CONFIG_XFRM_OFFLOAD 4404int mlx5_devlink_port_fn_ipsec_crypto_get(struct devlink_port *port, bool *is_enabled, 4405 struct netlink_ext_ack *extack) 4406{ 4407 struct mlx5_eswitch *esw; 4408 struct mlx5_vport *vport; 4409 int err = 0; 4410 4411 esw = mlx5_devlink_eswitch_get(port->devlink); 4412 if (IS_ERR(esw)) 4413 return PTR_ERR(esw); 4414 4415 if (!mlx5_esw_ipsec_vf_offload_supported(esw->dev)) { 4416 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPSec crypto"); 4417 return -EOPNOTSUPP; 4418 } 4419 4420 vport = mlx5_devlink_port_vport_get(port); 4421 4422 mutex_lock(&esw->state_lock); 4423 if (!vport->enabled) { 4424 err = -EOPNOTSUPP; 4425 goto unlock; 4426 } 4427 4428 *is_enabled = vport->info.ipsec_crypto_enabled; 4429unlock: 4430 mutex_unlock(&esw->state_lock); 4431 return err; 4432} 4433 4434int mlx5_devlink_port_fn_ipsec_crypto_set(struct devlink_port *port, bool enable, 4435 struct netlink_ext_ack *extack) 4436{ 4437 struct mlx5_eswitch *esw; 4438 struct mlx5_vport *vport; 4439 u16 vport_num; 4440 int err; 4441 4442 esw = mlx5_devlink_eswitch_get(port->devlink); 4443 if (IS_ERR(esw)) 4444 return PTR_ERR(esw); 4445 4446 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); 4447 err = mlx5_esw_ipsec_vf_crypto_offload_supported(esw->dev, vport_num); 4448 if (err) { 4449 NL_SET_ERR_MSG_MOD(extack, 4450 "Device doesn't support IPsec crypto"); 4451 return err; 4452 } 4453 4454 vport = mlx5_devlink_port_vport_get(port); 4455 4456 mutex_lock(&esw->state_lock); 4457 if (!vport->enabled) { 4458 err = -EOPNOTSUPP; 4459 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); 4460 goto unlock; 4461 } 4462 4463 if (vport->info.ipsec_crypto_enabled == enable) 4464 goto unlock; 4465 4466 if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) { 4467 err = -EBUSY; 4468 goto unlock; 4469 } 4470 4471 err = mlx5_esw_ipsec_vf_crypto_offload_set(esw, vport, enable); 4472 if (err) { 4473 NL_SET_ERR_MSG_MOD(extack, "Failed to set IPsec crypto"); 4474 goto unlock; 4475 } 4476 4477 vport->info.ipsec_crypto_enabled = enable; 4478 if (enable) 4479 esw->enabled_ipsec_vf_count++; 4480 else 4481 esw->enabled_ipsec_vf_count--; 4482unlock: 4483 mutex_unlock(&esw->state_lock); 4484 return err; 4485} 4486 4487int mlx5_devlink_port_fn_ipsec_packet_get(struct devlink_port *port, bool *is_enabled, 4488 struct netlink_ext_ack *extack) 4489{ 4490 struct mlx5_eswitch *esw; 4491 struct mlx5_vport *vport; 4492 int err = 0; 4493 4494 esw = mlx5_devlink_eswitch_get(port->devlink); 4495 if (IS_ERR(esw)) 4496 return PTR_ERR(esw); 4497 4498 if (!mlx5_esw_ipsec_vf_offload_supported(esw->dev)) { 4499 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPsec packet"); 4500 return -EOPNOTSUPP; 4501 } 4502 4503 vport = mlx5_devlink_port_vport_get(port); 4504 4505 mutex_lock(&esw->state_lock); 4506 if (!vport->enabled) { 4507 err = -EOPNOTSUPP; 4508 goto unlock; 4509 } 4510 4511 *is_enabled = vport->info.ipsec_packet_enabled; 4512unlock: 4513 mutex_unlock(&esw->state_lock); 4514 return err; 4515} 4516 4517int mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port *port, 4518 bool enable, 4519 struct netlink_ext_ack *extack) 4520{ 4521 struct mlx5_eswitch *esw; 4522 struct mlx5_vport *vport; 4523 u16 vport_num; 4524 int err; 4525 4526 esw = mlx5_devlink_eswitch_get(port->devlink); 4527 if (IS_ERR(esw)) 4528 return PTR_ERR(esw); 4529 4530 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); 4531 err = mlx5_esw_ipsec_vf_packet_offload_supported(esw->dev, vport_num); 4532 if (err) { 4533 NL_SET_ERR_MSG_MOD(extack, 4534 "Device doesn't support IPsec packet mode"); 4535 return err; 4536 } 4537 4538 vport = mlx5_devlink_port_vport_get(port); 4539 mutex_lock(&esw->state_lock); 4540 if (!vport->enabled) { 4541 err = -EOPNOTSUPP; 4542 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); 4543 goto unlock; 4544 } 4545 4546 if (vport->info.ipsec_packet_enabled == enable) 4547 goto unlock; 4548 4549 if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) { 4550 err = -EBUSY; 4551 goto unlock; 4552 } 4553 4554 err = mlx5_esw_ipsec_vf_packet_offload_set(esw, vport, enable); 4555 if (err) { 4556 NL_SET_ERR_MSG_MOD(extack, 4557 "Failed to set IPsec packet mode"); 4558 goto unlock; 4559 } 4560 4561 vport->info.ipsec_packet_enabled = enable; 4562 if (enable) 4563 esw->enabled_ipsec_vf_count++; 4564 else 4565 esw->enabled_ipsec_vf_count--; 4566unlock: 4567 mutex_unlock(&esw->state_lock); 4568 return err; 4569} 4570#endif /* CONFIG_XFRM_OFFLOAD */ 4571