1/* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#include <linux/etherdevice.h> 34#include <linux/idr.h> 35#include <linux/mlx5/driver.h> 36#include <linux/mlx5/mlx5_ifc.h> 37#include <linux/mlx5/vport.h> 38#include <linux/mlx5/fs.h> 39#include "mlx5_core.h" 40#include "eswitch.h" 41#include "esw/indir_table.h" 42#include "esw/acl/ofld.h" 43#include "rdma.h" 44#include "en.h" 45#include "fs_core.h" 46#include "lib/mlx5.h" 47#include "lib/devcom.h" 48#include "lib/eq.h" 49#include "lib/fs_chains.h" 50#include "en_tc.h" 51#include "en/mapping.h" 52#include "devlink.h" 53#include "lag/lag.h" 54#include "en/tc/post_meter.h" 55 56#define mlx5_esw_for_each_rep(esw, i, rep) \ 57 xa_for_each(&((esw)->offloads.vport_reps), i, rep) 58 59/* There are two match-all miss flows, one for unicast dst mac and 60 * one for multicast. 61 */ 62#define MLX5_ESW_MISS_FLOWS (2) 63#define UPLINK_REP_INDEX 0 64 65#define MLX5_ESW_VPORT_TBL_SIZE 128 66#define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4 67 68#define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1) 69 70#define MLX5_ESW_MAX_CTRL_EQS 4 71 72static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = { 73 .max_fte = MLX5_ESW_VPORT_TBL_SIZE, 74 .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS, 75 .flags = 0, 76}; 77 78static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw, 79 u16 vport_num) 80{ 81 return xa_load(&esw->offloads.vport_reps, vport_num); 82} 83 84static void 85mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw, 86 struct mlx5_flow_spec *spec, 87 struct mlx5_esw_flow_attr *attr) 88{ 89 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) || !attr || !attr->in_rep) 90 return; 91 92 if (attr->int_port) { 93 spec->flow_context.flow_source = mlx5e_tc_int_port_get_flow_source(attr->int_port); 94 95 return; 96 } 97 98 spec->flow_context.flow_source = (attr->in_rep->vport == MLX5_VPORT_UPLINK) ? 99 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK : 100 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; 101} 102 103/* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits 104 * are not needed as well in the following process. So clear them all for simplicity. 105 */ 106void 107mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec) 108{ 109 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 110 void *misc2; 111 112 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 113 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0); 114 115 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 116 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0); 117 118 if (!memchr_inv(misc2, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2))) 119 spec->match_criteria_enable &= ~MLX5_MATCH_MISC_PARAMETERS_2; 120 } 121} 122 123static void 124mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw, 125 struct mlx5_flow_spec *spec, 126 struct mlx5_flow_attr *attr, 127 struct mlx5_eswitch *src_esw, 128 u16 vport) 129{ 130 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 131 u32 metadata; 132 void *misc2; 133 void *misc; 134 135 /* Use metadata matching because vport is not represented by single 136 * VHCA in dual-port RoCE mode, and matching on source vport may fail. 137 */ 138 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 139 if (mlx5_esw_indir_table_decap_vport(attr)) 140 vport = mlx5_esw_indir_table_decap_vport(attr); 141 142 if (!attr->chain && esw_attr && esw_attr->int_port) 143 metadata = 144 mlx5e_tc_int_port_get_metadata_for_match(esw_attr->int_port); 145 else 146 metadata = 147 mlx5_eswitch_get_vport_metadata_for_match(src_esw, vport); 148 149 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 150 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, metadata); 151 152 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 153 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 154 mlx5_eswitch_get_vport_metadata_mask()); 155 156 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 157 } else { 158 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 159 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 160 161 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 162 MLX5_SET(fte_match_set_misc, misc, 163 source_eswitch_owner_vhca_id, 164 MLX5_CAP_GEN(src_esw->dev, vhca_id)); 165 166 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 167 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 168 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 169 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 170 source_eswitch_owner_vhca_id); 171 172 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 173 } 174} 175 176static int 177esw_setup_decap_indir(struct mlx5_eswitch *esw, 178 struct mlx5_flow_attr *attr) 179{ 180 struct mlx5_flow_table *ft; 181 182 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 183 return -EOPNOTSUPP; 184 185 ft = mlx5_esw_indir_table_get(esw, attr, 186 mlx5_esw_indir_table_decap_vport(attr), true); 187 return PTR_ERR_OR_ZERO(ft); 188} 189 190static void 191esw_cleanup_decap_indir(struct mlx5_eswitch *esw, 192 struct mlx5_flow_attr *attr) 193{ 194 if (mlx5_esw_indir_table_decap_vport(attr)) 195 mlx5_esw_indir_table_put(esw, 196 mlx5_esw_indir_table_decap_vport(attr), 197 true); 198} 199 200static int 201esw_setup_mtu_dest(struct mlx5_flow_destination *dest, 202 struct mlx5e_meter_attr *meter, 203 int i) 204{ 205 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_RANGE; 206 dest[i].range.field = MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN; 207 dest[i].range.min = 0; 208 dest[i].range.max = meter->params.mtu; 209 dest[i].range.hit_ft = mlx5e_post_meter_get_mtu_true_ft(meter->post_meter); 210 dest[i].range.miss_ft = mlx5e_post_meter_get_mtu_false_ft(meter->post_meter); 211 212 return 0; 213} 214 215static int 216esw_setup_sampler_dest(struct mlx5_flow_destination *dest, 217 struct mlx5_flow_act *flow_act, 218 u32 sampler_id, 219 int i) 220{ 221 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 222 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER; 223 dest[i].sampler_id = sampler_id; 224 225 return 0; 226} 227 228static int 229esw_setup_ft_dest(struct mlx5_flow_destination *dest, 230 struct mlx5_flow_act *flow_act, 231 struct mlx5_eswitch *esw, 232 struct mlx5_flow_attr *attr, 233 int i) 234{ 235 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 236 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 237 dest[i].ft = attr->dest_ft; 238 239 if (mlx5_esw_indir_table_decap_vport(attr)) 240 return esw_setup_decap_indir(esw, attr); 241 return 0; 242} 243 244static void 245esw_setup_accept_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 246 struct mlx5_fs_chains *chains, int i) 247{ 248 if (mlx5_chains_ignore_flow_level_supported(chains)) 249 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 250 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 251 dest[i].ft = mlx5_chains_get_tc_end_ft(chains); 252} 253 254static void 255esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 256 struct mlx5_eswitch *esw, int i) 257{ 258 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level)) 259 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 260 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 261 dest[i].ft = mlx5_eswitch_get_slow_fdb(esw); 262} 263 264static int 265esw_setup_chain_dest(struct mlx5_flow_destination *dest, 266 struct mlx5_flow_act *flow_act, 267 struct mlx5_fs_chains *chains, 268 u32 chain, u32 prio, u32 level, 269 int i) 270{ 271 struct mlx5_flow_table *ft; 272 273 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 274 ft = mlx5_chains_get_table(chains, chain, prio, level); 275 if (IS_ERR(ft)) 276 return PTR_ERR(ft); 277 278 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 279 dest[i].ft = ft; 280 return 0; 281} 282 283static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr, 284 int from, int to) 285{ 286 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 287 struct mlx5_fs_chains *chains = esw_chains(esw); 288 int i; 289 290 for (i = from; i < to; i++) 291 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 292 mlx5_chains_put_table(chains, 0, 1, 0); 293 else if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].vport, 294 esw_attr->dests[i].mdev)) 295 mlx5_esw_indir_table_put(esw, esw_attr->dests[i].vport, false); 296} 297 298static bool 299esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr) 300{ 301 int i; 302 303 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) 304 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 305 return true; 306 return false; 307} 308 309static int 310esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest, 311 struct mlx5_flow_act *flow_act, 312 struct mlx5_eswitch *esw, 313 struct mlx5_fs_chains *chains, 314 struct mlx5_flow_attr *attr, 315 int *i) 316{ 317 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 318 int err; 319 320 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 321 return -EOPNOTSUPP; 322 323 /* flow steering cannot handle more than one dest with the same ft 324 * in a single flow 325 */ 326 if (esw_attr->out_count - esw_attr->split_count > 1) 327 return -EOPNOTSUPP; 328 329 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i); 330 if (err) 331 return err; 332 333 if (esw_attr->dests[esw_attr->split_count].pkt_reformat) { 334 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 335 flow_act->pkt_reformat = esw_attr->dests[esw_attr->split_count].pkt_reformat; 336 } 337 (*i)++; 338 339 return 0; 340} 341 342static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw, 343 struct mlx5_flow_attr *attr) 344{ 345 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 346 347 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count); 348} 349 350static bool 351esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) 352{ 353 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 354 bool result = false; 355 int i; 356 357 /* Indirect table is supported only for flows with in_port uplink 358 * and the destination is vport on the same eswitch as the uplink, 359 * return false in case at least one of destinations doesn't meet 360 * this criteria. 361 */ 362 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) { 363 if (esw_attr->dests[i].vport_valid && 364 mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].vport, 365 esw_attr->dests[i].mdev)) { 366 result = true; 367 } else { 368 result = false; 369 break; 370 } 371 } 372 return result; 373} 374 375static int 376esw_setup_indir_table(struct mlx5_flow_destination *dest, 377 struct mlx5_flow_act *flow_act, 378 struct mlx5_eswitch *esw, 379 struct mlx5_flow_attr *attr, 380 int *i) 381{ 382 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 383 int j, err; 384 385 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 386 return -EOPNOTSUPP; 387 388 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) { 389 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 390 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 391 392 dest[*i].ft = mlx5_esw_indir_table_get(esw, attr, 393 esw_attr->dests[j].vport, false); 394 if (IS_ERR(dest[*i].ft)) { 395 err = PTR_ERR(dest[*i].ft); 396 goto err_indir_tbl_get; 397 } 398 } 399 400 if (mlx5_esw_indir_table_decap_vport(attr)) { 401 err = esw_setup_decap_indir(esw, attr); 402 if (err) 403 goto err_indir_tbl_get; 404 } 405 406 return 0; 407 408err_indir_tbl_get: 409 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j); 410 return err; 411} 412 413static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) 414{ 415 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 416 417 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count); 418 esw_cleanup_decap_indir(esw, attr); 419} 420 421static void 422esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level) 423{ 424 mlx5_chains_put_table(chains, chain, prio, level); 425} 426 427static bool esw_same_vhca_id(struct mlx5_core_dev *mdev1, struct mlx5_core_dev *mdev2) 428{ 429 return MLX5_CAP_GEN(mdev1, vhca_id) == MLX5_CAP_GEN(mdev2, vhca_id); 430} 431 432static bool esw_setup_uplink_fwd_ipsec_needed(struct mlx5_eswitch *esw, 433 struct mlx5_esw_flow_attr *esw_attr, 434 int attr_idx) 435{ 436 if (esw->offloads.ft_ipsec_tx_pol && 437 esw_attr->dests[attr_idx].vport_valid && 438 esw_attr->dests[attr_idx].vport == MLX5_VPORT_UPLINK && 439 /* To be aligned with software, encryption is needed only for tunnel device */ 440 (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) && 441 esw_attr->dests[attr_idx].vport != esw_attr->in_rep->vport && 442 esw_same_vhca_id(esw_attr->dests[attr_idx].mdev, esw->dev)) 443 return true; 444 445 return false; 446} 447 448static bool esw_flow_dests_fwd_ipsec_check(struct mlx5_eswitch *esw, 449 struct mlx5_esw_flow_attr *esw_attr) 450{ 451 int i; 452 453 if (!esw->offloads.ft_ipsec_tx_pol) 454 return true; 455 456 for (i = 0; i < esw_attr->split_count; i++) 457 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i)) 458 return false; 459 460 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) 461 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i) && 462 (esw_attr->out_count - esw_attr->split_count > 1)) 463 return false; 464 465 return true; 466} 467 468static void 469esw_setup_dest_fwd_vport(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 470 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 471 int attr_idx, int dest_idx, bool pkt_reformat) 472{ 473 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 474 dest[dest_idx].vport.num = esw_attr->dests[attr_idx].vport; 475 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) { 476 dest[dest_idx].vport.vhca_id = 477 MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id); 478 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 479 if (dest[dest_idx].vport.num == MLX5_VPORT_UPLINK && 480 mlx5_lag_is_mpesw(esw->dev)) 481 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_UPLINK; 482 } 483 if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) { 484 if (pkt_reformat) { 485 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 486 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 487 } 488 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID; 489 dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 490 } 491} 492 493static void 494esw_setup_dest_fwd_ipsec(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 495 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 496 int attr_idx, int dest_idx, bool pkt_reformat) 497{ 498 dest[dest_idx].ft = esw->offloads.ft_ipsec_tx_pol; 499 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 500 if (pkt_reformat && 501 esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) { 502 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 503 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 504 } 505} 506 507static void 508esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 509 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 510 int attr_idx, int dest_idx, bool pkt_reformat) 511{ 512 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx)) 513 esw_setup_dest_fwd_ipsec(dest, flow_act, esw, esw_attr, 514 attr_idx, dest_idx, pkt_reformat); 515 else 516 esw_setup_dest_fwd_vport(dest, flow_act, esw, esw_attr, 517 attr_idx, dest_idx, pkt_reformat); 518} 519 520static int 521esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 522 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 523 int i) 524{ 525 int j; 526 527 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++) 528 esw_setup_vport_dest(dest, flow_act, esw, esw_attr, j, i, true); 529 return i; 530} 531 532static bool 533esw_src_port_rewrite_supported(struct mlx5_eswitch *esw) 534{ 535 return MLX5_CAP_GEN(esw->dev, reg_c_preserve) && 536 mlx5_eswitch_vport_match_metadata_enabled(esw) && 537 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level); 538} 539 540static bool 541esw_dests_to_int_external(struct mlx5_flow_destination *dests, int max_dest) 542{ 543 bool internal_dest = false, external_dest = false; 544 int i; 545 546 for (i = 0; i < max_dest; i++) { 547 if (dests[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT && 548 dests[i].type != MLX5_FLOW_DESTINATION_TYPE_UPLINK) 549 continue; 550 551 /* Uplink dest is external, but considered as internal 552 * if there is reformat because firmware uses LB+hairpin to support it. 553 */ 554 if (dests[i].vport.num == MLX5_VPORT_UPLINK && 555 !(dests[i].vport.flags & MLX5_FLOW_DEST_VPORT_REFORMAT_ID)) 556 external_dest = true; 557 else 558 internal_dest = true; 559 560 if (internal_dest && external_dest) 561 return true; 562 } 563 564 return false; 565} 566 567static int 568esw_setup_dests(struct mlx5_flow_destination *dest, 569 struct mlx5_flow_act *flow_act, 570 struct mlx5_eswitch *esw, 571 struct mlx5_flow_attr *attr, 572 struct mlx5_flow_spec *spec, 573 int *i) 574{ 575 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 576 struct mlx5_fs_chains *chains = esw_chains(esw); 577 int err = 0; 578 579 if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) && 580 esw_src_port_rewrite_supported(esw)) 581 attr->flags |= MLX5_ATTR_FLAG_SRC_REWRITE; 582 583 if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) { 584 esw_setup_slow_path_dest(dest, flow_act, esw, *i); 585 (*i)++; 586 goto out; 587 } 588 589 if (attr->flags & MLX5_ATTR_FLAG_SAMPLE) { 590 esw_setup_sampler_dest(dest, flow_act, attr->sample_attr.sampler_id, *i); 591 (*i)++; 592 } else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) { 593 esw_setup_accept_dest(dest, flow_act, chains, *i); 594 (*i)++; 595 } else if (attr->flags & MLX5_ATTR_FLAG_MTU) { 596 err = esw_setup_mtu_dest(dest, &attr->meter_attr, *i); 597 (*i)++; 598 } else if (esw_is_indir_table(esw, attr)) { 599 err = esw_setup_indir_table(dest, flow_act, esw, attr, i); 600 } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) { 601 err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i); 602 } else { 603 *i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i); 604 605 if (attr->dest_ft) { 606 err = esw_setup_ft_dest(dest, flow_act, esw, attr, *i); 607 (*i)++; 608 } else if (attr->dest_chain) { 609 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 610 1, 0, *i); 611 (*i)++; 612 } 613 } 614 615out: 616 return err; 617} 618 619static void 620esw_cleanup_dests(struct mlx5_eswitch *esw, 621 struct mlx5_flow_attr *attr) 622{ 623 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 624 struct mlx5_fs_chains *chains = esw_chains(esw); 625 626 if (attr->dest_ft) { 627 esw_cleanup_decap_indir(esw, attr); 628 } else if (!mlx5e_tc_attr_flags_skip(attr->flags)) { 629 if (attr->dest_chain) 630 esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0); 631 else if (esw_is_indir_table(esw, attr)) 632 esw_cleanup_indir_table(esw, attr); 633 else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) 634 esw_cleanup_chain_src_port_rewrite(esw, attr); 635 } 636} 637 638static void 639esw_setup_meter(struct mlx5_flow_attr *attr, struct mlx5_flow_act *flow_act) 640{ 641 struct mlx5e_flow_meter_handle *meter; 642 643 meter = attr->meter_attr.meter; 644 flow_act->exe_aso.type = attr->exe_aso_type; 645 flow_act->exe_aso.object_id = meter->obj_id; 646 flow_act->exe_aso.flow_meter.meter_idx = meter->idx; 647 flow_act->exe_aso.flow_meter.init_color = MLX5_FLOW_METER_COLOR_GREEN; 648 /* use metadata reg 5 for packet color */ 649 flow_act->exe_aso.return_reg_id = 5; 650} 651 652struct mlx5_flow_handle * 653mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, 654 struct mlx5_flow_spec *spec, 655 struct mlx5_flow_attr *attr) 656{ 657 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 658 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 659 struct mlx5_fs_chains *chains = esw_chains(esw); 660 bool split = !!(esw_attr->split_count); 661 struct mlx5_vport_tbl_attr fwd_attr; 662 struct mlx5_flow_destination *dest; 663 struct mlx5_flow_handle *rule; 664 struct mlx5_flow_table *fdb; 665 int i = 0; 666 667 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 668 return ERR_PTR(-EOPNOTSUPP); 669 670 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 671 return ERR_PTR(-EOPNOTSUPP); 672 673 if (!esw_flow_dests_fwd_ipsec_check(esw, esw_attr)) 674 return ERR_PTR(-EOPNOTSUPP); 675 676 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL); 677 if (!dest) 678 return ERR_PTR(-ENOMEM); 679 680 flow_act.action = attr->action; 681 682 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) { 683 flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]); 684 flow_act.vlan[0].vid = esw_attr->vlan_vid[0]; 685 flow_act.vlan[0].prio = esw_attr->vlan_prio[0]; 686 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) { 687 flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]); 688 flow_act.vlan[1].vid = esw_attr->vlan_vid[1]; 689 flow_act.vlan[1].prio = esw_attr->vlan_prio[1]; 690 } 691 } 692 693 mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr); 694 695 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) { 696 int err; 697 698 err = esw_setup_dests(dest, &flow_act, esw, attr, spec, &i); 699 if (err) { 700 rule = ERR_PTR(err); 701 goto err_create_goto_table; 702 } 703 704 /* Header rewrite with combined wire+loopback in FDB is not allowed */ 705 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) && 706 esw_dests_to_int_external(dest, i)) { 707 esw_warn(esw->dev, 708 "FDB: Header rewrite with forwarding to both internal and external dests is not allowed\n"); 709 rule = ERR_PTR(-EINVAL); 710 goto err_esw_get; 711 } 712 } 713 714 if (esw_attr->decap_pkt_reformat) 715 flow_act.pkt_reformat = esw_attr->decap_pkt_reformat; 716 717 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 718 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 719 dest[i].counter_id = mlx5_fc_id(attr->counter); 720 i++; 721 } 722 723 if (attr->outer_match_level != MLX5_MATCH_NONE) 724 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 725 if (attr->inner_match_level != MLX5_MATCH_NONE) 726 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS; 727 728 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 729 flow_act.modify_hdr = attr->modify_hdr; 730 731 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) && 732 attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER) 733 esw_setup_meter(attr, &flow_act); 734 735 if (split) { 736 fwd_attr.chain = attr->chain; 737 fwd_attr.prio = attr->prio; 738 fwd_attr.vport = esw_attr->in_rep->vport; 739 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 740 741 fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr); 742 } else { 743 if (attr->chain || attr->prio) 744 fdb = mlx5_chains_get_table(chains, attr->chain, 745 attr->prio, 0); 746 else 747 fdb = attr->ft; 748 749 if (!(attr->flags & MLX5_ATTR_FLAG_NO_IN_PORT)) 750 mlx5_eswitch_set_rule_source_port(esw, spec, attr, 751 esw_attr->in_mdev->priv.eswitch, 752 esw_attr->in_rep->vport); 753 } 754 if (IS_ERR(fdb)) { 755 rule = ERR_CAST(fdb); 756 goto err_esw_get; 757 } 758 759 if (!i) { 760 kfree(dest); 761 dest = NULL; 762 } 763 764 if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec)) 765 rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr, 766 &flow_act, dest, i); 767 else 768 rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i); 769 if (IS_ERR(rule)) 770 goto err_add_rule; 771 else 772 atomic64_inc(&esw->offloads.num_flows); 773 774 kfree(dest); 775 return rule; 776 777err_add_rule: 778 if (split) 779 mlx5_esw_vporttbl_put(esw, &fwd_attr); 780 else if (attr->chain || attr->prio) 781 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 782err_esw_get: 783 esw_cleanup_dests(esw, attr); 784err_create_goto_table: 785 kfree(dest); 786 return rule; 787} 788 789struct mlx5_flow_handle * 790mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, 791 struct mlx5_flow_spec *spec, 792 struct mlx5_flow_attr *attr) 793{ 794 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 795 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 796 struct mlx5_fs_chains *chains = esw_chains(esw); 797 struct mlx5_vport_tbl_attr fwd_attr; 798 struct mlx5_flow_destination *dest; 799 struct mlx5_flow_table *fast_fdb; 800 struct mlx5_flow_table *fwd_fdb; 801 struct mlx5_flow_handle *rule; 802 int i, err = 0; 803 804 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL); 805 if (!dest) 806 return ERR_PTR(-ENOMEM); 807 808 fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0); 809 if (IS_ERR(fast_fdb)) { 810 rule = ERR_CAST(fast_fdb); 811 goto err_get_fast; 812 } 813 814 fwd_attr.chain = attr->chain; 815 fwd_attr.prio = attr->prio; 816 fwd_attr.vport = esw_attr->in_rep->vport; 817 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 818 fwd_fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr); 819 if (IS_ERR(fwd_fdb)) { 820 rule = ERR_CAST(fwd_fdb); 821 goto err_get_fwd; 822 } 823 824 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 825 for (i = 0; i < esw_attr->split_count; i++) { 826 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 827 /* Source port rewrite (forward to ovs internal port or statck device) isn't 828 * supported in the rule of split action. 829 */ 830 err = -EOPNOTSUPP; 831 else 832 esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false); 833 834 if (err) { 835 rule = ERR_PTR(err); 836 goto err_chain_src_rewrite; 837 } 838 } 839 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 840 dest[i].ft = fwd_fdb; 841 i++; 842 843 mlx5_eswitch_set_rule_source_port(esw, spec, attr, 844 esw_attr->in_mdev->priv.eswitch, 845 esw_attr->in_rep->vport); 846 847 if (attr->outer_match_level != MLX5_MATCH_NONE) 848 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 849 850 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 851 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i); 852 853 if (IS_ERR(rule)) { 854 i = esw_attr->split_count; 855 goto err_chain_src_rewrite; 856 } 857 858 atomic64_inc(&esw->offloads.num_flows); 859 860 kfree(dest); 861 return rule; 862err_chain_src_rewrite: 863 mlx5_esw_vporttbl_put(esw, &fwd_attr); 864err_get_fwd: 865 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 866err_get_fast: 867 kfree(dest); 868 return rule; 869} 870 871static void 872__mlx5_eswitch_del_rule(struct mlx5_eswitch *esw, 873 struct mlx5_flow_handle *rule, 874 struct mlx5_flow_attr *attr, 875 bool fwd_rule) 876{ 877 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 878 struct mlx5_fs_chains *chains = esw_chains(esw); 879 bool split = (esw_attr->split_count > 0); 880 struct mlx5_vport_tbl_attr fwd_attr; 881 int i; 882 883 mlx5_del_flow_rules(rule); 884 885 if (!mlx5e_tc_attr_flags_skip(attr->flags)) { 886 /* unref the term table */ 887 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) { 888 if (esw_attr->dests[i].termtbl) 889 mlx5_eswitch_termtbl_put(esw, esw_attr->dests[i].termtbl); 890 } 891 } 892 893 atomic64_dec(&esw->offloads.num_flows); 894 895 if (fwd_rule || split) { 896 fwd_attr.chain = attr->chain; 897 fwd_attr.prio = attr->prio; 898 fwd_attr.vport = esw_attr->in_rep->vport; 899 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 900 } 901 902 if (fwd_rule) { 903 mlx5_esw_vporttbl_put(esw, &fwd_attr); 904 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 905 } else { 906 if (split) 907 mlx5_esw_vporttbl_put(esw, &fwd_attr); 908 else if (attr->chain || attr->prio) 909 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 910 esw_cleanup_dests(esw, attr); 911 } 912} 913 914void 915mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw, 916 struct mlx5_flow_handle *rule, 917 struct mlx5_flow_attr *attr) 918{ 919 __mlx5_eswitch_del_rule(esw, rule, attr, false); 920} 921 922void 923mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw, 924 struct mlx5_flow_handle *rule, 925 struct mlx5_flow_attr *attr) 926{ 927 __mlx5_eswitch_del_rule(esw, rule, attr, true); 928} 929 930struct mlx5_flow_handle * 931mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw, 932 struct mlx5_eswitch *from_esw, 933 struct mlx5_eswitch_rep *rep, 934 u32 sqn) 935{ 936 struct mlx5_flow_act flow_act = {0}; 937 struct mlx5_flow_destination dest = {}; 938 struct mlx5_flow_handle *flow_rule; 939 struct mlx5_flow_spec *spec; 940 void *misc; 941 u16 vport; 942 943 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 944 if (!spec) { 945 flow_rule = ERR_PTR(-ENOMEM); 946 goto out; 947 } 948 949 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 950 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn); 951 952 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 953 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn); 954 955 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 956 957 /* source vport is the esw manager */ 958 vport = from_esw->manager_vport; 959 960 if (mlx5_eswitch_vport_match_metadata_enabled(on_esw)) { 961 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 962 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 963 mlx5_eswitch_get_vport_metadata_for_match(from_esw, vport)); 964 965 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 966 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 967 mlx5_eswitch_get_vport_metadata_mask()); 968 969 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 970 } else { 971 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 972 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 973 974 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch)) 975 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 976 MLX5_CAP_GEN(from_esw->dev, vhca_id)); 977 978 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 979 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 980 981 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch)) 982 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 983 source_eswitch_owner_vhca_id); 984 985 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 986 } 987 988 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 989 dest.vport.num = rep->vport; 990 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id); 991 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 992 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 993 994 if (rep->vport == MLX5_VPORT_UPLINK && 995 on_esw == from_esw && on_esw->offloads.ft_ipsec_tx_pol) { 996 dest.ft = on_esw->offloads.ft_ipsec_tx_pol; 997 flow_act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL; 998 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 999 } else { 1000 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1001 dest.vport.num = rep->vport; 1002 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id); 1003 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 1004 } 1005 1006 if (MLX5_CAP_ESW_FLOWTABLE(on_esw->dev, flow_source) && 1007 rep->vport == MLX5_VPORT_UPLINK) 1008 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; 1009 1010 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(on_esw), 1011 spec, &flow_act, &dest, 1); 1012 if (IS_ERR(flow_rule)) 1013 esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %ld\n", 1014 PTR_ERR(flow_rule)); 1015out: 1016 kvfree(spec); 1017 return flow_rule; 1018} 1019EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule); 1020 1021void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule) 1022{ 1023 mlx5_del_flow_rules(rule); 1024} 1025 1026void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule) 1027{ 1028 if (rule) 1029 mlx5_del_flow_rules(rule); 1030} 1031 1032struct mlx5_flow_handle * 1033mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num) 1034{ 1035 struct mlx5_flow_destination dest = {}; 1036 struct mlx5_flow_act flow_act = {0}; 1037 struct mlx5_flow_handle *flow_rule; 1038 struct mlx5_flow_spec *spec; 1039 1040 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1041 if (!spec) 1042 return ERR_PTR(-ENOMEM); 1043 1044 MLX5_SET(fte_match_param, spec->match_criteria, 1045 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); 1046 MLX5_SET(fte_match_param, spec->match_criteria, 1047 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); 1048 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1, 1049 ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK); 1050 1051 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1052 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1053 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1054 1055 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0, 1056 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num)); 1057 dest.vport.num = vport_num; 1058 1059 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1060 spec, &flow_act, &dest, 1); 1061 if (IS_ERR(flow_rule)) 1062 esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %ld\n", 1063 vport_num, PTR_ERR(flow_rule)); 1064 1065 kvfree(spec); 1066 return flow_rule; 1067} 1068 1069static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw) 1070{ 1071 return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 1072 MLX5_FDB_TO_VPORT_REG_C_1; 1073} 1074 1075static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable) 1076{ 1077 u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {}; 1078 u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {}; 1079 u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {}; 1080 u8 curr, wanted; 1081 int err; 1082 1083 if (!mlx5_eswitch_reg_c1_loopback_supported(esw) && 1084 !mlx5_eswitch_vport_match_metadata_enabled(esw)) 1085 return 0; 1086 1087 MLX5_SET(query_esw_vport_context_in, in, opcode, 1088 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT); 1089 err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out); 1090 if (err) 1091 return err; 1092 1093 curr = MLX5_GET(query_esw_vport_context_out, out, 1094 esw_vport_context.fdb_to_vport_reg_c_id); 1095 wanted = MLX5_FDB_TO_VPORT_REG_C_0; 1096 if (mlx5_eswitch_reg_c1_loopback_supported(esw)) 1097 wanted |= MLX5_FDB_TO_VPORT_REG_C_1; 1098 1099 if (enable) 1100 curr |= wanted; 1101 else 1102 curr &= ~wanted; 1103 1104 MLX5_SET(modify_esw_vport_context_in, min, 1105 esw_vport_context.fdb_to_vport_reg_c_id, curr); 1106 MLX5_SET(modify_esw_vport_context_in, min, 1107 field_select.fdb_to_vport_reg_c_id, 1); 1108 1109 err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min); 1110 if (!err) { 1111 if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1)) 1112 esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 1113 else 1114 esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 1115 } 1116 1117 return err; 1118} 1119 1120static void peer_miss_rules_setup(struct mlx5_eswitch *esw, 1121 struct mlx5_core_dev *peer_dev, 1122 struct mlx5_flow_spec *spec, 1123 struct mlx5_flow_destination *dest) 1124{ 1125 void *misc; 1126 1127 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1128 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1129 misc_parameters_2); 1130 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1131 mlx5_eswitch_get_vport_metadata_mask()); 1132 1133 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1134 } else { 1135 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1136 misc_parameters); 1137 1138 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 1139 MLX5_CAP_GEN(peer_dev, vhca_id)); 1140 1141 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 1142 1143 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1144 misc_parameters); 1145 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1146 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 1147 source_eswitch_owner_vhca_id); 1148 } 1149 1150 dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1151 dest->vport.num = peer_dev->priv.eswitch->manager_vport; 1152 dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id); 1153 dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 1154} 1155 1156static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw, 1157 struct mlx5_eswitch *peer_esw, 1158 struct mlx5_flow_spec *spec, 1159 u16 vport) 1160{ 1161 void *misc; 1162 1163 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1164 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1165 misc_parameters_2); 1166 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1167 mlx5_eswitch_get_vport_metadata_for_match(peer_esw, 1168 vport)); 1169 } else { 1170 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1171 misc_parameters); 1172 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 1173 } 1174} 1175 1176static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, 1177 struct mlx5_core_dev *peer_dev) 1178{ 1179 struct mlx5_flow_destination dest = {}; 1180 struct mlx5_flow_act flow_act = {0}; 1181 struct mlx5_flow_handle **flows; 1182 /* total vports is the same for both e-switches */ 1183 int nvports = esw->total_vports; 1184 struct mlx5_flow_handle *flow; 1185 struct mlx5_flow_spec *spec; 1186 struct mlx5_vport *vport; 1187 int err, pfindex; 1188 unsigned long i; 1189 void *misc; 1190 1191 if (!MLX5_VPORT_MANAGER(esw->dev) && !mlx5_core_is_ecpf_esw_manager(esw->dev)) 1192 return 0; 1193 1194 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1195 if (!spec) 1196 return -ENOMEM; 1197 1198 peer_miss_rules_setup(esw, peer_dev, spec, &dest); 1199 1200 flows = kvcalloc(nvports, sizeof(*flows), GFP_KERNEL); 1201 if (!flows) { 1202 err = -ENOMEM; 1203 goto alloc_flows_err; 1204 } 1205 1206 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1207 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1208 misc_parameters); 1209 1210 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1211 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); 1212 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch, 1213 spec, MLX5_VPORT_PF); 1214 1215 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1216 spec, &flow_act, &dest, 1); 1217 if (IS_ERR(flow)) { 1218 err = PTR_ERR(flow); 1219 goto add_pf_flow_err; 1220 } 1221 flows[vport->index] = flow; 1222 } 1223 1224 if (mlx5_ecpf_vport_exists(esw->dev)) { 1225 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); 1226 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF); 1227 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1228 spec, &flow_act, &dest, 1); 1229 if (IS_ERR(flow)) { 1230 err = PTR_ERR(flow); 1231 goto add_ecpf_flow_err; 1232 } 1233 flows[vport->index] = flow; 1234 } 1235 1236 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) { 1237 esw_set_peer_miss_rule_source_port(esw, 1238 peer_dev->priv.eswitch, 1239 spec, vport->vport); 1240 1241 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1242 spec, &flow_act, &dest, 1); 1243 if (IS_ERR(flow)) { 1244 err = PTR_ERR(flow); 1245 goto add_vf_flow_err; 1246 } 1247 flows[vport->index] = flow; 1248 } 1249 1250 if (mlx5_core_ec_sriov_enabled(esw->dev)) { 1251 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) { 1252 if (i >= mlx5_core_max_ec_vfs(peer_dev)) 1253 break; 1254 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch, 1255 spec, vport->vport); 1256 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1257 spec, &flow_act, &dest, 1); 1258 if (IS_ERR(flow)) { 1259 err = PTR_ERR(flow); 1260 goto add_ec_vf_flow_err; 1261 } 1262 flows[vport->index] = flow; 1263 } 1264 } 1265 1266 pfindex = mlx5_get_dev_index(peer_dev); 1267 if (pfindex >= MLX5_MAX_PORTS) { 1268 esw_warn(esw->dev, "Peer dev index(%d) is over the max num defined(%d)\n", 1269 pfindex, MLX5_MAX_PORTS); 1270 err = -EINVAL; 1271 goto add_ec_vf_flow_err; 1272 } 1273 esw->fdb_table.offloads.peer_miss_rules[pfindex] = flows; 1274 1275 kvfree(spec); 1276 return 0; 1277 1278add_ec_vf_flow_err: 1279 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) { 1280 if (!flows[vport->index]) 1281 continue; 1282 mlx5_del_flow_rules(flows[vport->index]); 1283 } 1284add_vf_flow_err: 1285 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) { 1286 if (!flows[vport->index]) 1287 continue; 1288 mlx5_del_flow_rules(flows[vport->index]); 1289 } 1290 if (mlx5_ecpf_vport_exists(esw->dev)) { 1291 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); 1292 mlx5_del_flow_rules(flows[vport->index]); 1293 } 1294add_ecpf_flow_err: 1295 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1296 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); 1297 mlx5_del_flow_rules(flows[vport->index]); 1298 } 1299add_pf_flow_err: 1300 esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err); 1301 kvfree(flows); 1302alloc_flows_err: 1303 kvfree(spec); 1304 return err; 1305} 1306 1307static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw, 1308 struct mlx5_core_dev *peer_dev) 1309{ 1310 u16 peer_index = mlx5_get_dev_index(peer_dev); 1311 struct mlx5_flow_handle **flows; 1312 struct mlx5_vport *vport; 1313 unsigned long i; 1314 1315 flows = esw->fdb_table.offloads.peer_miss_rules[peer_index]; 1316 if (!flows) 1317 return; 1318 1319 if (mlx5_core_ec_sriov_enabled(esw->dev)) { 1320 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) { 1321 /* The flow for a particular vport could be NULL if the other ECPF 1322 * has fewer or no VFs enabled 1323 */ 1324 if (!flows[vport->index]) 1325 continue; 1326 mlx5_del_flow_rules(flows[vport->index]); 1327 } 1328 } 1329 1330 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) 1331 mlx5_del_flow_rules(flows[vport->index]); 1332 1333 if (mlx5_ecpf_vport_exists(esw->dev)) { 1334 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); 1335 mlx5_del_flow_rules(flows[vport->index]); 1336 } 1337 1338 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1339 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); 1340 mlx5_del_flow_rules(flows[vport->index]); 1341 } 1342 1343 kvfree(flows); 1344 esw->fdb_table.offloads.peer_miss_rules[peer_index] = NULL; 1345} 1346 1347static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw) 1348{ 1349 struct mlx5_flow_act flow_act = {0}; 1350 struct mlx5_flow_destination dest = {}; 1351 struct mlx5_flow_handle *flow_rule = NULL; 1352 struct mlx5_flow_spec *spec; 1353 void *headers_c; 1354 void *headers_v; 1355 int err = 0; 1356 u8 *dmac_c; 1357 u8 *dmac_v; 1358 1359 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1360 if (!spec) { 1361 err = -ENOMEM; 1362 goto out; 1363 } 1364 1365 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1366 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1367 outer_headers); 1368 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, 1369 outer_headers.dmac_47_16); 1370 dmac_c[0] = 0x01; 1371 1372 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1373 dest.vport.num = esw->manager_vport; 1374 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1375 1376 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1377 spec, &flow_act, &dest, 1); 1378 if (IS_ERR(flow_rule)) { 1379 err = PTR_ERR(flow_rule); 1380 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err); 1381 goto out; 1382 } 1383 1384 esw->fdb_table.offloads.miss_rule_uni = flow_rule; 1385 1386 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1387 outer_headers); 1388 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, 1389 outer_headers.dmac_47_16); 1390 dmac_v[0] = 0x01; 1391 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1392 spec, &flow_act, &dest, 1); 1393 if (IS_ERR(flow_rule)) { 1394 err = PTR_ERR(flow_rule); 1395 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err); 1396 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1397 goto out; 1398 } 1399 1400 esw->fdb_table.offloads.miss_rule_multi = flow_rule; 1401 1402out: 1403 kvfree(spec); 1404 return err; 1405} 1406 1407struct mlx5_flow_handle * 1408esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag) 1409{ 1410 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 1411 struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore; 1412 struct mlx5_flow_context *flow_context; 1413 struct mlx5_flow_handle *flow_rule; 1414 struct mlx5_flow_destination dest; 1415 struct mlx5_flow_spec *spec; 1416 void *misc; 1417 1418 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 1419 return ERR_PTR(-EOPNOTSUPP); 1420 1421 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1422 if (!spec) 1423 return ERR_PTR(-ENOMEM); 1424 1425 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1426 misc_parameters_2); 1427 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1428 ESW_REG_C0_USER_DATA_METADATA_MASK); 1429 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1430 misc_parameters_2); 1431 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag); 1432 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1433 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 1434 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 1435 flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id; 1436 1437 flow_context = &spec->flow_context; 1438 flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 1439 flow_context->flow_tag = tag; 1440 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1441 dest.ft = esw->offloads.ft_offloads; 1442 1443 flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 1444 kvfree(spec); 1445 1446 if (IS_ERR(flow_rule)) 1447 esw_warn(esw->dev, 1448 "Failed to create restore rule for tag: %d, err(%d)\n", 1449 tag, (int)PTR_ERR(flow_rule)); 1450 1451 return flow_rule; 1452} 1453 1454#define MAX_PF_SQ 256 1455#define MAX_SQ_NVPORTS 32 1456 1457void 1458mlx5_esw_set_flow_group_source_port(struct mlx5_eswitch *esw, 1459 u32 *flow_group_in, 1460 int match_params) 1461{ 1462 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1463 flow_group_in, 1464 match_criteria); 1465 1466 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1467 MLX5_SET(create_flow_group_in, flow_group_in, 1468 match_criteria_enable, 1469 MLX5_MATCH_MISC_PARAMETERS_2 | match_params); 1470 1471 MLX5_SET(fte_match_param, match_criteria, 1472 misc_parameters_2.metadata_reg_c_0, 1473 mlx5_eswitch_get_vport_metadata_mask()); 1474 } else { 1475 MLX5_SET(create_flow_group_in, flow_group_in, 1476 match_criteria_enable, 1477 MLX5_MATCH_MISC_PARAMETERS | match_params); 1478 1479 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1480 misc_parameters.source_port); 1481 } 1482} 1483 1484#if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 1485static void esw_vport_tbl_put(struct mlx5_eswitch *esw) 1486{ 1487 struct mlx5_vport_tbl_attr attr; 1488 struct mlx5_vport *vport; 1489 unsigned long i; 1490 1491 attr.chain = 0; 1492 attr.prio = 1; 1493 mlx5_esw_for_each_vport(esw, i, vport) { 1494 attr.vport = vport->vport; 1495 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 1496 mlx5_esw_vporttbl_put(esw, &attr); 1497 } 1498} 1499 1500static int esw_vport_tbl_get(struct mlx5_eswitch *esw) 1501{ 1502 struct mlx5_vport_tbl_attr attr; 1503 struct mlx5_flow_table *fdb; 1504 struct mlx5_vport *vport; 1505 unsigned long i; 1506 1507 attr.chain = 0; 1508 attr.prio = 1; 1509 mlx5_esw_for_each_vport(esw, i, vport) { 1510 attr.vport = vport->vport; 1511 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 1512 fdb = mlx5_esw_vporttbl_get(esw, &attr); 1513 if (IS_ERR(fdb)) 1514 goto out; 1515 } 1516 return 0; 1517 1518out: 1519 esw_vport_tbl_put(esw); 1520 return PTR_ERR(fdb); 1521} 1522 1523#define fdb_modify_header_fwd_to_table_supported(esw) \ 1524 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table)) 1525static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags) 1526{ 1527 struct mlx5_core_dev *dev = esw->dev; 1528 1529 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level)) 1530 *flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED; 1531 1532 if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) && 1533 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) { 1534 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1535 esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n"); 1536 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) { 1537 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1538 esw_warn(dev, "Tc chains and priorities offload aren't supported\n"); 1539 } else if (!fdb_modify_header_fwd_to_table_supported(esw)) { 1540 /* Disabled when ttl workaround is needed, e.g 1541 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig 1542 */ 1543 esw_warn(dev, 1544 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n"); 1545 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1546 } else { 1547 *flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1548 esw_info(dev, "Supported tc chains and prios offload\n"); 1549 } 1550 1551 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1552 *flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED; 1553} 1554 1555static int 1556esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1557{ 1558 struct mlx5_core_dev *dev = esw->dev; 1559 struct mlx5_flow_table *nf_ft, *ft; 1560 struct mlx5_chains_attr attr = {}; 1561 struct mlx5_fs_chains *chains; 1562 int err; 1563 1564 esw_init_chains_offload_flags(esw, &attr.flags); 1565 attr.ns = MLX5_FLOW_NAMESPACE_FDB; 1566 attr.max_grp_num = esw->params.large_group_num; 1567 attr.default_ft = miss_fdb; 1568 attr.mapping = esw->offloads.reg_c0_obj_pool; 1569 1570 chains = mlx5_chains_create(dev, &attr); 1571 if (IS_ERR(chains)) { 1572 err = PTR_ERR(chains); 1573 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err); 1574 return err; 1575 } 1576 mlx5_chains_print_info(chains); 1577 1578 esw->fdb_table.offloads.esw_chains_priv = chains; 1579 1580 /* Create tc_end_ft which is the always created ft chain */ 1581 nf_ft = mlx5_chains_get_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1582 1, 0); 1583 if (IS_ERR(nf_ft)) { 1584 err = PTR_ERR(nf_ft); 1585 goto nf_ft_err; 1586 } 1587 1588 /* Always open the root for fast path */ 1589 ft = mlx5_chains_get_table(chains, 0, 1, 0); 1590 if (IS_ERR(ft)) { 1591 err = PTR_ERR(ft); 1592 goto level_0_err; 1593 } 1594 1595 /* Open level 1 for split fdb rules now if prios isn't supported */ 1596 if (!mlx5_chains_prios_supported(chains)) { 1597 err = esw_vport_tbl_get(esw); 1598 if (err) 1599 goto level_1_err; 1600 } 1601 1602 mlx5_chains_set_end_ft(chains, nf_ft); 1603 1604 return 0; 1605 1606level_1_err: 1607 mlx5_chains_put_table(chains, 0, 1, 0); 1608level_0_err: 1609 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1610nf_ft_err: 1611 mlx5_chains_destroy(chains); 1612 esw->fdb_table.offloads.esw_chains_priv = NULL; 1613 1614 return err; 1615} 1616 1617static void 1618esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1619{ 1620 if (!mlx5_chains_prios_supported(chains)) 1621 esw_vport_tbl_put(esw); 1622 mlx5_chains_put_table(chains, 0, 1, 0); 1623 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1624 mlx5_chains_destroy(chains); 1625} 1626 1627#else /* CONFIG_MLX5_CLS_ACT */ 1628 1629static int 1630esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1631{ return 0; } 1632 1633static void 1634esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1635{} 1636 1637#endif 1638 1639static int 1640esw_create_send_to_vport_group(struct mlx5_eswitch *esw, 1641 struct mlx5_flow_table *fdb, 1642 u32 *flow_group_in, 1643 int *ix) 1644{ 1645 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1646 struct mlx5_flow_group *g; 1647 void *match_criteria; 1648 int count, err = 0; 1649 1650 memset(flow_group_in, 0, inlen); 1651 1652 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, MLX5_MATCH_MISC_PARAMETERS); 1653 1654 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1655 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn); 1656 1657 if (!mlx5_eswitch_vport_match_metadata_enabled(esw) && 1658 MLX5_CAP_ESW(esw->dev, merged_eswitch)) { 1659 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1660 misc_parameters.source_eswitch_owner_vhca_id); 1661 MLX5_SET(create_flow_group_in, flow_group_in, 1662 source_eswitch_owner_vhca_id_valid, 1); 1663 } 1664 1665 /* See comment at table_size calculation */ 1666 count = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ); 1667 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1668 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, *ix + count - 1); 1669 *ix += count; 1670 1671 g = mlx5_create_flow_group(fdb, flow_group_in); 1672 if (IS_ERR(g)) { 1673 err = PTR_ERR(g); 1674 esw_warn(esw->dev, "Failed to create send-to-vport flow group err(%d)\n", err); 1675 goto out; 1676 } 1677 esw->fdb_table.offloads.send_to_vport_grp = g; 1678 1679out: 1680 return err; 1681} 1682 1683static int 1684esw_create_meta_send_to_vport_group(struct mlx5_eswitch *esw, 1685 struct mlx5_flow_table *fdb, 1686 u32 *flow_group_in, 1687 int *ix) 1688{ 1689 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1690 struct mlx5_flow_group *g; 1691 void *match_criteria; 1692 int err = 0; 1693 1694 if (!esw_src_port_rewrite_supported(esw)) 1695 return 0; 1696 1697 memset(flow_group_in, 0, inlen); 1698 1699 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1700 MLX5_MATCH_MISC_PARAMETERS_2); 1701 1702 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1703 1704 MLX5_SET(fte_match_param, match_criteria, 1705 misc_parameters_2.metadata_reg_c_0, 1706 mlx5_eswitch_get_vport_metadata_mask()); 1707 MLX5_SET(fte_match_param, match_criteria, 1708 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); 1709 1710 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1711 MLX5_SET(create_flow_group_in, flow_group_in, 1712 end_flow_index, *ix + esw->total_vports - 1); 1713 *ix += esw->total_vports; 1714 1715 g = mlx5_create_flow_group(fdb, flow_group_in); 1716 if (IS_ERR(g)) { 1717 err = PTR_ERR(g); 1718 esw_warn(esw->dev, 1719 "Failed to create send-to-vport meta flow group err(%d)\n", err); 1720 goto send_vport_meta_err; 1721 } 1722 esw->fdb_table.offloads.send_to_vport_meta_grp = g; 1723 1724 return 0; 1725 1726send_vport_meta_err: 1727 return err; 1728} 1729 1730static int 1731esw_create_peer_esw_miss_group(struct mlx5_eswitch *esw, 1732 struct mlx5_flow_table *fdb, 1733 u32 *flow_group_in, 1734 int *ix) 1735{ 1736 int max_peer_ports = (esw->total_vports - 1) * (MLX5_MAX_PORTS - 1); 1737 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1738 struct mlx5_flow_group *g; 1739 void *match_criteria; 1740 int err = 0; 1741 1742 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1743 return 0; 1744 1745 memset(flow_group_in, 0, inlen); 1746 1747 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, 0); 1748 1749 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1750 match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1751 flow_group_in, 1752 match_criteria); 1753 1754 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1755 misc_parameters.source_eswitch_owner_vhca_id); 1756 1757 MLX5_SET(create_flow_group_in, flow_group_in, 1758 source_eswitch_owner_vhca_id_valid, 1); 1759 } 1760 1761 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1762 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1763 *ix + max_peer_ports); 1764 *ix += max_peer_ports + 1; 1765 1766 g = mlx5_create_flow_group(fdb, flow_group_in); 1767 if (IS_ERR(g)) { 1768 err = PTR_ERR(g); 1769 esw_warn(esw->dev, "Failed to create peer miss flow group err(%d)\n", err); 1770 goto out; 1771 } 1772 esw->fdb_table.offloads.peer_miss_grp = g; 1773 1774out: 1775 return err; 1776} 1777 1778static int 1779esw_create_miss_group(struct mlx5_eswitch *esw, 1780 struct mlx5_flow_table *fdb, 1781 u32 *flow_group_in, 1782 int *ix) 1783{ 1784 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1785 struct mlx5_flow_group *g; 1786 void *match_criteria; 1787 int err = 0; 1788 u8 *dmac; 1789 1790 memset(flow_group_in, 0, inlen); 1791 1792 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1793 MLX5_MATCH_OUTER_HEADERS); 1794 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 1795 match_criteria); 1796 dmac = MLX5_ADDR_OF(fte_match_param, match_criteria, 1797 outer_headers.dmac_47_16); 1798 dmac[0] = 0x01; 1799 1800 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1801 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1802 *ix + MLX5_ESW_MISS_FLOWS); 1803 1804 g = mlx5_create_flow_group(fdb, flow_group_in); 1805 if (IS_ERR(g)) { 1806 err = PTR_ERR(g); 1807 esw_warn(esw->dev, "Failed to create miss flow group err(%d)\n", err); 1808 goto miss_err; 1809 } 1810 esw->fdb_table.offloads.miss_grp = g; 1811 1812 err = esw_add_fdb_miss_rule(esw); 1813 if (err) 1814 goto miss_rule_err; 1815 1816 return 0; 1817 1818miss_rule_err: 1819 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1820miss_err: 1821 return err; 1822} 1823 1824static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw) 1825{ 1826 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1827 struct mlx5_flow_table_attr ft_attr = {}; 1828 struct mlx5_core_dev *dev = esw->dev; 1829 struct mlx5_flow_namespace *root_ns; 1830 struct mlx5_flow_table *fdb = NULL; 1831 int table_size, ix = 0, err = 0; 1832 u32 flags = 0, *flow_group_in; 1833 1834 esw_debug(esw->dev, "Create offloads FDB Tables\n"); 1835 1836 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1837 if (!flow_group_in) 1838 return -ENOMEM; 1839 1840 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); 1841 if (!root_ns) { 1842 esw_warn(dev, "Failed to get FDB flow namespace\n"); 1843 err = -EOPNOTSUPP; 1844 goto ns_err; 1845 } 1846 esw->fdb_table.offloads.ns = root_ns; 1847 err = mlx5_flow_namespace_set_mode(root_ns, 1848 esw->dev->priv.steering->mode); 1849 if (err) { 1850 esw_warn(dev, "Failed to set FDB namespace steering mode\n"); 1851 goto ns_err; 1852 } 1853 1854 /* To be strictly correct: 1855 * MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) 1856 * should be: 1857 * esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ + 1858 * peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ 1859 * but as the peer device might not be in switchdev mode it's not 1860 * possible. We use the fact that by default FW sets max vfs and max sfs 1861 * to the same value on both devices. If it needs to be changed in the future note 1862 * the peer miss group should also be created based on the number of 1863 * total vports of the peer (currently is also uses esw->total_vports). 1864 */ 1865 table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) + 1866 esw->total_vports * MLX5_MAX_PORTS + MLX5_ESW_MISS_FLOWS; 1867 1868 /* create the slow path fdb with encap set, so further table instances 1869 * can be created at run time while VFs are probed if the FW allows that. 1870 */ 1871 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1872 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT | 1873 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); 1874 1875 ft_attr.flags = flags; 1876 ft_attr.max_fte = table_size; 1877 ft_attr.prio = FDB_SLOW_PATH; 1878 1879 fdb = mlx5_create_flow_table(root_ns, &ft_attr); 1880 if (IS_ERR(fdb)) { 1881 err = PTR_ERR(fdb); 1882 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err); 1883 goto slow_fdb_err; 1884 } 1885 esw->fdb_table.offloads.slow_fdb = fdb; 1886 1887 /* Create empty TC-miss managed table. This allows plugging in following 1888 * priorities without directly exposing their level 0 table to 1889 * eswitch_offloads and passing it as miss_fdb to following call to 1890 * esw_chains_create(). 1891 */ 1892 memset(&ft_attr, 0, sizeof(ft_attr)); 1893 ft_attr.prio = FDB_TC_MISS; 1894 esw->fdb_table.offloads.tc_miss_table = mlx5_create_flow_table(root_ns, &ft_attr); 1895 if (IS_ERR(esw->fdb_table.offloads.tc_miss_table)) { 1896 err = PTR_ERR(esw->fdb_table.offloads.tc_miss_table); 1897 esw_warn(dev, "Failed to create TC miss FDB Table err %d\n", err); 1898 goto tc_miss_table_err; 1899 } 1900 1901 err = esw_chains_create(esw, esw->fdb_table.offloads.tc_miss_table); 1902 if (err) { 1903 esw_warn(dev, "Failed to open fdb chains err(%d)\n", err); 1904 goto fdb_chains_err; 1905 } 1906 1907 err = esw_create_send_to_vport_group(esw, fdb, flow_group_in, &ix); 1908 if (err) 1909 goto send_vport_err; 1910 1911 err = esw_create_meta_send_to_vport_group(esw, fdb, flow_group_in, &ix); 1912 if (err) 1913 goto send_vport_meta_err; 1914 1915 err = esw_create_peer_esw_miss_group(esw, fdb, flow_group_in, &ix); 1916 if (err) 1917 goto peer_miss_err; 1918 1919 err = esw_create_miss_group(esw, fdb, flow_group_in, &ix); 1920 if (err) 1921 goto miss_err; 1922 1923 kvfree(flow_group_in); 1924 return 0; 1925 1926miss_err: 1927 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1928 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 1929peer_miss_err: 1930 if (esw->fdb_table.offloads.send_to_vport_meta_grp) 1931 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); 1932send_vport_meta_err: 1933 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 1934send_vport_err: 1935 esw_chains_destroy(esw, esw_chains(esw)); 1936fdb_chains_err: 1937 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); 1938tc_miss_table_err: 1939 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw)); 1940slow_fdb_err: 1941 /* Holds true only as long as DMFS is the default */ 1942 mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS); 1943ns_err: 1944 kvfree(flow_group_in); 1945 return err; 1946} 1947 1948static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) 1949{ 1950 if (!mlx5_eswitch_get_slow_fdb(esw)) 1951 return; 1952 1953 esw_debug(esw->dev, "Destroy offloads FDB Tables\n"); 1954 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi); 1955 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1956 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 1957 if (esw->fdb_table.offloads.send_to_vport_meta_grp) 1958 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); 1959 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1960 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 1961 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1962 1963 esw_chains_destroy(esw, esw_chains(esw)); 1964 1965 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); 1966 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw)); 1967 /* Holds true only as long as DMFS is the default */ 1968 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns, 1969 MLX5_FLOW_STEERING_MODE_DMFS); 1970 atomic64_set(&esw->user_count, 0); 1971} 1972 1973static int esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch *esw) 1974{ 1975 int nvports; 1976 1977 nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS; 1978 if (mlx5e_tc_int_port_supported(esw)) 1979 nvports += MLX5E_TC_MAX_INT_PORT_NUM; 1980 1981 return nvports; 1982} 1983 1984static int esw_create_offloads_table(struct mlx5_eswitch *esw) 1985{ 1986 struct mlx5_flow_table_attr ft_attr = {}; 1987 struct mlx5_core_dev *dev = esw->dev; 1988 struct mlx5_flow_table *ft_offloads; 1989 struct mlx5_flow_namespace *ns; 1990 int err = 0; 1991 1992 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 1993 if (!ns) { 1994 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 1995 return -EOPNOTSUPP; 1996 } 1997 1998 ft_attr.max_fte = esw_get_nr_ft_offloads_steering_src_ports(esw) + 1999 MLX5_ESW_FT_OFFLOADS_DROP_RULE; 2000 ft_attr.prio = 1; 2001 2002 ft_offloads = mlx5_create_flow_table(ns, &ft_attr); 2003 if (IS_ERR(ft_offloads)) { 2004 err = PTR_ERR(ft_offloads); 2005 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err); 2006 return err; 2007 } 2008 2009 esw->offloads.ft_offloads = ft_offloads; 2010 return 0; 2011} 2012 2013static void esw_destroy_offloads_table(struct mlx5_eswitch *esw) 2014{ 2015 struct mlx5_esw_offload *offloads = &esw->offloads; 2016 2017 mlx5_destroy_flow_table(offloads->ft_offloads); 2018} 2019 2020static int esw_create_vport_rx_group(struct mlx5_eswitch *esw) 2021{ 2022 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2023 struct mlx5_flow_group *g; 2024 u32 *flow_group_in; 2025 int nvports; 2026 int err = 0; 2027 2028 nvports = esw_get_nr_ft_offloads_steering_src_ports(esw); 2029 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2030 if (!flow_group_in) 2031 return -ENOMEM; 2032 2033 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, 0); 2034 2035 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2036 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1); 2037 2038 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); 2039 2040 if (IS_ERR(g)) { 2041 err = PTR_ERR(g); 2042 mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err); 2043 goto out; 2044 } 2045 2046 esw->offloads.vport_rx_group = g; 2047out: 2048 kvfree(flow_group_in); 2049 return err; 2050} 2051 2052static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw) 2053{ 2054 mlx5_destroy_flow_group(esw->offloads.vport_rx_group); 2055} 2056 2057static int esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch *esw) 2058{ 2059 /* ft_offloads table is enlarged by MLX5_ESW_FT_OFFLOADS_DROP_RULE (1) 2060 * for the drop rule, which is placed at the end of the table. 2061 * So return the total of vport and int_port as rule index. 2062 */ 2063 return esw_get_nr_ft_offloads_steering_src_ports(esw); 2064} 2065 2066static int esw_create_vport_rx_drop_group(struct mlx5_eswitch *esw) 2067{ 2068 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2069 struct mlx5_flow_group *g; 2070 u32 *flow_group_in; 2071 int flow_index; 2072 int err = 0; 2073 2074 flow_index = esw_create_vport_rx_drop_rule_index(esw); 2075 2076 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2077 if (!flow_group_in) 2078 return -ENOMEM; 2079 2080 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_index); 2081 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_index); 2082 2083 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); 2084 2085 if (IS_ERR(g)) { 2086 err = PTR_ERR(g); 2087 mlx5_core_warn(esw->dev, "Failed to create vport rx drop group err %d\n", err); 2088 goto out; 2089 } 2090 2091 esw->offloads.vport_rx_drop_group = g; 2092out: 2093 kvfree(flow_group_in); 2094 return err; 2095} 2096 2097static void esw_destroy_vport_rx_drop_group(struct mlx5_eswitch *esw) 2098{ 2099 if (esw->offloads.vport_rx_drop_group) 2100 mlx5_destroy_flow_group(esw->offloads.vport_rx_drop_group); 2101} 2102 2103void 2104mlx5_esw_set_spec_source_port(struct mlx5_eswitch *esw, 2105 u16 vport, 2106 struct mlx5_flow_spec *spec) 2107{ 2108 void *misc; 2109 2110 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 2111 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 2112 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2113 mlx5_eswitch_get_vport_metadata_for_match(esw, vport)); 2114 2115 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 2116 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2117 mlx5_eswitch_get_vport_metadata_mask()); 2118 2119 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 2120 } else { 2121 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 2122 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 2123 2124 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 2125 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 2126 2127 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 2128 } 2129} 2130 2131struct mlx5_flow_handle * 2132mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport, 2133 struct mlx5_flow_destination *dest) 2134{ 2135 struct mlx5_flow_act flow_act = {0}; 2136 struct mlx5_flow_handle *flow_rule; 2137 struct mlx5_flow_spec *spec; 2138 2139 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2140 if (!spec) { 2141 flow_rule = ERR_PTR(-ENOMEM); 2142 goto out; 2143 } 2144 2145 mlx5_esw_set_spec_source_port(esw, vport, spec); 2146 2147 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2148 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec, 2149 &flow_act, dest, 1); 2150 if (IS_ERR(flow_rule)) { 2151 esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule)); 2152 goto out; 2153 } 2154 2155out: 2156 kvfree(spec); 2157 return flow_rule; 2158} 2159 2160static int esw_create_vport_rx_drop_rule(struct mlx5_eswitch *esw) 2161{ 2162 struct mlx5_flow_act flow_act = {}; 2163 struct mlx5_flow_handle *flow_rule; 2164 2165 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; 2166 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, NULL, 2167 &flow_act, NULL, 0); 2168 if (IS_ERR(flow_rule)) { 2169 esw_warn(esw->dev, 2170 "fs offloads: Failed to add vport rx drop rule err %ld\n", 2171 PTR_ERR(flow_rule)); 2172 return PTR_ERR(flow_rule); 2173 } 2174 2175 esw->offloads.vport_rx_drop_rule = flow_rule; 2176 2177 return 0; 2178} 2179 2180static void esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch *esw) 2181{ 2182 if (esw->offloads.vport_rx_drop_rule) 2183 mlx5_del_flow_rules(esw->offloads.vport_rx_drop_rule); 2184} 2185 2186static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode) 2187{ 2188 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2; 2189 struct mlx5_core_dev *dev = esw->dev; 2190 struct mlx5_vport *vport; 2191 unsigned long i; 2192 2193 if (!MLX5_CAP_GEN(dev, vport_group_manager)) 2194 return -EOPNOTSUPP; 2195 2196 if (!mlx5_esw_is_fdb_created(esw)) 2197 return -EOPNOTSUPP; 2198 2199 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 2200 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 2201 mlx5_mode = MLX5_INLINE_MODE_NONE; 2202 goto out; 2203 case MLX5_CAP_INLINE_MODE_L2: 2204 mlx5_mode = MLX5_INLINE_MODE_L2; 2205 goto out; 2206 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 2207 goto query_vports; 2208 } 2209 2210query_vports: 2211 mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode); 2212 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 2213 mlx5_query_nic_vport_min_inline(dev, vport->vport, &mlx5_mode); 2214 if (prev_mlx5_mode != mlx5_mode) 2215 return -EINVAL; 2216 prev_mlx5_mode = mlx5_mode; 2217 } 2218 2219out: 2220 *mode = mlx5_mode; 2221 return 0; 2222} 2223 2224static void esw_destroy_restore_table(struct mlx5_eswitch *esw) 2225{ 2226 struct mlx5_esw_offload *offloads = &esw->offloads; 2227 2228 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 2229 return; 2230 2231 mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id); 2232 mlx5_destroy_flow_group(offloads->restore_group); 2233 mlx5_destroy_flow_table(offloads->ft_offloads_restore); 2234} 2235 2236static int esw_create_restore_table(struct mlx5_eswitch *esw) 2237{ 2238 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 2239 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2240 struct mlx5_flow_table_attr ft_attr = {}; 2241 struct mlx5_core_dev *dev = esw->dev; 2242 struct mlx5_flow_namespace *ns; 2243 struct mlx5_modify_hdr *mod_hdr; 2244 void *match_criteria, *misc; 2245 struct mlx5_flow_table *ft; 2246 struct mlx5_flow_group *g; 2247 u32 *flow_group_in; 2248 int err = 0; 2249 2250 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 2251 return 0; 2252 2253 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 2254 if (!ns) { 2255 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 2256 return -EOPNOTSUPP; 2257 } 2258 2259 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2260 if (!flow_group_in) { 2261 err = -ENOMEM; 2262 goto out_free; 2263 } 2264 2265 ft_attr.max_fte = 1 << ESW_REG_C0_USER_DATA_METADATA_BITS; 2266 ft = mlx5_create_flow_table(ns, &ft_attr); 2267 if (IS_ERR(ft)) { 2268 err = PTR_ERR(ft); 2269 esw_warn(esw->dev, "Failed to create restore table, err %d\n", 2270 err); 2271 goto out_free; 2272 } 2273 2274 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 2275 match_criteria); 2276 misc = MLX5_ADDR_OF(fte_match_param, match_criteria, 2277 misc_parameters_2); 2278 2279 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2280 ESW_REG_C0_USER_DATA_METADATA_MASK); 2281 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2282 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 2283 ft_attr.max_fte - 1); 2284 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 2285 MLX5_MATCH_MISC_PARAMETERS_2); 2286 g = mlx5_create_flow_group(ft, flow_group_in); 2287 if (IS_ERR(g)) { 2288 err = PTR_ERR(g); 2289 esw_warn(dev, "Failed to create restore flow group, err: %d\n", 2290 err); 2291 goto err_group; 2292 } 2293 2294 MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY); 2295 MLX5_SET(copy_action_in, modact, src_field, 2296 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1); 2297 MLX5_SET(copy_action_in, modact, dst_field, 2298 MLX5_ACTION_IN_FIELD_METADATA_REG_B); 2299 mod_hdr = mlx5_modify_header_alloc(esw->dev, 2300 MLX5_FLOW_NAMESPACE_KERNEL, 1, 2301 modact); 2302 if (IS_ERR(mod_hdr)) { 2303 err = PTR_ERR(mod_hdr); 2304 esw_warn(dev, "Failed to create restore mod header, err: %d\n", 2305 err); 2306 goto err_mod_hdr; 2307 } 2308 2309 esw->offloads.ft_offloads_restore = ft; 2310 esw->offloads.restore_group = g; 2311 esw->offloads.restore_copy_hdr_id = mod_hdr; 2312 2313 kvfree(flow_group_in); 2314 2315 return 0; 2316 2317err_mod_hdr: 2318 mlx5_destroy_flow_group(g); 2319err_group: 2320 mlx5_destroy_flow_table(ft); 2321out_free: 2322 kvfree(flow_group_in); 2323 2324 return err; 2325} 2326 2327static int esw_offloads_start(struct mlx5_eswitch *esw, 2328 struct netlink_ext_ack *extack) 2329{ 2330 int err; 2331 2332 esw->mode = MLX5_ESWITCH_OFFLOADS; 2333 err = mlx5_eswitch_enable_locked(esw, esw->dev->priv.sriov.num_vfs); 2334 if (err) { 2335 NL_SET_ERR_MSG_MOD(extack, 2336 "Failed setting eswitch to offloads"); 2337 esw->mode = MLX5_ESWITCH_LEGACY; 2338 mlx5_rescan_drivers(esw->dev); 2339 return err; 2340 } 2341 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) { 2342 if (mlx5_eswitch_inline_mode_get(esw, 2343 &esw->offloads.inline_mode)) { 2344 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2; 2345 NL_SET_ERR_MSG_MOD(extack, 2346 "Inline mode is different between vports"); 2347 } 2348 } 2349 return 0; 2350} 2351 2352static int mlx5_esw_offloads_rep_init(struct mlx5_eswitch *esw, const struct mlx5_vport *vport) 2353{ 2354 struct mlx5_eswitch_rep *rep; 2355 int rep_type; 2356 int err; 2357 2358 rep = kzalloc(sizeof(*rep), GFP_KERNEL); 2359 if (!rep) 2360 return -ENOMEM; 2361 2362 rep->vport = vport->vport; 2363 rep->vport_index = vport->index; 2364 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) 2365 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED); 2366 2367 err = xa_insert(&esw->offloads.vport_reps, rep->vport, rep, GFP_KERNEL); 2368 if (err) 2369 goto insert_err; 2370 2371 return 0; 2372 2373insert_err: 2374 kfree(rep); 2375 return err; 2376} 2377 2378static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch *esw, 2379 struct mlx5_eswitch_rep *rep) 2380{ 2381 xa_erase(&esw->offloads.vport_reps, rep->vport); 2382 kfree(rep); 2383} 2384 2385static void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw) 2386{ 2387 struct mlx5_eswitch_rep *rep; 2388 unsigned long i; 2389 2390 mlx5_esw_for_each_rep(esw, i, rep) 2391 mlx5_esw_offloads_rep_cleanup(esw, rep); 2392 xa_destroy(&esw->offloads.vport_reps); 2393} 2394 2395static int esw_offloads_init_reps(struct mlx5_eswitch *esw) 2396{ 2397 struct mlx5_vport *vport; 2398 unsigned long i; 2399 int err; 2400 2401 xa_init(&esw->offloads.vport_reps); 2402 2403 mlx5_esw_for_each_vport(esw, i, vport) { 2404 err = mlx5_esw_offloads_rep_init(esw, vport); 2405 if (err) 2406 goto err; 2407 } 2408 return 0; 2409 2410err: 2411 esw_offloads_cleanup_reps(esw); 2412 return err; 2413} 2414 2415static int esw_port_metadata_set(struct devlink *devlink, u32 id, 2416 struct devlink_param_gset_ctx *ctx, 2417 struct netlink_ext_ack *extack) 2418{ 2419 struct mlx5_core_dev *dev = devlink_priv(devlink); 2420 struct mlx5_eswitch *esw = dev->priv.eswitch; 2421 int err = 0; 2422 2423 down_write(&esw->mode_lock); 2424 if (mlx5_esw_is_fdb_created(esw)) { 2425 err = -EBUSY; 2426 goto done; 2427 } 2428 if (!mlx5_esw_vport_match_metadata_supported(esw)) { 2429 err = -EOPNOTSUPP; 2430 goto done; 2431 } 2432 if (ctx->val.vbool) 2433 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA; 2434 else 2435 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA; 2436done: 2437 up_write(&esw->mode_lock); 2438 return err; 2439} 2440 2441static int esw_port_metadata_get(struct devlink *devlink, u32 id, 2442 struct devlink_param_gset_ctx *ctx) 2443{ 2444 struct mlx5_core_dev *dev = devlink_priv(devlink); 2445 2446 ctx->val.vbool = mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch); 2447 return 0; 2448} 2449 2450static int esw_port_metadata_validate(struct devlink *devlink, u32 id, 2451 union devlink_param_value val, 2452 struct netlink_ext_ack *extack) 2453{ 2454 struct mlx5_core_dev *dev = devlink_priv(devlink); 2455 u8 esw_mode; 2456 2457 esw_mode = mlx5_eswitch_mode(dev); 2458 if (esw_mode == MLX5_ESWITCH_OFFLOADS) { 2459 NL_SET_ERR_MSG_MOD(extack, 2460 "E-Switch must either disabled or non switchdev mode"); 2461 return -EBUSY; 2462 } 2463 return 0; 2464} 2465 2466static const struct devlink_param esw_devlink_params[] = { 2467 DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_ESW_PORT_METADATA, 2468 "esw_port_metadata", DEVLINK_PARAM_TYPE_BOOL, 2469 BIT(DEVLINK_PARAM_CMODE_RUNTIME), 2470 esw_port_metadata_get, 2471 esw_port_metadata_set, 2472 esw_port_metadata_validate), 2473}; 2474 2475int esw_offloads_init(struct mlx5_eswitch *esw) 2476{ 2477 int err; 2478 2479 err = esw_offloads_init_reps(esw); 2480 if (err) 2481 return err; 2482 2483 if (MLX5_ESWITCH_MANAGER(esw->dev) && 2484 mlx5_esw_vport_match_metadata_supported(esw)) 2485 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA; 2486 2487 err = devl_params_register(priv_to_devlink(esw->dev), 2488 esw_devlink_params, 2489 ARRAY_SIZE(esw_devlink_params)); 2490 if (err) 2491 goto err_params; 2492 2493 return 0; 2494 2495err_params: 2496 esw_offloads_cleanup_reps(esw); 2497 return err; 2498} 2499 2500void esw_offloads_cleanup(struct mlx5_eswitch *esw) 2501{ 2502 devl_params_unregister(priv_to_devlink(esw->dev), 2503 esw_devlink_params, 2504 ARRAY_SIZE(esw_devlink_params)); 2505 esw_offloads_cleanup_reps(esw); 2506} 2507 2508static int __esw_offloads_load_rep(struct mlx5_eswitch *esw, 2509 struct mlx5_eswitch_rep *rep, u8 rep_type) 2510{ 2511 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 2512 REP_REGISTERED, REP_LOADED) == REP_REGISTERED) 2513 return esw->offloads.rep_ops[rep_type]->load(esw->dev, rep); 2514 2515 return 0; 2516} 2517 2518static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw, 2519 struct mlx5_eswitch_rep *rep, u8 rep_type) 2520{ 2521 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 2522 REP_LOADED, REP_REGISTERED) == REP_LOADED) 2523 esw->offloads.rep_ops[rep_type]->unload(rep); 2524} 2525 2526static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type) 2527{ 2528 struct mlx5_eswitch_rep *rep; 2529 unsigned long i; 2530 2531 mlx5_esw_for_each_rep(esw, i, rep) 2532 __esw_offloads_unload_rep(esw, rep, rep_type); 2533} 2534 2535static int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num) 2536{ 2537 struct mlx5_eswitch_rep *rep; 2538 int rep_type; 2539 int err; 2540 2541 rep = mlx5_eswitch_get_rep(esw, vport_num); 2542 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) { 2543 err = __esw_offloads_load_rep(esw, rep, rep_type); 2544 if (err) 2545 goto err_reps; 2546 } 2547 2548 return 0; 2549 2550err_reps: 2551 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED); 2552 for (--rep_type; rep_type >= 0; rep_type--) 2553 __esw_offloads_unload_rep(esw, rep, rep_type); 2554 return err; 2555} 2556 2557static void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num) 2558{ 2559 struct mlx5_eswitch_rep *rep; 2560 int rep_type; 2561 2562 rep = mlx5_eswitch_get_rep(esw, vport_num); 2563 for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--) 2564 __esw_offloads_unload_rep(esw, rep, rep_type); 2565} 2566 2567int mlx5_esw_offloads_init_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2568{ 2569 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2570 return 0; 2571 2572 return mlx5_esw_offloads_pf_vf_devlink_port_init(esw, vport); 2573} 2574 2575void mlx5_esw_offloads_cleanup_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2576{ 2577 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2578 return; 2579 2580 mlx5_esw_offloads_pf_vf_devlink_port_cleanup(esw, vport); 2581} 2582 2583int mlx5_esw_offloads_init_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport, 2584 struct mlx5_devlink_port *dl_port, 2585 u32 controller, u32 sfnum) 2586{ 2587 return mlx5_esw_offloads_sf_devlink_port_init(esw, vport, dl_port, controller, sfnum); 2588} 2589 2590void mlx5_esw_offloads_cleanup_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2591{ 2592 mlx5_esw_offloads_sf_devlink_port_cleanup(esw, vport); 2593} 2594 2595int mlx5_esw_offloads_load_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2596{ 2597 int err; 2598 2599 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2600 return 0; 2601 2602 err = mlx5_esw_offloads_devlink_port_register(esw, vport); 2603 if (err) 2604 return err; 2605 2606 err = mlx5_esw_offloads_rep_load(esw, vport->vport); 2607 if (err) 2608 goto load_err; 2609 return err; 2610 2611load_err: 2612 mlx5_esw_offloads_devlink_port_unregister(esw, vport); 2613 return err; 2614} 2615 2616void mlx5_esw_offloads_unload_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2617{ 2618 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2619 return; 2620 2621 mlx5_esw_offloads_rep_unload(esw, vport->vport); 2622 2623 mlx5_esw_offloads_devlink_port_unregister(esw, vport); 2624} 2625 2626static int esw_set_slave_root_fdb(struct mlx5_core_dev *master, 2627 struct mlx5_core_dev *slave) 2628{ 2629 u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {}; 2630 u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {}; 2631 struct mlx5_flow_root_namespace *root; 2632 struct mlx5_flow_namespace *ns; 2633 int err; 2634 2635 MLX5_SET(set_flow_table_root_in, in, opcode, 2636 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT); 2637 MLX5_SET(set_flow_table_root_in, in, table_type, 2638 FS_FT_FDB); 2639 2640 if (master) { 2641 ns = mlx5_get_flow_namespace(master, 2642 MLX5_FLOW_NAMESPACE_FDB); 2643 root = find_root(&ns->node); 2644 mutex_lock(&root->chain_lock); 2645 MLX5_SET(set_flow_table_root_in, in, 2646 table_eswitch_owner_vhca_id_valid, 1); 2647 MLX5_SET(set_flow_table_root_in, in, 2648 table_eswitch_owner_vhca_id, 2649 MLX5_CAP_GEN(master, vhca_id)); 2650 MLX5_SET(set_flow_table_root_in, in, table_id, 2651 root->root_ft->id); 2652 } else { 2653 ns = mlx5_get_flow_namespace(slave, 2654 MLX5_FLOW_NAMESPACE_FDB); 2655 root = find_root(&ns->node); 2656 mutex_lock(&root->chain_lock); 2657 MLX5_SET(set_flow_table_root_in, in, table_id, 2658 root->root_ft->id); 2659 } 2660 2661 err = mlx5_cmd_exec(slave, in, sizeof(in), out, sizeof(out)); 2662 mutex_unlock(&root->chain_lock); 2663 2664 return err; 2665} 2666 2667static int __esw_set_master_egress_rule(struct mlx5_core_dev *master, 2668 struct mlx5_core_dev *slave, 2669 struct mlx5_vport *vport, 2670 struct mlx5_flow_table *acl) 2671{ 2672 u16 slave_index = MLX5_CAP_GEN(slave, vhca_id); 2673 struct mlx5_flow_handle *flow_rule = NULL; 2674 struct mlx5_flow_destination dest = {}; 2675 struct mlx5_flow_act flow_act = {}; 2676 struct mlx5_flow_spec *spec; 2677 int err = 0; 2678 void *misc; 2679 2680 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2681 if (!spec) 2682 return -ENOMEM; 2683 2684 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 2685 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 2686 misc_parameters); 2687 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_UPLINK); 2688 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, slave_index); 2689 2690 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 2691 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 2692 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 2693 source_eswitch_owner_vhca_id); 2694 2695 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2696 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 2697 dest.vport.num = slave->priv.eswitch->manager_vport; 2698 dest.vport.vhca_id = MLX5_CAP_GEN(slave, vhca_id); 2699 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 2700 2701 flow_rule = mlx5_add_flow_rules(acl, spec, &flow_act, 2702 &dest, 1); 2703 if (IS_ERR(flow_rule)) { 2704 err = PTR_ERR(flow_rule); 2705 } else { 2706 err = xa_insert(&vport->egress.offloads.bounce_rules, 2707 slave_index, flow_rule, GFP_KERNEL); 2708 if (err) 2709 mlx5_del_flow_rules(flow_rule); 2710 } 2711 2712 kvfree(spec); 2713 return err; 2714} 2715 2716static int esw_master_egress_create_resources(struct mlx5_eswitch *esw, 2717 struct mlx5_flow_namespace *egress_ns, 2718 struct mlx5_vport *vport, size_t count) 2719{ 2720 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2721 struct mlx5_flow_table_attr ft_attr = { 2722 .max_fte = count, .prio = 0, .level = 0, 2723 }; 2724 struct mlx5_flow_table *acl; 2725 struct mlx5_flow_group *g; 2726 void *match_criteria; 2727 u32 *flow_group_in; 2728 int err; 2729 2730 if (vport->egress.acl) 2731 return 0; 2732 2733 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2734 if (!flow_group_in) 2735 return -ENOMEM; 2736 2737 if (vport->vport || mlx5_core_is_ecpf(esw->dev)) 2738 ft_attr.flags = MLX5_FLOW_TABLE_OTHER_VPORT; 2739 2740 acl = mlx5_create_vport_flow_table(egress_ns, &ft_attr, vport->vport); 2741 if (IS_ERR(acl)) { 2742 err = PTR_ERR(acl); 2743 goto out; 2744 } 2745 2746 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 2747 match_criteria); 2748 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 2749 misc_parameters.source_port); 2750 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 2751 misc_parameters.source_eswitch_owner_vhca_id); 2752 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 2753 MLX5_MATCH_MISC_PARAMETERS); 2754 2755 MLX5_SET(create_flow_group_in, flow_group_in, 2756 source_eswitch_owner_vhca_id_valid, 1); 2757 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2758 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, count); 2759 2760 g = mlx5_create_flow_group(acl, flow_group_in); 2761 if (IS_ERR(g)) { 2762 err = PTR_ERR(g); 2763 goto err_group; 2764 } 2765 2766 vport->egress.acl = acl; 2767 vport->egress.offloads.bounce_grp = g; 2768 vport->egress.type = VPORT_EGRESS_ACL_TYPE_SHARED_FDB; 2769 xa_init_flags(&vport->egress.offloads.bounce_rules, XA_FLAGS_ALLOC); 2770 2771 kvfree(flow_group_in); 2772 2773 return 0; 2774 2775err_group: 2776 mlx5_destroy_flow_table(acl); 2777out: 2778 kvfree(flow_group_in); 2779 return err; 2780} 2781 2782static void esw_master_egress_destroy_resources(struct mlx5_vport *vport) 2783{ 2784 if (!xa_empty(&vport->egress.offloads.bounce_rules)) 2785 return; 2786 mlx5_destroy_flow_group(vport->egress.offloads.bounce_grp); 2787 vport->egress.offloads.bounce_grp = NULL; 2788 mlx5_destroy_flow_table(vport->egress.acl); 2789 vport->egress.acl = NULL; 2790} 2791 2792static int esw_set_master_egress_rule(struct mlx5_core_dev *master, 2793 struct mlx5_core_dev *slave, size_t count) 2794{ 2795 struct mlx5_eswitch *esw = master->priv.eswitch; 2796 u16 slave_index = MLX5_CAP_GEN(slave, vhca_id); 2797 struct mlx5_flow_namespace *egress_ns; 2798 struct mlx5_vport *vport; 2799 int err; 2800 2801 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport); 2802 if (IS_ERR(vport)) 2803 return PTR_ERR(vport); 2804 2805 egress_ns = mlx5_get_flow_vport_acl_namespace(master, 2806 MLX5_FLOW_NAMESPACE_ESW_EGRESS, 2807 vport->index); 2808 if (!egress_ns) 2809 return -EINVAL; 2810 2811 if (vport->egress.acl && vport->egress.type != VPORT_EGRESS_ACL_TYPE_SHARED_FDB) 2812 return 0; 2813 2814 err = esw_master_egress_create_resources(esw, egress_ns, vport, count); 2815 if (err) 2816 return err; 2817 2818 if (xa_load(&vport->egress.offloads.bounce_rules, slave_index)) 2819 return -EINVAL; 2820 2821 err = __esw_set_master_egress_rule(master, slave, vport, vport->egress.acl); 2822 if (err) 2823 goto err_rule; 2824 2825 return 0; 2826 2827err_rule: 2828 esw_master_egress_destroy_resources(vport); 2829 return err; 2830} 2831 2832static void esw_unset_master_egress_rule(struct mlx5_core_dev *dev, 2833 struct mlx5_core_dev *slave_dev) 2834{ 2835 struct mlx5_vport *vport; 2836 2837 vport = mlx5_eswitch_get_vport(dev->priv.eswitch, 2838 dev->priv.eswitch->manager_vport); 2839 2840 esw_acl_egress_ofld_bounce_rule_destroy(vport, MLX5_CAP_GEN(slave_dev, vhca_id)); 2841 2842 if (xa_empty(&vport->egress.offloads.bounce_rules)) { 2843 esw_acl_egress_ofld_cleanup(vport); 2844 xa_destroy(&vport->egress.offloads.bounce_rules); 2845 } 2846} 2847 2848int mlx5_eswitch_offloads_single_fdb_add_one(struct mlx5_eswitch *master_esw, 2849 struct mlx5_eswitch *slave_esw, int max_slaves) 2850{ 2851 int err; 2852 2853 err = esw_set_slave_root_fdb(master_esw->dev, 2854 slave_esw->dev); 2855 if (err) 2856 return err; 2857 2858 err = esw_set_master_egress_rule(master_esw->dev, 2859 slave_esw->dev, max_slaves); 2860 if (err) 2861 goto err_acl; 2862 2863 return err; 2864 2865err_acl: 2866 esw_set_slave_root_fdb(NULL, slave_esw->dev); 2867 return err; 2868} 2869 2870void mlx5_eswitch_offloads_single_fdb_del_one(struct mlx5_eswitch *master_esw, 2871 struct mlx5_eswitch *slave_esw) 2872{ 2873 esw_set_slave_root_fdb(NULL, slave_esw->dev); 2874 esw_unset_master_egress_rule(master_esw->dev, slave_esw->dev); 2875} 2876 2877#define ESW_OFFLOADS_DEVCOM_PAIR (0) 2878#define ESW_OFFLOADS_DEVCOM_UNPAIR (1) 2879 2880static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch *esw, 2881 struct mlx5_eswitch *peer_esw) 2882{ 2883 const struct mlx5_eswitch_rep_ops *ops; 2884 struct mlx5_eswitch_rep *rep; 2885 unsigned long i; 2886 u8 rep_type; 2887 2888 mlx5_esw_for_each_rep(esw, i, rep) { 2889 rep_type = NUM_REP_TYPES; 2890 while (rep_type--) { 2891 ops = esw->offloads.rep_ops[rep_type]; 2892 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 2893 ops->event) 2894 ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_UNPAIR, peer_esw); 2895 } 2896 } 2897} 2898 2899static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw, 2900 struct mlx5_eswitch *peer_esw) 2901{ 2902#if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 2903 mlx5e_tc_clean_fdb_peer_flows(esw); 2904#endif 2905 mlx5_esw_offloads_rep_event_unpair(esw, peer_esw); 2906 esw_del_fdb_peer_miss_rules(esw, peer_esw->dev); 2907} 2908 2909static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw, 2910 struct mlx5_eswitch *peer_esw) 2911{ 2912 const struct mlx5_eswitch_rep_ops *ops; 2913 struct mlx5_eswitch_rep *rep; 2914 unsigned long i; 2915 u8 rep_type; 2916 int err; 2917 2918 err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev); 2919 if (err) 2920 return err; 2921 2922 mlx5_esw_for_each_rep(esw, i, rep) { 2923 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) { 2924 ops = esw->offloads.rep_ops[rep_type]; 2925 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 2926 ops->event) { 2927 err = ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_PAIR, peer_esw); 2928 if (err) 2929 goto err_out; 2930 } 2931 } 2932 } 2933 2934 return 0; 2935 2936err_out: 2937 mlx5_esw_offloads_unpair(esw, peer_esw); 2938 return err; 2939} 2940 2941static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw, 2942 struct mlx5_eswitch *peer_esw, 2943 bool pair) 2944{ 2945 u16 peer_vhca_id = MLX5_CAP_GEN(peer_esw->dev, vhca_id); 2946 u16 vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id); 2947 struct mlx5_flow_root_namespace *peer_ns; 2948 struct mlx5_flow_root_namespace *ns; 2949 int err; 2950 2951 peer_ns = peer_esw->dev->priv.steering->fdb_root_ns; 2952 ns = esw->dev->priv.steering->fdb_root_ns; 2953 2954 if (pair) { 2955 err = mlx5_flow_namespace_set_peer(ns, peer_ns, peer_vhca_id); 2956 if (err) 2957 return err; 2958 2959 err = mlx5_flow_namespace_set_peer(peer_ns, ns, vhca_id); 2960 if (err) { 2961 mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id); 2962 return err; 2963 } 2964 } else { 2965 mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id); 2966 mlx5_flow_namespace_set_peer(peer_ns, NULL, vhca_id); 2967 } 2968 2969 return 0; 2970} 2971 2972static int mlx5_esw_offloads_devcom_event(int event, 2973 void *my_data, 2974 void *event_data) 2975{ 2976 struct mlx5_eswitch *esw = my_data; 2977 struct mlx5_eswitch *peer_esw = event_data; 2978 u16 esw_i, peer_esw_i; 2979 bool esw_paired; 2980 int err; 2981 2982 peer_esw_i = MLX5_CAP_GEN(peer_esw->dev, vhca_id); 2983 esw_i = MLX5_CAP_GEN(esw->dev, vhca_id); 2984 esw_paired = !!xa_load(&esw->paired, peer_esw_i); 2985 2986 switch (event) { 2987 case ESW_OFFLOADS_DEVCOM_PAIR: 2988 if (mlx5_eswitch_vport_match_metadata_enabled(esw) != 2989 mlx5_eswitch_vport_match_metadata_enabled(peer_esw)) 2990 break; 2991 2992 if (esw_paired) 2993 break; 2994 2995 err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true); 2996 if (err) 2997 goto err_out; 2998 2999 err = mlx5_esw_offloads_pair(esw, peer_esw); 3000 if (err) 3001 goto err_peer; 3002 3003 err = mlx5_esw_offloads_pair(peer_esw, esw); 3004 if (err) 3005 goto err_pair; 3006 3007 err = xa_insert(&esw->paired, peer_esw_i, peer_esw, GFP_KERNEL); 3008 if (err) 3009 goto err_xa; 3010 3011 err = xa_insert(&peer_esw->paired, esw_i, esw, GFP_KERNEL); 3012 if (err) 3013 goto err_peer_xa; 3014 3015 esw->num_peers++; 3016 peer_esw->num_peers++; 3017 mlx5_devcom_comp_set_ready(esw->devcom, true); 3018 break; 3019 3020 case ESW_OFFLOADS_DEVCOM_UNPAIR: 3021 if (!esw_paired) 3022 break; 3023 3024 peer_esw->num_peers--; 3025 esw->num_peers--; 3026 if (!esw->num_peers && !peer_esw->num_peers) 3027 mlx5_devcom_comp_set_ready(esw->devcom, false); 3028 xa_erase(&peer_esw->paired, esw_i); 3029 xa_erase(&esw->paired, peer_esw_i); 3030 mlx5_esw_offloads_unpair(peer_esw, esw); 3031 mlx5_esw_offloads_unpair(esw, peer_esw); 3032 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 3033 break; 3034 } 3035 3036 return 0; 3037 3038err_peer_xa: 3039 xa_erase(&esw->paired, peer_esw_i); 3040err_xa: 3041 mlx5_esw_offloads_unpair(peer_esw, esw); 3042err_pair: 3043 mlx5_esw_offloads_unpair(esw, peer_esw); 3044err_peer: 3045 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 3046err_out: 3047 mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d", 3048 event, err); 3049 return err; 3050} 3051 3052void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, u64 key) 3053{ 3054 int i; 3055 3056 for (i = 0; i < MLX5_MAX_PORTS; i++) 3057 INIT_LIST_HEAD(&esw->offloads.peer_flows[i]); 3058 mutex_init(&esw->offloads.peer_mutex); 3059 3060 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 3061 return; 3062 3063 if ((MLX5_VPORT_MANAGER(esw->dev) || mlx5_core_is_ecpf_esw_manager(esw->dev)) && 3064 !mlx5_lag_is_supported(esw->dev)) 3065 return; 3066 3067 xa_init(&esw->paired); 3068 esw->num_peers = 0; 3069 esw->devcom = mlx5_devcom_register_component(esw->dev->priv.devc, 3070 MLX5_DEVCOM_ESW_OFFLOADS, 3071 key, 3072 mlx5_esw_offloads_devcom_event, 3073 esw); 3074 if (IS_ERR(esw->devcom)) 3075 return; 3076 3077 mlx5_devcom_send_event(esw->devcom, 3078 ESW_OFFLOADS_DEVCOM_PAIR, 3079 ESW_OFFLOADS_DEVCOM_UNPAIR, 3080 esw); 3081} 3082 3083void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw) 3084{ 3085 if (IS_ERR_OR_NULL(esw->devcom)) 3086 return; 3087 3088 mlx5_devcom_send_event(esw->devcom, 3089 ESW_OFFLOADS_DEVCOM_UNPAIR, 3090 ESW_OFFLOADS_DEVCOM_UNPAIR, 3091 esw); 3092 3093 mlx5_devcom_unregister_component(esw->devcom); 3094 xa_destroy(&esw->paired); 3095 esw->devcom = NULL; 3096} 3097 3098bool mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch *esw) 3099{ 3100 return mlx5_devcom_comp_is_ready(esw->devcom); 3101} 3102 3103bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw) 3104{ 3105 if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl)) 3106 return false; 3107 3108 if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 3109 MLX5_FDB_TO_VPORT_REG_C_0)) 3110 return false; 3111 3112 return true; 3113} 3114 3115#define MLX5_ESW_METADATA_RSVD_UPLINK 1 3116 3117/* Share the same metadata for uplink's. This is fine because: 3118 * (a) In shared FDB mode (LAG) both uplink's are treated the 3119 * same and tagged with the same metadata. 3120 * (b) In non shared FDB mode, packets from physical port0 3121 * cannot hit eswitch of PF1 and vice versa. 3122 */ 3123static u32 mlx5_esw_match_metadata_reserved(struct mlx5_eswitch *esw) 3124{ 3125 return MLX5_ESW_METADATA_RSVD_UPLINK; 3126} 3127 3128u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw) 3129{ 3130 u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1; 3131 /* Reserve 0xf for internal port offload */ 3132 u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 2; 3133 u32 pf_num; 3134 int id; 3135 3136 /* Only 4 bits of pf_num */ 3137 pf_num = mlx5_get_dev_index(esw->dev); 3138 if (pf_num > max_pf_num) 3139 return 0; 3140 3141 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */ 3142 /* Use only non-zero vport_id (2-4095) for all PF's */ 3143 id = ida_alloc_range(&esw->offloads.vport_metadata_ida, 3144 MLX5_ESW_METADATA_RSVD_UPLINK + 1, 3145 vport_end_ida, GFP_KERNEL); 3146 if (id < 0) 3147 return 0; 3148 id = (pf_num << ESW_VPORT_BITS) | id; 3149 return id; 3150} 3151 3152void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata) 3153{ 3154 u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1; 3155 3156 /* Metadata contains only 12 bits of actual ida id */ 3157 ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask); 3158} 3159 3160static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw, 3161 struct mlx5_vport *vport) 3162{ 3163 if (vport->vport == MLX5_VPORT_UPLINK) 3164 vport->default_metadata = mlx5_esw_match_metadata_reserved(esw); 3165 else 3166 vport->default_metadata = mlx5_esw_match_metadata_alloc(esw); 3167 3168 vport->metadata = vport->default_metadata; 3169 return vport->metadata ? 0 : -ENOSPC; 3170} 3171 3172static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw, 3173 struct mlx5_vport *vport) 3174{ 3175 if (!vport->default_metadata) 3176 return; 3177 3178 if (vport->vport == MLX5_VPORT_UPLINK) 3179 return; 3180 3181 WARN_ON(vport->metadata != vport->default_metadata); 3182 mlx5_esw_match_metadata_free(esw, vport->default_metadata); 3183} 3184 3185static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw) 3186{ 3187 struct mlx5_vport *vport; 3188 unsigned long i; 3189 3190 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 3191 return; 3192 3193 mlx5_esw_for_each_vport(esw, i, vport) 3194 esw_offloads_vport_metadata_cleanup(esw, vport); 3195} 3196 3197static int esw_offloads_metadata_init(struct mlx5_eswitch *esw) 3198{ 3199 struct mlx5_vport *vport; 3200 unsigned long i; 3201 int err; 3202 3203 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 3204 return 0; 3205 3206 mlx5_esw_for_each_vport(esw, i, vport) { 3207 err = esw_offloads_vport_metadata_setup(esw, vport); 3208 if (err) 3209 goto metadata_err; 3210 } 3211 3212 return 0; 3213 3214metadata_err: 3215 esw_offloads_metadata_uninit(esw); 3216 return err; 3217} 3218 3219int 3220esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw, 3221 struct mlx5_vport *vport) 3222{ 3223 int err; 3224 3225 err = esw_acl_ingress_ofld_setup(esw, vport); 3226 if (err) 3227 return err; 3228 3229 err = esw_acl_egress_ofld_setup(esw, vport); 3230 if (err) 3231 goto egress_err; 3232 3233 return 0; 3234 3235egress_err: 3236 esw_acl_ingress_ofld_cleanup(esw, vport); 3237 return err; 3238} 3239 3240void 3241esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw, 3242 struct mlx5_vport *vport) 3243{ 3244 esw_acl_egress_ofld_cleanup(vport); 3245 esw_acl_ingress_ofld_cleanup(esw, vport); 3246} 3247 3248static int esw_create_offloads_acl_tables(struct mlx5_eswitch *esw) 3249{ 3250 struct mlx5_vport *uplink, *manager; 3251 int ret; 3252 3253 uplink = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 3254 if (IS_ERR(uplink)) 3255 return PTR_ERR(uplink); 3256 3257 ret = esw_vport_create_offloads_acl_tables(esw, uplink); 3258 if (ret) 3259 return ret; 3260 3261 manager = mlx5_eswitch_get_vport(esw, esw->manager_vport); 3262 if (IS_ERR(manager)) { 3263 ret = PTR_ERR(manager); 3264 goto err_manager; 3265 } 3266 3267 ret = esw_vport_create_offloads_acl_tables(esw, manager); 3268 if (ret) 3269 goto err_manager; 3270 3271 return 0; 3272 3273err_manager: 3274 esw_vport_destroy_offloads_acl_tables(esw, uplink); 3275 return ret; 3276} 3277 3278static void esw_destroy_offloads_acl_tables(struct mlx5_eswitch *esw) 3279{ 3280 struct mlx5_vport *vport; 3281 3282 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport); 3283 if (!IS_ERR(vport)) 3284 esw_vport_destroy_offloads_acl_tables(esw, vport); 3285 3286 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 3287 if (!IS_ERR(vport)) 3288 esw_vport_destroy_offloads_acl_tables(esw, vport); 3289} 3290 3291int mlx5_eswitch_reload_ib_reps(struct mlx5_eswitch *esw) 3292{ 3293 struct mlx5_eswitch_rep *rep; 3294 unsigned long i; 3295 int ret; 3296 3297 if (!esw || esw->mode != MLX5_ESWITCH_OFFLOADS) 3298 return 0; 3299 3300 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 3301 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED) 3302 return 0; 3303 3304 ret = __esw_offloads_load_rep(esw, rep, REP_IB); 3305 if (ret) 3306 return ret; 3307 3308 mlx5_esw_for_each_rep(esw, i, rep) { 3309 if (atomic_read(&rep->rep_data[REP_ETH].state) == REP_LOADED) 3310 __esw_offloads_load_rep(esw, rep, REP_IB); 3311 } 3312 3313 return 0; 3314} 3315 3316static int esw_offloads_steering_init(struct mlx5_eswitch *esw) 3317{ 3318 struct mlx5_esw_indir_table *indir; 3319 int err; 3320 3321 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb)); 3322 mutex_init(&esw->fdb_table.offloads.vports.lock); 3323 hash_init(esw->fdb_table.offloads.vports.table); 3324 atomic64_set(&esw->user_count, 0); 3325 3326 indir = mlx5_esw_indir_table_init(); 3327 if (IS_ERR(indir)) { 3328 err = PTR_ERR(indir); 3329 goto create_indir_err; 3330 } 3331 esw->fdb_table.offloads.indir = indir; 3332 3333 err = esw_create_offloads_acl_tables(esw); 3334 if (err) 3335 goto create_acl_err; 3336 3337 err = esw_create_offloads_table(esw); 3338 if (err) 3339 goto create_offloads_err; 3340 3341 err = esw_create_restore_table(esw); 3342 if (err) 3343 goto create_restore_err; 3344 3345 err = esw_create_offloads_fdb_tables(esw); 3346 if (err) 3347 goto create_fdb_err; 3348 3349 err = esw_create_vport_rx_group(esw); 3350 if (err) 3351 goto create_fg_err; 3352 3353 err = esw_create_vport_rx_drop_group(esw); 3354 if (err) 3355 goto create_rx_drop_fg_err; 3356 3357 err = esw_create_vport_rx_drop_rule(esw); 3358 if (err) 3359 goto create_rx_drop_rule_err; 3360 3361 return 0; 3362 3363create_rx_drop_rule_err: 3364 esw_destroy_vport_rx_drop_group(esw); 3365create_rx_drop_fg_err: 3366 esw_destroy_vport_rx_group(esw); 3367create_fg_err: 3368 esw_destroy_offloads_fdb_tables(esw); 3369create_fdb_err: 3370 esw_destroy_restore_table(esw); 3371create_restore_err: 3372 esw_destroy_offloads_table(esw); 3373create_offloads_err: 3374 esw_destroy_offloads_acl_tables(esw); 3375create_acl_err: 3376 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir); 3377create_indir_err: 3378 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 3379 return err; 3380} 3381 3382static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw) 3383{ 3384 esw_destroy_vport_rx_drop_rule(esw); 3385 esw_destroy_vport_rx_drop_group(esw); 3386 esw_destroy_vport_rx_group(esw); 3387 esw_destroy_offloads_fdb_tables(esw); 3388 esw_destroy_restore_table(esw); 3389 esw_destroy_offloads_table(esw); 3390 esw_destroy_offloads_acl_tables(esw); 3391 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir); 3392 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 3393} 3394 3395static void 3396esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) 3397{ 3398 struct devlink *devlink; 3399 bool host_pf_disabled; 3400 u16 new_num_vfs; 3401 3402 new_num_vfs = MLX5_GET(query_esw_functions_out, out, 3403 host_params_context.host_num_of_vfs); 3404 host_pf_disabled = MLX5_GET(query_esw_functions_out, out, 3405 host_params_context.host_pf_disabled); 3406 3407 if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled) 3408 return; 3409 3410 devlink = priv_to_devlink(esw->dev); 3411 devl_lock(devlink); 3412 /* Number of VFs can only change from "0 to x" or "x to 0". */ 3413 if (esw->esw_funcs.num_vfs > 0) { 3414 mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); 3415 } else { 3416 int err; 3417 3418 err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs, 3419 MLX5_VPORT_UC_ADDR_CHANGE); 3420 if (err) { 3421 devl_unlock(devlink); 3422 return; 3423 } 3424 } 3425 esw->esw_funcs.num_vfs = new_num_vfs; 3426 devl_unlock(devlink); 3427} 3428 3429static void esw_functions_changed_event_handler(struct work_struct *work) 3430{ 3431 struct mlx5_host_work *host_work; 3432 struct mlx5_eswitch *esw; 3433 const u32 *out; 3434 3435 host_work = container_of(work, struct mlx5_host_work, work); 3436 esw = host_work->esw; 3437 3438 out = mlx5_esw_query_functions(esw->dev); 3439 if (IS_ERR(out)) 3440 goto out; 3441 3442 esw_vfs_changed_event_handler(esw, out); 3443 kvfree(out); 3444out: 3445 kfree(host_work); 3446} 3447 3448int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data) 3449{ 3450 struct mlx5_esw_functions *esw_funcs; 3451 struct mlx5_host_work *host_work; 3452 struct mlx5_eswitch *esw; 3453 3454 host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC); 3455 if (!host_work) 3456 return NOTIFY_DONE; 3457 3458 esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb); 3459 esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs); 3460 3461 host_work->esw = esw; 3462 3463 INIT_WORK(&host_work->work, esw_functions_changed_event_handler); 3464 queue_work(esw->work_queue, &host_work->work); 3465 3466 return NOTIFY_OK; 3467} 3468 3469static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw) 3470{ 3471 const u32 *query_host_out; 3472 3473 if (!mlx5_core_is_ecpf_esw_manager(esw->dev)) 3474 return 0; 3475 3476 query_host_out = mlx5_esw_query_functions(esw->dev); 3477 if (IS_ERR(query_host_out)) 3478 return PTR_ERR(query_host_out); 3479 3480 /* Mark non local controller with non zero controller number. */ 3481 esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out, 3482 host_params_context.host_number); 3483 kvfree(query_host_out); 3484 return 0; 3485} 3486 3487bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch *esw, u32 controller) 3488{ 3489 /* Local controller is always valid */ 3490 if (controller == 0) 3491 return true; 3492 3493 if (!mlx5_core_is_ecpf_esw_manager(esw->dev)) 3494 return false; 3495 3496 /* External host number starts with zero in device */ 3497 return (controller == esw->offloads.host_number + 1); 3498} 3499 3500int esw_offloads_enable(struct mlx5_eswitch *esw) 3501{ 3502 struct mapping_ctx *reg_c0_obj_pool; 3503 struct mlx5_vport *vport; 3504 unsigned long i; 3505 u64 mapping_id; 3506 int err; 3507 3508 mutex_init(&esw->offloads.termtbl_mutex); 3509 mlx5_rdma_enable_roce(esw->dev); 3510 3511 err = mlx5_esw_host_number_init(esw); 3512 if (err) 3513 goto err_metadata; 3514 3515 err = esw_offloads_metadata_init(esw); 3516 if (err) 3517 goto err_metadata; 3518 3519 err = esw_set_passing_vport_metadata(esw, true); 3520 if (err) 3521 goto err_vport_metadata; 3522 3523 mapping_id = mlx5_query_nic_system_image_guid(esw->dev); 3524 3525 reg_c0_obj_pool = mapping_create_for_id(mapping_id, MAPPING_TYPE_CHAIN, 3526 sizeof(struct mlx5_mapped_obj), 3527 ESW_REG_C0_USER_DATA_METADATA_MASK, 3528 true); 3529 3530 if (IS_ERR(reg_c0_obj_pool)) { 3531 err = PTR_ERR(reg_c0_obj_pool); 3532 goto err_pool; 3533 } 3534 esw->offloads.reg_c0_obj_pool = reg_c0_obj_pool; 3535 3536 err = esw_offloads_steering_init(esw); 3537 if (err) 3538 goto err_steering_init; 3539 3540 /* Representor will control the vport link state */ 3541 mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs) 3542 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN; 3543 if (mlx5_core_ec_sriov_enabled(esw->dev)) 3544 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) 3545 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN; 3546 3547 /* Uplink vport rep must load first. */ 3548 err = mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK); 3549 if (err) 3550 goto err_uplink; 3551 3552 err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE); 3553 if (err) 3554 goto err_vports; 3555 3556 return 0; 3557 3558err_vports: 3559 mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK); 3560err_uplink: 3561 esw_offloads_steering_cleanup(esw); 3562err_steering_init: 3563 mapping_destroy(reg_c0_obj_pool); 3564err_pool: 3565 esw_set_passing_vport_metadata(esw, false); 3566err_vport_metadata: 3567 esw_offloads_metadata_uninit(esw); 3568err_metadata: 3569 mlx5_rdma_disable_roce(esw->dev); 3570 mutex_destroy(&esw->offloads.termtbl_mutex); 3571 return err; 3572} 3573 3574static int esw_offloads_stop(struct mlx5_eswitch *esw, 3575 struct netlink_ext_ack *extack) 3576{ 3577 int err; 3578 3579 esw->mode = MLX5_ESWITCH_LEGACY; 3580 3581 /* If changing from switchdev to legacy mode without sriov enabled, 3582 * no need to create legacy fdb. 3583 */ 3584 if (!mlx5_core_is_pf(esw->dev) || !mlx5_sriov_is_enabled(esw->dev)) 3585 return 0; 3586 3587 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS); 3588 if (err) 3589 NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy"); 3590 3591 return err; 3592} 3593 3594void esw_offloads_disable(struct mlx5_eswitch *esw) 3595{ 3596 mlx5_eswitch_disable_pf_vf_vports(esw); 3597 mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK); 3598 esw_set_passing_vport_metadata(esw, false); 3599 esw_offloads_steering_cleanup(esw); 3600 mapping_destroy(esw->offloads.reg_c0_obj_pool); 3601 esw_offloads_metadata_uninit(esw); 3602 mlx5_rdma_disable_roce(esw->dev); 3603 mutex_destroy(&esw->offloads.termtbl_mutex); 3604} 3605 3606static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode) 3607{ 3608 switch (mode) { 3609 case DEVLINK_ESWITCH_MODE_LEGACY: 3610 *mlx5_mode = MLX5_ESWITCH_LEGACY; 3611 break; 3612 case DEVLINK_ESWITCH_MODE_SWITCHDEV: 3613 *mlx5_mode = MLX5_ESWITCH_OFFLOADS; 3614 break; 3615 default: 3616 return -EINVAL; 3617 } 3618 3619 return 0; 3620} 3621 3622static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode) 3623{ 3624 switch (mlx5_mode) { 3625 case MLX5_ESWITCH_LEGACY: 3626 *mode = DEVLINK_ESWITCH_MODE_LEGACY; 3627 break; 3628 case MLX5_ESWITCH_OFFLOADS: 3629 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; 3630 break; 3631 default: 3632 return -EINVAL; 3633 } 3634 3635 return 0; 3636} 3637 3638static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode) 3639{ 3640 switch (mode) { 3641 case DEVLINK_ESWITCH_INLINE_MODE_NONE: 3642 *mlx5_mode = MLX5_INLINE_MODE_NONE; 3643 break; 3644 case DEVLINK_ESWITCH_INLINE_MODE_LINK: 3645 *mlx5_mode = MLX5_INLINE_MODE_L2; 3646 break; 3647 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK: 3648 *mlx5_mode = MLX5_INLINE_MODE_IP; 3649 break; 3650 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT: 3651 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP; 3652 break; 3653 default: 3654 return -EINVAL; 3655 } 3656 3657 return 0; 3658} 3659 3660static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode) 3661{ 3662 switch (mlx5_mode) { 3663 case MLX5_INLINE_MODE_NONE: 3664 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE; 3665 break; 3666 case MLX5_INLINE_MODE_L2: 3667 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK; 3668 break; 3669 case MLX5_INLINE_MODE_IP: 3670 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK; 3671 break; 3672 case MLX5_INLINE_MODE_TCP_UDP: 3673 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT; 3674 break; 3675 default: 3676 return -EINVAL; 3677 } 3678 3679 return 0; 3680} 3681 3682int mlx5_eswitch_block_mode(struct mlx5_core_dev *dev) 3683{ 3684 struct mlx5_eswitch *esw = dev->priv.eswitch; 3685 int err; 3686 3687 if (!mlx5_esw_allowed(esw)) 3688 return 0; 3689 3690 /* Take TC into account */ 3691 err = mlx5_esw_try_lock(esw); 3692 if (err < 0) 3693 return err; 3694 3695 esw->offloads.num_block_mode++; 3696 mlx5_esw_unlock(esw); 3697 return 0; 3698} 3699 3700void mlx5_eswitch_unblock_mode(struct mlx5_core_dev *dev) 3701{ 3702 struct mlx5_eswitch *esw = dev->priv.eswitch; 3703 3704 if (!mlx5_esw_allowed(esw)) 3705 return; 3706 3707 down_write(&esw->mode_lock); 3708 esw->offloads.num_block_mode--; 3709 up_write(&esw->mode_lock); 3710} 3711 3712int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, 3713 struct netlink_ext_ack *extack) 3714{ 3715 u16 cur_mlx5_mode, mlx5_mode = 0; 3716 struct mlx5_eswitch *esw; 3717 int err = 0; 3718 3719 esw = mlx5_devlink_eswitch_get(devlink); 3720 if (IS_ERR(esw)) 3721 return PTR_ERR(esw); 3722 3723 if (esw_mode_from_devlink(mode, &mlx5_mode)) 3724 return -EINVAL; 3725 3726 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && mlx5_get_sd(esw->dev)) { 3727 NL_SET_ERR_MSG_MOD(extack, 3728 "Can't change E-Switch mode to switchdev when multi-PF netdev (Socket Direct) is configured."); 3729 return -EPERM; 3730 } 3731 3732 mlx5_lag_disable_change(esw->dev); 3733 err = mlx5_esw_try_lock(esw); 3734 if (err < 0) { 3735 NL_SET_ERR_MSG_MOD(extack, "Can't change mode, E-Switch is busy"); 3736 goto enable_lag; 3737 } 3738 cur_mlx5_mode = err; 3739 err = 0; 3740 3741 if (cur_mlx5_mode == mlx5_mode) 3742 goto unlock; 3743 3744 if (esw->offloads.num_block_mode) { 3745 NL_SET_ERR_MSG_MOD(extack, 3746 "Can't change eswitch mode when IPsec SA and/or policies are configured"); 3747 err = -EOPNOTSUPP; 3748 goto unlock; 3749 } 3750 3751 esw->eswitch_operation_in_progress = true; 3752 up_write(&esw->mode_lock); 3753 3754 mlx5_eswitch_disable_locked(esw); 3755 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) { 3756 if (mlx5_devlink_trap_get_num_active(esw->dev)) { 3757 NL_SET_ERR_MSG_MOD(extack, 3758 "Can't change mode while devlink traps are active"); 3759 err = -EOPNOTSUPP; 3760 goto skip; 3761 } 3762 err = esw_offloads_start(esw, extack); 3763 } else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) { 3764 err = esw_offloads_stop(esw, extack); 3765 mlx5_rescan_drivers(esw->dev); 3766 } else { 3767 err = -EINVAL; 3768 } 3769 3770skip: 3771 down_write(&esw->mode_lock); 3772 esw->eswitch_operation_in_progress = false; 3773unlock: 3774 mlx5_esw_unlock(esw); 3775enable_lag: 3776 mlx5_lag_enable_change(esw->dev); 3777 return err; 3778} 3779 3780int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) 3781{ 3782 struct mlx5_eswitch *esw; 3783 3784 esw = mlx5_devlink_eswitch_get(devlink); 3785 if (IS_ERR(esw)) 3786 return PTR_ERR(esw); 3787 3788 return esw_mode_to_devlink(esw->mode, mode); 3789} 3790 3791static int mlx5_esw_vports_inline_set(struct mlx5_eswitch *esw, u8 mlx5_mode, 3792 struct netlink_ext_ack *extack) 3793{ 3794 struct mlx5_core_dev *dev = esw->dev; 3795 struct mlx5_vport *vport; 3796 u16 err_vport_num = 0; 3797 unsigned long i; 3798 int err = 0; 3799 3800 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 3801 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode); 3802 if (err) { 3803 err_vport_num = vport->vport; 3804 NL_SET_ERR_MSG_MOD(extack, 3805 "Failed to set min inline on vport"); 3806 goto revert_inline_mode; 3807 } 3808 } 3809 if (mlx5_core_ec_sriov_enabled(esw->dev)) { 3810 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) { 3811 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode); 3812 if (err) { 3813 err_vport_num = vport->vport; 3814 NL_SET_ERR_MSG_MOD(extack, 3815 "Failed to set min inline on vport"); 3816 goto revert_ec_vf_inline_mode; 3817 } 3818 } 3819 } 3820 return 0; 3821 3822revert_ec_vf_inline_mode: 3823 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) { 3824 if (vport->vport == err_vport_num) 3825 break; 3826 mlx5_modify_nic_vport_min_inline(dev, 3827 vport->vport, 3828 esw->offloads.inline_mode); 3829 } 3830revert_inline_mode: 3831 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 3832 if (vport->vport == err_vport_num) 3833 break; 3834 mlx5_modify_nic_vport_min_inline(dev, 3835 vport->vport, 3836 esw->offloads.inline_mode); 3837 } 3838 return err; 3839} 3840 3841int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode, 3842 struct netlink_ext_ack *extack) 3843{ 3844 struct mlx5_core_dev *dev = devlink_priv(devlink); 3845 struct mlx5_eswitch *esw; 3846 u8 mlx5_mode; 3847 int err; 3848 3849 esw = mlx5_devlink_eswitch_get(devlink); 3850 if (IS_ERR(esw)) 3851 return PTR_ERR(esw); 3852 3853 down_write(&esw->mode_lock); 3854 3855 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 3856 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 3857 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) { 3858 err = 0; 3859 goto out; 3860 } 3861 3862 fallthrough; 3863 case MLX5_CAP_INLINE_MODE_L2: 3864 NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set"); 3865 err = -EOPNOTSUPP; 3866 goto out; 3867 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 3868 break; 3869 } 3870 3871 if (atomic64_read(&esw->offloads.num_flows) > 0) { 3872 NL_SET_ERR_MSG_MOD(extack, 3873 "Can't set inline mode when flows are configured"); 3874 err = -EOPNOTSUPP; 3875 goto out; 3876 } 3877 3878 err = esw_inline_mode_from_devlink(mode, &mlx5_mode); 3879 if (err) 3880 goto out; 3881 3882 esw->eswitch_operation_in_progress = true; 3883 up_write(&esw->mode_lock); 3884 3885 err = mlx5_esw_vports_inline_set(esw, mlx5_mode, extack); 3886 if (!err) 3887 esw->offloads.inline_mode = mlx5_mode; 3888 3889 down_write(&esw->mode_lock); 3890 esw->eswitch_operation_in_progress = false; 3891 up_write(&esw->mode_lock); 3892 return 0; 3893 3894out: 3895 up_write(&esw->mode_lock); 3896 return err; 3897} 3898 3899int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode) 3900{ 3901 struct mlx5_eswitch *esw; 3902 3903 esw = mlx5_devlink_eswitch_get(devlink); 3904 if (IS_ERR(esw)) 3905 return PTR_ERR(esw); 3906 3907 return esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode); 3908} 3909 3910bool mlx5_eswitch_block_encap(struct mlx5_core_dev *dev) 3911{ 3912 struct mlx5_eswitch *esw = dev->priv.eswitch; 3913 3914 if (!mlx5_esw_allowed(esw)) 3915 return true; 3916 3917 down_write(&esw->mode_lock); 3918 if (esw->mode != MLX5_ESWITCH_LEGACY && 3919 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) { 3920 up_write(&esw->mode_lock); 3921 return false; 3922 } 3923 3924 esw->offloads.num_block_encap++; 3925 up_write(&esw->mode_lock); 3926 return true; 3927} 3928 3929void mlx5_eswitch_unblock_encap(struct mlx5_core_dev *dev) 3930{ 3931 struct mlx5_eswitch *esw = dev->priv.eswitch; 3932 3933 if (!mlx5_esw_allowed(esw)) 3934 return; 3935 3936 down_write(&esw->mode_lock); 3937 esw->offloads.num_block_encap--; 3938 up_write(&esw->mode_lock); 3939} 3940 3941int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, 3942 enum devlink_eswitch_encap_mode encap, 3943 struct netlink_ext_ack *extack) 3944{ 3945 struct mlx5_core_dev *dev = devlink_priv(devlink); 3946 struct mlx5_eswitch *esw; 3947 int err = 0; 3948 3949 esw = mlx5_devlink_eswitch_get(devlink); 3950 if (IS_ERR(esw)) 3951 return PTR_ERR(esw); 3952 3953 down_write(&esw->mode_lock); 3954 3955 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE && 3956 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) || 3957 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) { 3958 err = -EOPNOTSUPP; 3959 goto unlock; 3960 } 3961 3962 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) { 3963 err = -EOPNOTSUPP; 3964 goto unlock; 3965 } 3966 3967 if (esw->mode == MLX5_ESWITCH_LEGACY) { 3968 esw->offloads.encap = encap; 3969 goto unlock; 3970 } 3971 3972 if (esw->offloads.encap == encap) 3973 goto unlock; 3974 3975 if (atomic64_read(&esw->offloads.num_flows) > 0) { 3976 NL_SET_ERR_MSG_MOD(extack, 3977 "Can't set encapsulation when flows are configured"); 3978 err = -EOPNOTSUPP; 3979 goto unlock; 3980 } 3981 3982 if (esw->offloads.num_block_encap) { 3983 NL_SET_ERR_MSG_MOD(extack, 3984 "Can't set encapsulation when IPsec SA and/or policies are configured"); 3985 err = -EOPNOTSUPP; 3986 goto unlock; 3987 } 3988 3989 esw->eswitch_operation_in_progress = true; 3990 up_write(&esw->mode_lock); 3991 3992 esw_destroy_offloads_fdb_tables(esw); 3993 3994 esw->offloads.encap = encap; 3995 3996 err = esw_create_offloads_fdb_tables(esw); 3997 3998 if (err) { 3999 NL_SET_ERR_MSG_MOD(extack, 4000 "Failed re-creating fast FDB table"); 4001 esw->offloads.encap = !encap; 4002 (void)esw_create_offloads_fdb_tables(esw); 4003 } 4004 4005 down_write(&esw->mode_lock); 4006 esw->eswitch_operation_in_progress = false; 4007 4008unlock: 4009 up_write(&esw->mode_lock); 4010 return err; 4011} 4012 4013int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, 4014 enum devlink_eswitch_encap_mode *encap) 4015{ 4016 struct mlx5_eswitch *esw; 4017 4018 esw = mlx5_devlink_eswitch_get(devlink); 4019 if (IS_ERR(esw)) 4020 return PTR_ERR(esw); 4021 4022 *encap = esw->offloads.encap; 4023 return 0; 4024} 4025 4026static bool 4027mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num) 4028{ 4029 /* Currently, only ECPF based device has representor for host PF. */ 4030 if (vport_num == MLX5_VPORT_PF && 4031 !mlx5_core_is_ecpf_esw_manager(esw->dev)) 4032 return false; 4033 4034 if (vport_num == MLX5_VPORT_ECPF && 4035 !mlx5_ecpf_vport_exists(esw->dev)) 4036 return false; 4037 4038 return true; 4039} 4040 4041void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw, 4042 const struct mlx5_eswitch_rep_ops *ops, 4043 u8 rep_type) 4044{ 4045 struct mlx5_eswitch_rep_data *rep_data; 4046 struct mlx5_eswitch_rep *rep; 4047 unsigned long i; 4048 4049 esw->offloads.rep_ops[rep_type] = ops; 4050 mlx5_esw_for_each_rep(esw, i, rep) { 4051 if (likely(mlx5_eswitch_vport_has_rep(esw, rep->vport))) { 4052 rep->esw = esw; 4053 rep_data = &rep->rep_data[rep_type]; 4054 atomic_set(&rep_data->state, REP_REGISTERED); 4055 } 4056 } 4057} 4058EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps); 4059 4060void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type) 4061{ 4062 struct mlx5_eswitch_rep *rep; 4063 unsigned long i; 4064 4065 if (esw->mode == MLX5_ESWITCH_OFFLOADS) 4066 __unload_reps_all_vport(esw, rep_type); 4067 4068 mlx5_esw_for_each_rep(esw, i, rep) 4069 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED); 4070} 4071EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps); 4072 4073void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type) 4074{ 4075 struct mlx5_eswitch_rep *rep; 4076 4077 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 4078 return rep->rep_data[rep_type].priv; 4079} 4080 4081void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw, 4082 u16 vport, 4083 u8 rep_type) 4084{ 4085 struct mlx5_eswitch_rep *rep; 4086 4087 rep = mlx5_eswitch_get_rep(esw, vport); 4088 4089 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 4090 esw->offloads.rep_ops[rep_type]->get_proto_dev) 4091 return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep); 4092 return NULL; 4093} 4094EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev); 4095 4096void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type) 4097{ 4098 return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type); 4099} 4100EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev); 4101 4102struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw, 4103 u16 vport) 4104{ 4105 return mlx5_eswitch_get_rep(esw, vport); 4106} 4107EXPORT_SYMBOL(mlx5_eswitch_vport_rep); 4108 4109bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw) 4110{ 4111 return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED); 4112} 4113EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled); 4114 4115bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw) 4116{ 4117 return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA); 4118} 4119EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled); 4120 4121u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw, 4122 u16 vport_num) 4123{ 4124 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 4125 4126 if (WARN_ON_ONCE(IS_ERR(vport))) 4127 return 0; 4128 4129 return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS); 4130} 4131EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match); 4132 4133static int mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch *esw, u16 vport_num, u16 *vhca_id) 4134{ 4135 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 4136 void *query_ctx; 4137 void *hca_caps; 4138 int err; 4139 4140 *vhca_id = 0; 4141 4142 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 4143 if (!query_ctx) 4144 return -ENOMEM; 4145 4146 err = mlx5_vport_get_other_func_general_cap(esw->dev, vport_num, query_ctx); 4147 if (err) 4148 goto out_free; 4149 4150 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 4151 *vhca_id = MLX5_GET(cmd_hca_cap, hca_caps, vhca_id); 4152 4153out_free: 4154 kfree(query_ctx); 4155 return err; 4156} 4157 4158int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch *esw, u16 vport_num) 4159{ 4160 u16 *old_entry, *vhca_map_entry, vhca_id; 4161 int err; 4162 4163 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id); 4164 if (err) { 4165 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%u,err=%d)\n", 4166 vport_num, err); 4167 return err; 4168 } 4169 4170 vhca_map_entry = kmalloc(sizeof(*vhca_map_entry), GFP_KERNEL); 4171 if (!vhca_map_entry) 4172 return -ENOMEM; 4173 4174 *vhca_map_entry = vport_num; 4175 old_entry = xa_store(&esw->offloads.vhca_map, vhca_id, vhca_map_entry, GFP_KERNEL); 4176 if (xa_is_err(old_entry)) { 4177 kfree(vhca_map_entry); 4178 return xa_err(old_entry); 4179 } 4180 kfree(old_entry); 4181 return 0; 4182} 4183 4184void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch *esw, u16 vport_num) 4185{ 4186 u16 *vhca_map_entry, vhca_id; 4187 int err; 4188 4189 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id); 4190 if (err) 4191 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%hu,err=%d)\n", 4192 vport_num, err); 4193 4194 vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vhca_id); 4195 kfree(vhca_map_entry); 4196} 4197 4198int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num) 4199{ 4200 u16 *res = xa_load(&esw->offloads.vhca_map, vhca_id); 4201 4202 if (!res) 4203 return -ENOENT; 4204 4205 *vport_num = *res; 4206 return 0; 4207} 4208 4209u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw, 4210 u16 vport_num) 4211{ 4212 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 4213 4214 if (WARN_ON_ONCE(IS_ERR(vport))) 4215 return 0; 4216 4217 return vport->metadata; 4218} 4219EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set); 4220 4221int mlx5_devlink_port_fn_hw_addr_get(struct devlink_port *port, 4222 u8 *hw_addr, int *hw_addr_len, 4223 struct netlink_ext_ack *extack) 4224{ 4225 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4226 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4227 4228 mutex_lock(&esw->state_lock); 4229 ether_addr_copy(hw_addr, vport->info.mac); 4230 *hw_addr_len = ETH_ALEN; 4231 mutex_unlock(&esw->state_lock); 4232 return 0; 4233} 4234 4235int mlx5_devlink_port_fn_hw_addr_set(struct devlink_port *port, 4236 const u8 *hw_addr, int hw_addr_len, 4237 struct netlink_ext_ack *extack) 4238{ 4239 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4240 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4241 4242 return mlx5_eswitch_set_vport_mac(esw, vport->vport, hw_addr); 4243} 4244 4245int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enabled, 4246 struct netlink_ext_ack *extack) 4247{ 4248 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4249 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4250 4251 if (!MLX5_CAP_GEN(esw->dev, migration)) { 4252 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration"); 4253 return -EOPNOTSUPP; 4254 } 4255 4256 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 4257 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); 4258 return -EOPNOTSUPP; 4259 } 4260 4261 mutex_lock(&esw->state_lock); 4262 *is_enabled = vport->info.mig_enabled; 4263 mutex_unlock(&esw->state_lock); 4264 return 0; 4265} 4266 4267int mlx5_devlink_port_fn_migratable_set(struct devlink_port *port, bool enable, 4268 struct netlink_ext_ack *extack) 4269{ 4270 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4271 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4272 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 4273 void *query_ctx; 4274 void *hca_caps; 4275 int err; 4276 4277 if (!MLX5_CAP_GEN(esw->dev, migration)) { 4278 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration"); 4279 return -EOPNOTSUPP; 4280 } 4281 4282 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 4283 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); 4284 return -EOPNOTSUPP; 4285 } 4286 4287 mutex_lock(&esw->state_lock); 4288 4289 if (vport->info.mig_enabled == enable) { 4290 err = 0; 4291 goto out; 4292 } 4293 4294 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 4295 if (!query_ctx) { 4296 err = -ENOMEM; 4297 goto out; 4298 } 4299 4300 err = mlx5_vport_get_other_func_cap(esw->dev, vport->vport, query_ctx, 4301 MLX5_CAP_GENERAL_2); 4302 if (err) { 4303 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 4304 goto out_free; 4305 } 4306 4307 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 4308 MLX5_SET(cmd_hca_cap_2, hca_caps, migratable, enable); 4309 4310 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport->vport, 4311 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2); 4312 if (err) { 4313 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA migratable cap"); 4314 goto out_free; 4315 } 4316 4317 vport->info.mig_enabled = enable; 4318 4319out_free: 4320 kfree(query_ctx); 4321out: 4322 mutex_unlock(&esw->state_lock); 4323 return err; 4324} 4325 4326int mlx5_devlink_port_fn_roce_get(struct devlink_port *port, bool *is_enabled, 4327 struct netlink_ext_ack *extack) 4328{ 4329 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4330 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4331 4332 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 4333 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); 4334 return -EOPNOTSUPP; 4335 } 4336 4337 mutex_lock(&esw->state_lock); 4338 *is_enabled = vport->info.roce_enabled; 4339 mutex_unlock(&esw->state_lock); 4340 return 0; 4341} 4342 4343int mlx5_devlink_port_fn_roce_set(struct devlink_port *port, bool enable, 4344 struct netlink_ext_ack *extack) 4345{ 4346 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4347 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4348 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 4349 u16 vport_num = vport->vport; 4350 void *query_ctx; 4351 void *hca_caps; 4352 int err; 4353 4354 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 4355 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); 4356 return -EOPNOTSUPP; 4357 } 4358 4359 mutex_lock(&esw->state_lock); 4360 4361 if (vport->info.roce_enabled == enable) { 4362 err = 0; 4363 goto out; 4364 } 4365 4366 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 4367 if (!query_ctx) { 4368 err = -ENOMEM; 4369 goto out; 4370 } 4371 4372 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx, 4373 MLX5_CAP_GENERAL); 4374 if (err) { 4375 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 4376 goto out_free; 4377 } 4378 4379 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 4380 MLX5_SET(cmd_hca_cap, hca_caps, roce, enable); 4381 4382 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport_num, 4383 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); 4384 if (err) { 4385 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA roce cap"); 4386 goto out_free; 4387 } 4388 4389 vport->info.roce_enabled = enable; 4390 4391out_free: 4392 kfree(query_ctx); 4393out: 4394 mutex_unlock(&esw->state_lock); 4395 return err; 4396} 4397 4398int 4399mlx5_eswitch_restore_ipsec_rule(struct mlx5_eswitch *esw, struct mlx5_flow_handle *rule, 4400 struct mlx5_esw_flow_attr *esw_attr, int attr_idx) 4401{ 4402 struct mlx5_flow_destination new_dest = {}; 4403 struct mlx5_flow_destination old_dest = {}; 4404 4405 if (!esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx)) 4406 return 0; 4407 4408 esw_setup_dest_fwd_ipsec(&old_dest, NULL, esw, esw_attr, attr_idx, 0, false); 4409 esw_setup_dest_fwd_vport(&new_dest, NULL, esw, esw_attr, attr_idx, 0, false); 4410 4411 return mlx5_modify_rule_destination(rule, &new_dest, &old_dest); 4412} 4413 4414#ifdef CONFIG_XFRM_OFFLOAD 4415int mlx5_devlink_port_fn_ipsec_crypto_get(struct devlink_port *port, bool *is_enabled, 4416 struct netlink_ext_ack *extack) 4417{ 4418 struct mlx5_eswitch *esw; 4419 struct mlx5_vport *vport; 4420 int err = 0; 4421 4422 esw = mlx5_devlink_eswitch_get(port->devlink); 4423 if (IS_ERR(esw)) 4424 return PTR_ERR(esw); 4425 4426 if (!mlx5_esw_ipsec_vf_offload_supported(esw->dev)) { 4427 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPSec crypto"); 4428 return -EOPNOTSUPP; 4429 } 4430 4431 vport = mlx5_devlink_port_vport_get(port); 4432 4433 mutex_lock(&esw->state_lock); 4434 if (!vport->enabled) { 4435 err = -EOPNOTSUPP; 4436 goto unlock; 4437 } 4438 4439 *is_enabled = vport->info.ipsec_crypto_enabled; 4440unlock: 4441 mutex_unlock(&esw->state_lock); 4442 return err; 4443} 4444 4445int mlx5_devlink_port_fn_ipsec_crypto_set(struct devlink_port *port, bool enable, 4446 struct netlink_ext_ack *extack) 4447{ 4448 struct mlx5_eswitch *esw; 4449 struct mlx5_vport *vport; 4450 u16 vport_num; 4451 int err; 4452 4453 esw = mlx5_devlink_eswitch_get(port->devlink); 4454 if (IS_ERR(esw)) 4455 return PTR_ERR(esw); 4456 4457 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); 4458 err = mlx5_esw_ipsec_vf_crypto_offload_supported(esw->dev, vport_num); 4459 if (err) { 4460 NL_SET_ERR_MSG_MOD(extack, 4461 "Device doesn't support IPsec crypto"); 4462 return err; 4463 } 4464 4465 vport = mlx5_devlink_port_vport_get(port); 4466 4467 mutex_lock(&esw->state_lock); 4468 if (!vport->enabled) { 4469 err = -EOPNOTSUPP; 4470 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); 4471 goto unlock; 4472 } 4473 4474 if (vport->info.ipsec_crypto_enabled == enable) 4475 goto unlock; 4476 4477 if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) { 4478 err = -EBUSY; 4479 goto unlock; 4480 } 4481 4482 err = mlx5_esw_ipsec_vf_crypto_offload_set(esw, vport, enable); 4483 if (err) { 4484 NL_SET_ERR_MSG_MOD(extack, "Failed to set IPsec crypto"); 4485 goto unlock; 4486 } 4487 4488 vport->info.ipsec_crypto_enabled = enable; 4489 if (enable) 4490 esw->enabled_ipsec_vf_count++; 4491 else 4492 esw->enabled_ipsec_vf_count--; 4493unlock: 4494 mutex_unlock(&esw->state_lock); 4495 return err; 4496} 4497 4498int mlx5_devlink_port_fn_ipsec_packet_get(struct devlink_port *port, bool *is_enabled, 4499 struct netlink_ext_ack *extack) 4500{ 4501 struct mlx5_eswitch *esw; 4502 struct mlx5_vport *vport; 4503 int err = 0; 4504 4505 esw = mlx5_devlink_eswitch_get(port->devlink); 4506 if (IS_ERR(esw)) 4507 return PTR_ERR(esw); 4508 4509 if (!mlx5_esw_ipsec_vf_offload_supported(esw->dev)) { 4510 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPsec packet"); 4511 return -EOPNOTSUPP; 4512 } 4513 4514 vport = mlx5_devlink_port_vport_get(port); 4515 4516 mutex_lock(&esw->state_lock); 4517 if (!vport->enabled) { 4518 err = -EOPNOTSUPP; 4519 goto unlock; 4520 } 4521 4522 *is_enabled = vport->info.ipsec_packet_enabled; 4523unlock: 4524 mutex_unlock(&esw->state_lock); 4525 return err; 4526} 4527 4528int mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port *port, 4529 bool enable, 4530 struct netlink_ext_ack *extack) 4531{ 4532 struct mlx5_eswitch *esw; 4533 struct mlx5_vport *vport; 4534 u16 vport_num; 4535 int err; 4536 4537 esw = mlx5_devlink_eswitch_get(port->devlink); 4538 if (IS_ERR(esw)) 4539 return PTR_ERR(esw); 4540 4541 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); 4542 err = mlx5_esw_ipsec_vf_packet_offload_supported(esw->dev, vport_num); 4543 if (err) { 4544 NL_SET_ERR_MSG_MOD(extack, 4545 "Device doesn't support IPsec packet mode"); 4546 return err; 4547 } 4548 4549 vport = mlx5_devlink_port_vport_get(port); 4550 mutex_lock(&esw->state_lock); 4551 if (!vport->enabled) { 4552 err = -EOPNOTSUPP; 4553 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); 4554 goto unlock; 4555 } 4556 4557 if (vport->info.ipsec_packet_enabled == enable) 4558 goto unlock; 4559 4560 if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) { 4561 err = -EBUSY; 4562 goto unlock; 4563 } 4564 4565 err = mlx5_esw_ipsec_vf_packet_offload_set(esw, vport, enable); 4566 if (err) { 4567 NL_SET_ERR_MSG_MOD(extack, 4568 "Failed to set IPsec packet mode"); 4569 goto unlock; 4570 } 4571 4572 vport->info.ipsec_packet_enabled = enable; 4573 if (enable) 4574 esw->enabled_ipsec_vf_count++; 4575 else 4576 esw->enabled_ipsec_vf_count--; 4577unlock: 4578 mutex_unlock(&esw->state_lock); 4579 return err; 4580} 4581#endif /* CONFIG_XFRM_OFFLOAD */ 4582 4583int 4584mlx5_devlink_port_fn_max_io_eqs_get(struct devlink_port *port, u32 *max_io_eqs, 4585 struct netlink_ext_ack *extack) 4586{ 4587 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4588 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 4589 u16 vport_num = vport->vport; 4590 struct mlx5_eswitch *esw; 4591 void *query_ctx; 4592 void *hca_caps; 4593 u32 max_eqs; 4594 int err; 4595 4596 esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4597 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 4598 NL_SET_ERR_MSG_MOD(extack, 4599 "Device doesn't support VHCA management"); 4600 return -EOPNOTSUPP; 4601 } 4602 4603 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 4604 if (!query_ctx) 4605 return -ENOMEM; 4606 4607 mutex_lock(&esw->state_lock); 4608 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx, 4609 MLX5_CAP_GENERAL); 4610 if (err) { 4611 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 4612 goto out; 4613 } 4614 4615 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 4616 max_eqs = MLX5_GET(cmd_hca_cap, hca_caps, max_num_eqs); 4617 if (max_eqs < MLX5_ESW_MAX_CTRL_EQS) 4618 *max_io_eqs = 0; 4619 else 4620 *max_io_eqs = max_eqs - MLX5_ESW_MAX_CTRL_EQS; 4621out: 4622 mutex_unlock(&esw->state_lock); 4623 kfree(query_ctx); 4624 return err; 4625} 4626 4627int 4628mlx5_devlink_port_fn_max_io_eqs_set(struct devlink_port *port, u32 max_io_eqs, 4629 struct netlink_ext_ack *extack) 4630{ 4631 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4632 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 4633 u16 vport_num = vport->vport; 4634 struct mlx5_eswitch *esw; 4635 void *query_ctx; 4636 void *hca_caps; 4637 u16 max_eqs; 4638 int err; 4639 4640 esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4641 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 4642 NL_SET_ERR_MSG_MOD(extack, 4643 "Device doesn't support VHCA management"); 4644 return -EOPNOTSUPP; 4645 } 4646 4647 if (check_add_overflow(max_io_eqs, MLX5_ESW_MAX_CTRL_EQS, &max_eqs)) { 4648 NL_SET_ERR_MSG_MOD(extack, "Supplied value out of range"); 4649 return -EINVAL; 4650 } 4651 4652 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 4653 if (!query_ctx) 4654 return -ENOMEM; 4655 4656 mutex_lock(&esw->state_lock); 4657 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx, 4658 MLX5_CAP_GENERAL); 4659 if (err) { 4660 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 4661 goto out; 4662 } 4663 4664 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 4665 MLX5_SET(cmd_hca_cap, hca_caps, max_num_eqs, max_eqs); 4666 4667 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport_num, 4668 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); 4669 if (err) 4670 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA caps"); 4671 4672out: 4673 mutex_unlock(&esw->state_lock); 4674 kfree(query_ctx); 4675 return err; 4676} 4677