1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2/* Copyright (c) 2021 Mellanox Technologies. */ 3 4#include <linux/etherdevice.h> 5#include <linux/idr.h> 6#include <linux/mlx5/driver.h> 7#include <linux/mlx5/mlx5_ifc.h> 8#include <linux/mlx5/vport.h> 9#include <linux/mlx5/fs.h> 10#include "mlx5_core.h" 11#include "eswitch.h" 12#include "en.h" 13#include "en_tc.h" 14#include "fs_core.h" 15#include "esw/indir_table.h" 16#include "lib/fs_chains.h" 17#include "en/mod_hdr.h" 18 19#define MLX5_ESW_INDIR_TABLE_SIZE 2 20#define MLX5_ESW_INDIR_TABLE_RECIRC_IDX (MLX5_ESW_INDIR_TABLE_SIZE - 2) 21#define MLX5_ESW_INDIR_TABLE_FWD_IDX (MLX5_ESW_INDIR_TABLE_SIZE - 1) 22 23struct mlx5_esw_indir_table_rule { 24 struct mlx5_flow_handle *handle; 25 struct mlx5_modify_hdr *mh; 26 refcount_t refcnt; 27}; 28 29struct mlx5_esw_indir_table_entry { 30 struct hlist_node hlist; 31 struct mlx5_flow_table *ft; 32 struct mlx5_flow_group *recirc_grp; 33 struct mlx5_flow_group *fwd_grp; 34 struct mlx5_flow_handle *fwd_rule; 35 struct mlx5_esw_indir_table_rule *recirc_rule; 36 int fwd_ref; 37 38 u16 vport; 39}; 40 41struct mlx5_esw_indir_table { 42 struct mutex lock; /* protects table */ 43 DECLARE_HASHTABLE(table, 8); 44}; 45 46struct mlx5_esw_indir_table * 47mlx5_esw_indir_table_init(void) 48{ 49 struct mlx5_esw_indir_table *indir = kvzalloc(sizeof(*indir), GFP_KERNEL); 50 51 if (!indir) 52 return ERR_PTR(-ENOMEM); 53 54 mutex_init(&indir->lock); 55 hash_init(indir->table); 56 return indir; 57} 58 59void 60mlx5_esw_indir_table_destroy(struct mlx5_esw_indir_table *indir) 61{ 62 mutex_destroy(&indir->lock); 63 kvfree(indir); 64} 65 66bool 67mlx5_esw_indir_table_needed(struct mlx5_eswitch *esw, 68 struct mlx5_flow_attr *attr, 69 u16 vport_num, 70 struct mlx5_core_dev *dest_mdev) 71{ 72 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 73 bool vf_sf_vport; 74 75 vf_sf_vport = mlx5_eswitch_is_vf_vport(esw, vport_num) || 76 mlx5_esw_is_sf_vport(esw, vport_num); 77 78 /* Use indirect table for all IP traffic from UL to VF with vport 79 * destination when source rewrite flag is set. 80 */ 81 return esw_attr->in_rep->vport == MLX5_VPORT_UPLINK && 82 vf_sf_vport && 83 esw->dev == dest_mdev && 84 attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE; 85} 86 87u16 88mlx5_esw_indir_table_decap_vport(struct mlx5_flow_attr *attr) 89{ 90 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 91 92 return esw_attr->rx_tun_attr ? esw_attr->rx_tun_attr->decap_vport : 0; 93} 94 95static int mlx5_esw_indir_table_rule_get(struct mlx5_eswitch *esw, 96 struct mlx5_flow_attr *attr, 97 struct mlx5_esw_indir_table_entry *e) 98{ 99 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 100 struct mlx5_fs_chains *chains = esw_chains(esw); 101 struct mlx5e_tc_mod_hdr_acts mod_acts = {}; 102 struct mlx5_flow_destination dest = {}; 103 struct mlx5_esw_indir_table_rule *rule; 104 struct mlx5_flow_act flow_act = {}; 105 struct mlx5_flow_handle *handle; 106 int err = 0; 107 u32 data; 108 109 if (e->recirc_rule) { 110 refcount_inc(&e->recirc_rule->refcnt); 111 return 0; 112 } 113 114 rule = kzalloc(sizeof(*rule), GFP_KERNEL); 115 if (!rule) 116 return -ENOMEM; 117 118 /* Modify flow source to recirculate packet */ 119 data = mlx5_eswitch_get_vport_metadata_for_set(esw, esw_attr->rx_tun_attr->decap_vport); 120 err = mlx5e_tc_match_to_reg_set(esw->dev, &mod_acts, MLX5_FLOW_NAMESPACE_FDB, 121 VPORT_TO_REG, data); 122 if (err) 123 goto err_mod_hdr_regc0; 124 125 err = mlx5e_tc_match_to_reg_set(esw->dev, &mod_acts, MLX5_FLOW_NAMESPACE_FDB, 126 TUNNEL_TO_REG, ESW_TUN_SLOW_TABLE_GOTO_VPORT); 127 if (err) 128 goto err_mod_hdr_regc1; 129 130 flow_act.modify_hdr = mlx5_modify_header_alloc(esw->dev, MLX5_FLOW_NAMESPACE_FDB, 131 mod_acts.num_actions, mod_acts.actions); 132 if (IS_ERR(flow_act.modify_hdr)) { 133 err = PTR_ERR(flow_act.modify_hdr); 134 goto err_mod_hdr_alloc; 135 } 136 137 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 138 flow_act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL | FLOW_ACT_NO_APPEND; 139 flow_act.fg = e->recirc_grp; 140 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 141 dest.ft = mlx5_chains_get_table(chains, 0, 1, 0); 142 if (IS_ERR(dest.ft)) { 143 err = PTR_ERR(dest.ft); 144 goto err_table; 145 } 146 handle = mlx5_add_flow_rules(e->ft, NULL, &flow_act, &dest, 1); 147 if (IS_ERR(handle)) { 148 err = PTR_ERR(handle); 149 goto err_handle; 150 } 151 152 mlx5e_mod_hdr_dealloc(&mod_acts); 153 rule->handle = handle; 154 rule->mh = flow_act.modify_hdr; 155 refcount_set(&rule->refcnt, 1); 156 e->recirc_rule = rule; 157 return 0; 158 159err_handle: 160 mlx5_chains_put_table(chains, 0, 1, 0); 161err_table: 162 mlx5_modify_header_dealloc(esw->dev, flow_act.modify_hdr); 163err_mod_hdr_alloc: 164err_mod_hdr_regc1: 165 mlx5e_mod_hdr_dealloc(&mod_acts); 166err_mod_hdr_regc0: 167 kfree(rule); 168 return err; 169} 170 171static void mlx5_esw_indir_table_rule_put(struct mlx5_eswitch *esw, 172 struct mlx5_esw_indir_table_entry *e) 173{ 174 struct mlx5_esw_indir_table_rule *rule = e->recirc_rule; 175 struct mlx5_fs_chains *chains = esw_chains(esw); 176 177 if (!rule) 178 return; 179 180 if (!refcount_dec_and_test(&rule->refcnt)) 181 return; 182 183 mlx5_del_flow_rules(rule->handle); 184 mlx5_chains_put_table(chains, 0, 1, 0); 185 mlx5_modify_header_dealloc(esw->dev, rule->mh); 186 kfree(rule); 187 e->recirc_rule = NULL; 188} 189 190static int mlx5_create_indir_recirc_group(struct mlx5_esw_indir_table_entry *e) 191{ 192 int err = 0, inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 193 u32 *in; 194 195 in = kvzalloc(inlen, GFP_KERNEL); 196 if (!in) 197 return -ENOMEM; 198 199 MLX5_SET(create_flow_group_in, in, start_flow_index, 0); 200 MLX5_SET(create_flow_group_in, in, end_flow_index, MLX5_ESW_INDIR_TABLE_RECIRC_IDX); 201 e->recirc_grp = mlx5_create_flow_group(e->ft, in); 202 if (IS_ERR(e->recirc_grp)) 203 err = PTR_ERR(e->recirc_grp); 204 205 kvfree(in); 206 return err; 207} 208 209static int mlx5_create_indir_fwd_group(struct mlx5_eswitch *esw, 210 struct mlx5_esw_indir_table_entry *e) 211{ 212 int err = 0, inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 213 struct mlx5_flow_destination dest = {}; 214 struct mlx5_flow_act flow_act = {}; 215 u32 *in; 216 217 in = kvzalloc(inlen, GFP_KERNEL); 218 if (!in) 219 return -ENOMEM; 220 221 /* Hold one entry */ 222 MLX5_SET(create_flow_group_in, in, start_flow_index, MLX5_ESW_INDIR_TABLE_FWD_IDX); 223 MLX5_SET(create_flow_group_in, in, end_flow_index, MLX5_ESW_INDIR_TABLE_FWD_IDX); 224 e->fwd_grp = mlx5_create_flow_group(e->ft, in); 225 if (IS_ERR(e->fwd_grp)) { 226 err = PTR_ERR(e->fwd_grp); 227 goto err_out; 228 } 229 230 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 231 flow_act.fg = e->fwd_grp; 232 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 233 dest.vport.num = e->vport; 234 dest.vport.vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id); 235 dest.vport.flags = MLX5_FLOW_DEST_VPORT_VHCA_ID; 236 e->fwd_rule = mlx5_add_flow_rules(e->ft, NULL, &flow_act, &dest, 1); 237 if (IS_ERR(e->fwd_rule)) { 238 mlx5_destroy_flow_group(e->fwd_grp); 239 err = PTR_ERR(e->fwd_rule); 240 } 241 242err_out: 243 kvfree(in); 244 return err; 245} 246 247static struct mlx5_esw_indir_table_entry * 248mlx5_esw_indir_table_entry_create(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr, 249 u16 vport, bool decap) 250{ 251 struct mlx5_flow_table_attr ft_attr = {}; 252 struct mlx5_flow_namespace *root_ns; 253 struct mlx5_esw_indir_table_entry *e; 254 struct mlx5_flow_table *ft; 255 int err = 0; 256 257 root_ns = mlx5_get_flow_namespace(esw->dev, MLX5_FLOW_NAMESPACE_FDB); 258 if (!root_ns) 259 return ERR_PTR(-ENOENT); 260 261 e = kzalloc(sizeof(*e), GFP_KERNEL); 262 if (!e) 263 return ERR_PTR(-ENOMEM); 264 265 ft_attr.prio = FDB_TC_OFFLOAD; 266 ft_attr.max_fte = MLX5_ESW_INDIR_TABLE_SIZE; 267 ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED; 268 ft_attr.level = 1; 269 270 ft = mlx5_create_flow_table(root_ns, &ft_attr); 271 if (IS_ERR(ft)) { 272 err = PTR_ERR(ft); 273 goto tbl_err; 274 } 275 e->ft = ft; 276 e->vport = vport; 277 e->fwd_ref = !decap; 278 279 err = mlx5_create_indir_recirc_group(e); 280 if (err) 281 goto recirc_grp_err; 282 283 if (decap) { 284 err = mlx5_esw_indir_table_rule_get(esw, attr, e); 285 if (err) 286 goto recirc_rule_err; 287 } 288 289 err = mlx5_create_indir_fwd_group(esw, e); 290 if (err) 291 goto fwd_grp_err; 292 293 hash_add(esw->fdb_table.offloads.indir->table, &e->hlist, 294 vport << 16); 295 296 return e; 297 298fwd_grp_err: 299 if (decap) 300 mlx5_esw_indir_table_rule_put(esw, e); 301recirc_rule_err: 302 mlx5_destroy_flow_group(e->recirc_grp); 303recirc_grp_err: 304 mlx5_destroy_flow_table(e->ft); 305tbl_err: 306 kfree(e); 307 return ERR_PTR(err); 308} 309 310static struct mlx5_esw_indir_table_entry * 311mlx5_esw_indir_table_entry_lookup(struct mlx5_eswitch *esw, u16 vport) 312{ 313 struct mlx5_esw_indir_table_entry *e; 314 u32 key = vport << 16; 315 316 hash_for_each_possible(esw->fdb_table.offloads.indir->table, e, hlist, key) 317 if (e->vport == vport) 318 return e; 319 320 return NULL; 321} 322 323struct mlx5_flow_table *mlx5_esw_indir_table_get(struct mlx5_eswitch *esw, 324 struct mlx5_flow_attr *attr, 325 u16 vport, bool decap) 326{ 327 struct mlx5_esw_indir_table_entry *e; 328 int err; 329 330 mutex_lock(&esw->fdb_table.offloads.indir->lock); 331 e = mlx5_esw_indir_table_entry_lookup(esw, vport); 332 if (e) { 333 if (!decap) { 334 e->fwd_ref++; 335 } else { 336 err = mlx5_esw_indir_table_rule_get(esw, attr, e); 337 if (err) 338 goto out_err; 339 } 340 } else { 341 e = mlx5_esw_indir_table_entry_create(esw, attr, vport, decap); 342 if (IS_ERR(e)) { 343 err = PTR_ERR(e); 344 esw_warn(esw->dev, "Failed to create indirection table, err %d.\n", err); 345 goto out_err; 346 } 347 } 348 mutex_unlock(&esw->fdb_table.offloads.indir->lock); 349 return e->ft; 350 351out_err: 352 mutex_unlock(&esw->fdb_table.offloads.indir->lock); 353 return ERR_PTR(err); 354} 355 356void mlx5_esw_indir_table_put(struct mlx5_eswitch *esw, 357 u16 vport, bool decap) 358{ 359 struct mlx5_esw_indir_table_entry *e; 360 361 mutex_lock(&esw->fdb_table.offloads.indir->lock); 362 e = mlx5_esw_indir_table_entry_lookup(esw, vport); 363 if (!e) 364 goto out; 365 366 if (!decap) 367 e->fwd_ref--; 368 else 369 mlx5_esw_indir_table_rule_put(esw, e); 370 371 if (e->fwd_ref || e->recirc_rule) 372 goto out; 373 374 hash_del(&e->hlist); 375 mlx5_destroy_flow_group(e->recirc_grp); 376 mlx5_del_flow_rules(e->fwd_rule); 377 mlx5_destroy_flow_group(e->fwd_grp); 378 mlx5_destroy_flow_table(e->ft); 379 kfree(e); 380out: 381 mutex_unlock(&esw->fdb_table.offloads.indir->lock); 382} 383