1/*- 2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD: stable/10/sys/dev/mlx5/mlx5_core/mlx5_flow_table.c 306244 2016-09-23 08:28:44Z hselasky $ 26 */ 27 28#include <dev/mlx5/driver.h> 29#include <dev/mlx5/flow_table.h> 30#include "mlx5_core.h" 31 32struct mlx5_ftg { 33 struct mlx5_flow_table_group g; 34 u32 id; 35 u32 start_ix; 36}; 37 38struct mlx5_flow_table { 39 struct mlx5_core_dev *dev; 40 u8 level; 41 u8 type; 42 u32 id; 43 u16 vport; 44 struct mutex mutex; /* sync bitmap alloc */ 45 u16 num_groups; 46 struct mlx5_ftg *group; 47 unsigned long *bitmap; 48 u32 size; 49}; 50 51static int mlx5_set_flow_entry_cmd(struct mlx5_flow_table *ft, u32 group_ix, 52 u32 flow_index, void *flow_context) 53{ 54 u32 out[MLX5_ST_SZ_DW(set_fte_out)]; 55 u32 *in; 56 void *in_flow_context; 57 int fcdls = 58 MLX5_GET(flow_context, flow_context, destination_list_size) * 59 MLX5_ST_SZ_BYTES(dest_format_struct); 60 int inlen = MLX5_ST_SZ_BYTES(set_fte_in) + fcdls; 61 int err; 62 63 in = mlx5_vzalloc(inlen); 64 if (!in) { 65 mlx5_core_warn(ft->dev, "failed to allocate inbox\n"); 66 return -ENOMEM; 67 } 68 69 MLX5_SET(set_fte_in, in, vport_number, ft->vport); 70 MLX5_SET(set_fte_in, in, other_vport, !!ft->vport); 71 MLX5_SET(set_fte_in, in, table_type, ft->type); 72 MLX5_SET(set_fte_in, in, table_id, ft->id); 73 MLX5_SET(set_fte_in, in, flow_index, flow_index); 74 MLX5_SET(set_fte_in, in, opcode, MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY); 75 76 in_flow_context = MLX5_ADDR_OF(set_fte_in, in, flow_context); 77 memcpy(in_flow_context, flow_context, 78 MLX5_ST_SZ_BYTES(flow_context) + fcdls); 79 80 MLX5_SET(flow_context, in_flow_context, group_id, ft->group[group_ix].id); 81 82 memset(out, 0, sizeof(out)); 83 err = mlx5_cmd_exec_check_status(ft->dev, in, inlen, out, 84 sizeof(out)); 85 kvfree(in); 86 87 return err; 88} 89 90static int mlx5_del_flow_entry_cmd(struct mlx5_flow_table *ft, u32 flow_index) 91{ 92 u32 in[MLX5_ST_SZ_DW(delete_fte_in)]; 93 u32 out[MLX5_ST_SZ_DW(delete_fte_out)]; 94 95 memset(in, 0, sizeof(in)); 96 memset(out, 0, sizeof(out)); 97 98#define MLX5_SET_DFTEI(p, x, v) MLX5_SET(delete_fte_in, p, x, v) 99 MLX5_SET_DFTEI(in, vport_number, ft->vport); 100 MLX5_SET_DFTEI(in, other_vport, !!ft->vport); 101 MLX5_SET_DFTEI(in, table_type, ft->type); 102 MLX5_SET_DFTEI(in, table_id, ft->id); 103 MLX5_SET_DFTEI(in, flow_index, flow_index); 104 MLX5_SET_DFTEI(in, opcode, MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY); 105 106 return mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out, 107 sizeof(out)); 108} 109 110static void mlx5_destroy_flow_group_cmd(struct mlx5_flow_table *ft, int i) 111{ 112 u32 in[MLX5_ST_SZ_DW(destroy_flow_group_in)]; 113 u32 out[MLX5_ST_SZ_DW(destroy_flow_group_out)]; 114 115 memset(in, 0, sizeof(in)); 116 memset(out, 0, sizeof(out)); 117 118#define MLX5_SET_DFGI(p, x, v) MLX5_SET(destroy_flow_group_in, p, x, v) 119 MLX5_SET_DFGI(in, vport_number, ft->vport); 120 MLX5_SET_DFGI(in, other_vport, !!ft->vport); 121 MLX5_SET_DFGI(in, table_type, ft->type); 122 MLX5_SET_DFGI(in, table_id, ft->id); 123 MLX5_SET_DFGI(in, opcode, MLX5_CMD_OP_DESTROY_FLOW_GROUP); 124 MLX5_SET_DFGI(in, group_id, ft->group[i].id); 125 mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out, sizeof(out)); 126} 127 128static int mlx5_create_flow_group_cmd(struct mlx5_flow_table *ft, int i) 129{ 130 u32 out[MLX5_ST_SZ_DW(create_flow_group_out)]; 131 u32 *in; 132 void *in_match_criteria; 133 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 134 struct mlx5_flow_table_group *g = &ft->group[i].g; 135 u32 start_ix = ft->group[i].start_ix; 136 u32 end_ix = start_ix + (1 << g->log_sz) - 1; 137 int err; 138 139 in = mlx5_vzalloc(inlen); 140 if (!in) { 141 mlx5_core_warn(ft->dev, "failed to allocate inbox\n"); 142 return -ENOMEM; 143 } 144 in_match_criteria = MLX5_ADDR_OF(create_flow_group_in, in, 145 match_criteria); 146 147 memset(out, 0, sizeof(out)); 148 149#define MLX5_SET_CFGI(p, x, v) MLX5_SET(create_flow_group_in, p, x, v) 150 MLX5_SET_CFGI(in, vport_number, ft->vport); 151 MLX5_SET_CFGI(in, other_vport, !!ft->vport); 152 MLX5_SET_CFGI(in, table_type, ft->type); 153 MLX5_SET_CFGI(in, table_id, ft->id); 154 MLX5_SET_CFGI(in, opcode, MLX5_CMD_OP_CREATE_FLOW_GROUP); 155 MLX5_SET_CFGI(in, start_flow_index, start_ix); 156 MLX5_SET_CFGI(in, end_flow_index, end_ix); 157 MLX5_SET_CFGI(in, match_criteria_enable, g->match_criteria_enable); 158 159 memcpy(in_match_criteria, g->match_criteria, 160 MLX5_ST_SZ_BYTES(fte_match_param)); 161 162 err = mlx5_cmd_exec_check_status(ft->dev, in, inlen, out, 163 sizeof(out)); 164 if (!err) 165 ft->group[i].id = MLX5_GET(create_flow_group_out, out, 166 group_id); 167 168 kvfree(in); 169 170 return err; 171} 172 173static void mlx5_destroy_flow_table_groups(struct mlx5_flow_table *ft) 174{ 175 int i; 176 177 for (i = 0; i < ft->num_groups; i++) 178 mlx5_destroy_flow_group_cmd(ft, i); 179} 180 181static int mlx5_create_flow_table_groups(struct mlx5_flow_table *ft) 182{ 183 int err; 184 int i; 185 186 for (i = 0; i < ft->num_groups; i++) { 187 err = mlx5_create_flow_group_cmd(ft, i); 188 if (err) 189 goto err_destroy_flow_table_groups; 190 } 191 192 return 0; 193 194err_destroy_flow_table_groups: 195 for (i--; i >= 0; i--) 196 mlx5_destroy_flow_group_cmd(ft, i); 197 198 return err; 199} 200 201static int mlx5_create_flow_table_cmd(struct mlx5_flow_table *ft) 202{ 203 u32 in[MLX5_ST_SZ_DW(create_flow_table_in)]; 204 u32 out[MLX5_ST_SZ_DW(create_flow_table_out)]; 205 int err; 206 207 memset(in, 0, sizeof(in)); 208 209 MLX5_SET(create_flow_table_in, in, vport_number, ft->vport); 210 MLX5_SET(create_flow_table_in, in, other_vport, !!ft->vport); 211 MLX5_SET(create_flow_table_in, in, table_type, ft->type); 212 MLX5_SET(create_flow_table_in, in, level, ft->level); 213 MLX5_SET(create_flow_table_in, in, log_size, order_base_2(ft->size)); 214 215 MLX5_SET(create_flow_table_in, in, opcode, 216 MLX5_CMD_OP_CREATE_FLOW_TABLE); 217 218 memset(out, 0, sizeof(out)); 219 err = mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out, 220 sizeof(out)); 221 if (err) 222 return err; 223 224 ft->id = MLX5_GET(create_flow_table_out, out, table_id); 225 226 return 0; 227} 228 229static void mlx5_destroy_flow_table_cmd(struct mlx5_flow_table *ft) 230{ 231 u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)]; 232 u32 out[MLX5_ST_SZ_DW(destroy_flow_table_out)]; 233 234 memset(in, 0, sizeof(in)); 235 memset(out, 0, sizeof(out)); 236 237#define MLX5_SET_DFTI(p, x, v) MLX5_SET(destroy_flow_table_in, p, x, v) 238 MLX5_SET_DFTI(in, vport_number, ft->vport); 239 MLX5_SET_DFTI(in, other_vport, !!ft->vport); 240 MLX5_SET_DFTI(in, table_type, ft->type); 241 MLX5_SET_DFTI(in, table_id, ft->id); 242 MLX5_SET_DFTI(in, opcode, MLX5_CMD_OP_DESTROY_FLOW_TABLE); 243 244 mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out, sizeof(out)); 245} 246 247static int mlx5_find_group(struct mlx5_flow_table *ft, u8 match_criteria_enable, 248 u32 *match_criteria, int *group_ix) 249{ 250 void *mc_outer = MLX5_ADDR_OF(fte_match_param, match_criteria, 251 outer_headers); 252 void *mc_misc = MLX5_ADDR_OF(fte_match_param, match_criteria, 253 misc_parameters); 254 void *mc_inner = MLX5_ADDR_OF(fte_match_param, match_criteria, 255 inner_headers); 256 int mc_outer_sz = MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4); 257 int mc_misc_sz = MLX5_ST_SZ_BYTES(fte_match_set_misc); 258 int mc_inner_sz = MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4); 259 int i; 260 261 for (i = 0; i < ft->num_groups; i++) { 262 struct mlx5_flow_table_group *g = &ft->group[i].g; 263 void *gmc_outer = MLX5_ADDR_OF(fte_match_param, 264 g->match_criteria, 265 outer_headers); 266 void *gmc_misc = MLX5_ADDR_OF(fte_match_param, 267 g->match_criteria, 268 misc_parameters); 269 void *gmc_inner = MLX5_ADDR_OF(fte_match_param, 270 g->match_criteria, 271 inner_headers); 272 273 if (g->match_criteria_enable != match_criteria_enable) 274 continue; 275 276 if (match_criteria_enable & MLX5_MATCH_OUTER_HEADERS) 277 if (memcmp(mc_outer, gmc_outer, mc_outer_sz)) 278 continue; 279 280 if (match_criteria_enable & MLX5_MATCH_MISC_PARAMETERS) 281 if (memcmp(mc_misc, gmc_misc, mc_misc_sz)) 282 continue; 283 284 if (match_criteria_enable & MLX5_MATCH_INNER_HEADERS) 285 if (memcmp(mc_inner, gmc_inner, mc_inner_sz)) 286 continue; 287 288 *group_ix = i; 289 return 0; 290 } 291 292 return -EINVAL; 293} 294 295static int alloc_flow_index(struct mlx5_flow_table *ft, int group_ix, u32 *ix) 296{ 297 struct mlx5_ftg *g = &ft->group[group_ix]; 298 int err = 0; 299 300 mutex_lock(&ft->mutex); 301 302 *ix = find_next_zero_bit(ft->bitmap, ft->size, g->start_ix); 303 if (*ix >= (g->start_ix + (1 << g->g.log_sz))) 304 err = -ENOSPC; 305 else 306 __set_bit(*ix, ft->bitmap); 307 308 mutex_unlock(&ft->mutex); 309 310 return err; 311} 312 313static void mlx5_free_flow_index(struct mlx5_flow_table *ft, u32 ix) 314{ 315 __clear_bit(ix, ft->bitmap); 316} 317 318int mlx5_add_flow_table_entry(void *flow_table, u8 match_criteria_enable, 319 void *match_criteria, void *flow_context, 320 u32 *flow_index) 321{ 322 struct mlx5_flow_table *ft = flow_table; 323 int group_ix; 324 int err; 325 326 err = mlx5_find_group(ft, match_criteria_enable, match_criteria, 327 &group_ix); 328 if (err) { 329 mlx5_core_warn(ft->dev, "mlx5_find_group failed\n"); 330 return err; 331 } 332 333 err = alloc_flow_index(ft, group_ix, flow_index); 334 if (err) { 335 mlx5_core_warn(ft->dev, "alloc_flow_index failed\n"); 336 return err; 337 } 338 339 err = mlx5_set_flow_entry_cmd(ft, group_ix, *flow_index, flow_context); 340 if (err) 341 mlx5_free_flow_index(ft, *flow_index); 342 343 return err; 344} 345EXPORT_SYMBOL(mlx5_add_flow_table_entry); 346 347int mlx5_del_flow_table_entry(void *flow_table, u32 flow_index) 348{ 349 struct mlx5_flow_table *ft = flow_table; 350 int ret; 351 352 ret = mlx5_del_flow_entry_cmd(ft, flow_index); 353 if (!ret) 354 mlx5_free_flow_index(ft, flow_index); 355 return ret; 356} 357EXPORT_SYMBOL(mlx5_del_flow_table_entry); 358 359void *mlx5_create_flow_table(struct mlx5_core_dev *dev, u8 level, u8 table_type, 360 u16 vport, 361 u16 num_groups, 362 struct mlx5_flow_table_group *group) 363{ 364 struct mlx5_flow_table *ft; 365 u32 start_ix = 0; 366 u32 ft_size = 0; 367 void *gr; 368 void *bm; 369 int err; 370 int i; 371 372 for (i = 0; i < num_groups; i++) 373 ft_size += (1 << group[i].log_sz); 374 375 ft = kzalloc(sizeof(*ft), GFP_KERNEL); 376 gr = kcalloc(num_groups, sizeof(struct mlx5_ftg), GFP_KERNEL); 377 bm = kcalloc(BITS_TO_LONGS(ft_size), sizeof(uintptr_t), GFP_KERNEL); 378 379 ft->group = gr; 380 ft->bitmap = bm; 381 ft->num_groups = num_groups; 382 ft->level = level; 383 ft->vport = vport; 384 ft->type = table_type; 385 ft->size = ft_size; 386 ft->dev = dev; 387 mutex_init(&ft->mutex); 388 389 for (i = 0; i < ft->num_groups; i++) { 390 memcpy(&ft->group[i].g, &group[i], sizeof(*group)); 391 ft->group[i].start_ix = start_ix; 392 start_ix += 1 << group[i].log_sz; 393 } 394 395 err = mlx5_create_flow_table_cmd(ft); 396 if (err) 397 goto err_free_ft; 398 399 err = mlx5_create_flow_table_groups(ft); 400 if (err) 401 goto err_destroy_flow_table_cmd; 402 403 return ft; 404 405err_destroy_flow_table_cmd: 406 mlx5_destroy_flow_table_cmd(ft); 407 408err_free_ft: 409 mlx5_core_warn(dev, "failed to alloc flow table\n"); 410 kfree(bm); 411 kfree(gr); 412 kfree(ft); 413 414 return NULL; 415} 416EXPORT_SYMBOL(mlx5_create_flow_table); 417 418void mlx5_destroy_flow_table(void *flow_table) 419{ 420 struct mlx5_flow_table *ft = flow_table; 421 422 mlx5_destroy_flow_table_groups(ft); 423 mlx5_destroy_flow_table_cmd(ft); 424 kfree(ft->bitmap); 425 kfree(ft->group); 426 kfree(ft); 427} 428EXPORT_SYMBOL(mlx5_destroy_flow_table); 429 430u32 mlx5_get_flow_table_id(void *flow_table) 431{ 432 struct mlx5_flow_table *ft = flow_table; 433 434 return ft->id; 435} 436EXPORT_SYMBOL(mlx5_get_flow_table_id); 437 438int mlx5_set_flow_table_root(struct mlx5_core_dev *mdev, u16 op_mod, 439 u8 vport_num, u8 table_type, u32 table_id, 440 u32 underlay_qpn) 441{ 442 u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)]; 443 u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)]; 444 int err; 445 int is_group_manager; 446 447 is_group_manager = MLX5_CAP_GEN(mdev, vport_group_manager); 448 449 memset(in, 0, sizeof(in)); 450 451 MLX5_SET(set_flow_table_root_in, in, op_mod, op_mod); 452 MLX5_SET(set_flow_table_root_in, in, table_type, table_type); 453 MLX5_SET(set_flow_table_root_in, in, underlay_qpn, underlay_qpn); 454 if (op_mod == MLX5_SET_FLOW_TABLE_ROOT_OPMOD_SET) 455 MLX5_SET(set_flow_table_root_in, in, table_id, table_id); 456 457 MLX5_SET(set_flow_table_root_in, in, opcode, 458 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT); 459 460 if (vport_num) { 461 if (is_group_manager) { 462 MLX5_SET(set_flow_table_root_in, in, other_vport, 463 1); 464 MLX5_SET(set_flow_table_root_in, in, vport_number, 465 vport_num); 466 } else { 467 return -EPERM; 468 } 469 } 470 471 memset(out, 0, sizeof(out)); 472 err = mlx5_cmd_exec_check_status(mdev, in, sizeof(in), out, 473 sizeof(out)); 474 if (err) 475 return err; 476 477 return 0; 478} 479EXPORT_SYMBOL(mlx5_set_flow_table_root); 480