1/*- 2 * Copyright (c) 2020-2021, Mellanox Technologies, Ltd. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26#include "opt_inet.h" 27#include "opt_inet6.h" 28 29#include <dev/mlx5/mlx5_en/en.h> 30 31#include <dev/mlx5/mlx5_core/fs_core.h> 32#include <dev/mlx5/mlx5_core/fs_tcp.h> 33#include <dev/mlx5/device.h> 34 35#include <sys/domain.h> 36 37#include <netinet/in_pcb.h> 38 39#if defined(INET) || defined(INET6) 40static void 41accel_fs_tcp_set_ipv4_flow(struct mlx5_flow_spec *spec, struct inpcb *inp) 42{ 43 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); 44 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP); 45 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); 46 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4); 47 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 48 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), 49 &inp->inp_faddr, 4); 50 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 51 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 52 &inp->inp_laddr, 4); 53 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 54 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); 55 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 56 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 57} 58#endif 59 60#ifdef INET6 61static void 62accel_fs_tcp_set_ipv6_flow(struct mlx5_flow_spec *spec, struct inpcb *inp) 63{ 64 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); 65 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP); 66 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); 67 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6); 68 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 69 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 70 &inp->in6p_faddr, 16); 71 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 72 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 73 &inp->in6p_laddr, 16); 74 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 75 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 76 0xff, 16); 77 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 78 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 79 0xff, 16); 80} 81#endif 82 83void 84mlx5e_accel_fs_del_inpcb(struct mlx5_flow_rule *rule) 85{ 86 mlx5_del_flow_rule(&rule); 87} 88 89struct mlx5_flow_rule * 90mlx5e_accel_fs_add_inpcb(struct mlx5e_priv *priv, 91 struct inpcb *inp, uint32_t tirn, uint32_t flow_tag, 92 uint16_t vlan_id) 93{ 94 struct mlx5_flow_destination dest = {}; 95 struct mlx5e_flow_table *ft = NULL; 96#if defined(INET) || defined(INET6) 97 struct mlx5e_accel_fs_tcp *fs_tcp = &priv->fts.accel_tcp; 98#endif 99 struct mlx5_flow_rule *flow; 100 struct mlx5_flow_spec *spec; 101 struct mlx5_flow_act flow_act = { 102 .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG, 103 .flow_tag = flow_tag, 104 }; 105 106 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 107 if (!spec) 108 return (ERR_PTR(-ENOMEM)); 109 110 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 111 112 INP_RLOCK(inp); 113 /* Set VLAN ID to match, if any. */ 114 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.cvlan_tag); 115 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.first_vid); 116 if (vlan_id != MLX5E_ACCEL_FS_ADD_INPCB_NO_VLAN) { 117 MLX5_SET_TO_ONES(fte_match_param, spec->match_value, outer_headers.cvlan_tag); 118 MLX5_SET(fte_match_param, spec->match_value, outer_headers.first_vid, vlan_id); 119 } 120 121 /* Set TCP port numbers. */ 122 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 123 outer_headers.tcp_dport); 124 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 125 outer_headers.tcp_sport); 126 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport, 127 ntohs(inp->inp_lport)); 128 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport, 129 ntohs(inp->inp_fport)); 130 131 /* Set IP addresses. */ 132 switch (INP_SOCKAF(inp->inp_socket)) { 133#ifdef INET 134 case AF_INET: 135 accel_fs_tcp_set_ipv4_flow(spec, inp); 136 ft = &fs_tcp->tables[MLX5E_ACCEL_FS_IPV4_TCP]; 137 break; 138#endif 139#ifdef INET6 140 case AF_INET6: 141 if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0 && 142 IN6_IS_ADDR_V4MAPPED(&inp->in6p_faddr)) { 143 accel_fs_tcp_set_ipv4_flow(spec, inp); 144 ft = &fs_tcp->tables[MLX5E_ACCEL_FS_IPV4_TCP]; 145 } else { 146 accel_fs_tcp_set_ipv6_flow(spec, inp); 147 ft = &fs_tcp->tables[MLX5E_ACCEL_FS_IPV6_TCP]; 148 } 149 break; 150#endif 151 default: 152 break; 153 } 154 INP_RUNLOCK(inp); 155 156 if (!ft) { 157 flow = ERR_PTR(-EINVAL); 158 goto out; 159 } 160 161 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 162 dest.tir_num = tirn; 163 164 flow = mlx5_add_flow_rule(ft->t, spec->match_criteria_enable, 165 spec->match_criteria, 166 spec->match_value, 167 MLX5_FLOW_RULE_FWD_ACTION_DEST, 168 &flow_act, 169 &dest); 170out: 171 kvfree(spec); 172 return (flow); 173} 174 175static int 176accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv, int type) 177{ 178 static u32 match_criteria[MLX5_ST_SZ_DW(fte_match_param)]; 179 static u32 match_value[MLX5_ST_SZ_DW(fte_match_param)]; 180 struct mlx5_flow_destination dest = {}; 181 struct mlx5e_accel_fs_tcp *fs_tcp; 182 struct mlx5_flow_rule *rule; 183 struct mlx5_flow_act flow_act = { 184 .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG, 185 .flow_tag = MLX5_FS_DEFAULT_FLOW_TAG, 186 }; 187 188 fs_tcp = &priv->fts.accel_tcp; 189 190 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 191 192 /* 193 * Traffic not matched by flow table rules should be forwarded 194 * to the next flow table in order to not be dropped by the 195 * default action. Refer to the diagram in 196 * mlx5_en_flow_table.c for more information about the order 197 * of flow tables. 198 */ 199 dest.ft = (type == MLX5E_ACCEL_FS_TCP_NUM_TYPES - 1) ? 200 priv->fts.vlan.t : fs_tcp->tables[type + 1].t; 201 202 rule = mlx5_add_flow_rule(fs_tcp->tables[type].t, 0, match_criteria, match_value, 203 MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest); 204 if (IS_ERR(rule)) 205 return (PTR_ERR(rule)); 206 207 fs_tcp->default_rules[type] = rule; 208 return (0); 209} 210 211#define MLX5E_ACCEL_FS_TCP_NUM_GROUPS (2) 212#define MLX5E_ACCEL_FS_TCP_GROUP1_SIZE (BIT(16) - 1) 213#define MLX5E_ACCEL_FS_TCP_GROUP2_SIZE (BIT(0)) 214#define MLX5E_ACCEL_FS_TCP_TABLE_SIZE (MLX5E_ACCEL_FS_TCP_GROUP1_SIZE +\ 215 MLX5E_ACCEL_FS_TCP_GROUP2_SIZE) 216static int 217accel_fs_tcp_create_groups(struct mlx5e_flow_table *ft, int type) 218{ 219 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 220 void *outer_headers_c; 221 int ix = 0; 222 u32 *in; 223 int err; 224 u8 *mc; 225 226 ft->g = kcalloc(MLX5E_ACCEL_FS_TCP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL); 227 in = kvzalloc(inlen, GFP_KERNEL); 228 if (!in || !ft->g) { 229 kfree(ft->g); 230 kvfree(in); 231 return (-ENOMEM); 232 } 233 234 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 235 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers); 236 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol); 237 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_version); 238 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, cvlan_tag); 239 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, first_vid); 240 241 switch (type) { 242 case MLX5E_ACCEL_FS_IPV4_TCP: 243 case MLX5E_ACCEL_FS_IPV6_TCP: 244 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport); 245 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport); 246 break; 247 default: 248 err = -EINVAL; 249 goto out; 250 } 251 252 switch (type) { 253 case MLX5E_ACCEL_FS_IPV4_TCP: 254 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, 255 src_ipv4_src_ipv6.ipv4_layout.ipv4); 256 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, 257 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 258 break; 259 case MLX5E_ACCEL_FS_IPV6_TCP: 260 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, 261 src_ipv4_src_ipv6.ipv6_layout.ipv6), 262 0xff, 16); 263 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, 264 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 265 0xff, 16); 266 break; 267 default: 268 err = -EINVAL; 269 goto out; 270 } 271 272 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 273 MLX5_SET_CFG(in, start_flow_index, ix); 274 ix += MLX5E_ACCEL_FS_TCP_GROUP1_SIZE; 275 MLX5_SET_CFG(in, end_flow_index, ix - 1); 276 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 277 if (IS_ERR(ft->g[ft->num_groups])) 278 goto err; 279 ft->num_groups++; 280 281 /* Default Flow Group */ 282 memset(in, 0, inlen); 283 MLX5_SET_CFG(in, start_flow_index, ix); 284 ix += MLX5E_ACCEL_FS_TCP_GROUP2_SIZE; 285 MLX5_SET_CFG(in, end_flow_index, ix - 1); 286 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 287 if (IS_ERR(ft->g[ft->num_groups])) 288 goto err; 289 ft->num_groups++; 290 291 kvfree(in); 292 return (0); 293 294err: 295 err = PTR_ERR(ft->g[ft->num_groups]); 296 ft->g[ft->num_groups] = NULL; 297out: 298 kvfree(in); 299 300 return (err); 301} 302 303static void 304accel_fs_tcp_destroy_groups(struct mlx5e_flow_table *ft) 305{ 306 int i; 307 308 for (i = ft->num_groups - 1; i >= 0; i--) { 309 if (!IS_ERR_OR_NULL(ft->g[i])) 310 mlx5_destroy_flow_group(ft->g[i]); 311 ft->g[i] = NULL; 312 } 313 ft->num_groups = 0; 314} 315 316static int 317accel_fs_tcp_create_table(struct mlx5e_priv *priv, int type) 318{ 319 struct mlx5e_flow_table *ft = &priv->fts.accel_tcp.tables[type]; 320 int err; 321 322 ft->num_groups = 0; 323 ft->t = mlx5_create_flow_table(priv->fts.accel_tcp.ns, 0, "tcp", 324 MLX5E_ACCEL_FS_TCP_TABLE_SIZE); 325 if (IS_ERR(ft->t)) { 326 err = PTR_ERR(ft->t); 327 ft->t = NULL; 328 return (err); 329 } 330 331 err = accel_fs_tcp_create_groups(ft, type); 332 if (err) 333 goto err_destroy_flow_table; 334 335 return (0); 336 337err_destroy_flow_table: 338 mlx5_destroy_flow_table(ft->t); 339 ft->t = NULL; 340 return (err); 341} 342 343static void 344accel_fs_tcp_destroy_table(struct mlx5e_priv *priv, int i) 345{ 346 struct mlx5e_accel_fs_tcp *fs_tcp; 347 struct mlx5e_flow_table *ft; 348 349 fs_tcp = &priv->fts.accel_tcp; 350 ft = fs_tcp->tables + i; 351 352 accel_fs_tcp_destroy_groups(ft); 353 kfree(ft->g); 354 ft->g = NULL; 355 mlx5_destroy_flow_table(ft->t); 356 ft->t = NULL; 357} 358 359void 360mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv) 361{ 362 int i; 363 364 if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version)) 365 return; 366 367 for (i = 0; i < MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) { 368 mlx5_del_flow_rule(&priv->fts.accel_tcp.default_rules[i]); 369 accel_fs_tcp_destroy_table(priv, i); 370 } 371} 372 373int 374mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv) 375{ 376 int i, err; 377 378 if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version)) 379 return (0); 380 381 /* Setup namespace pointer. */ 382 priv->fts.accel_tcp.ns = mlx5_get_flow_namespace( 383 priv->mdev, MLX5_FLOW_NAMESPACE_OFFLOADS); 384 385 /* 386 * Create flow tables first, because the priority level is 387 * assigned at allocation time. 388 */ 389 for (i = 0; i != MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) { 390 err = accel_fs_tcp_create_table(priv, i); 391 if (err) 392 goto err_destroy_tables; 393 } 394 395 /* Create default rules last. */ 396 for (i = 0; i != MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) { 397 err = accel_fs_tcp_add_default_rule(priv, i); 398 if (err) 399 goto err_destroy_rules; 400 } 401 return (0); 402 403err_destroy_rules: 404 while (i--) 405 mlx5_del_flow_rule(&priv->fts.accel_tcp.default_rules[i]); 406 i = MLX5E_ACCEL_FS_TCP_NUM_TYPES; 407 408err_destroy_tables: 409 while (i--) 410 accel_fs_tcp_destroy_table(priv, i); 411 return (err); 412} 413