1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2// Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES.
3
4#include <linux/ip.h>
5#include <linux/ipv6.h>
6#include <linux/tcp.h>
7#include <linux/mlx5/fs.h>
8#include <linux/mlx5/driver.h>
9#include "mlx5_core.h"
10#include "lib/fs_ttc.h"
11
12#define MLX5_TTC_NUM_GROUPS	3
13#define MLX5_TTC_GROUP1_SIZE	(BIT(3) + MLX5_NUM_TUNNEL_TT)
14#define MLX5_TTC_GROUP2_SIZE	 BIT(1)
15#define MLX5_TTC_GROUP3_SIZE	 BIT(0)
16#define MLX5_TTC_TABLE_SIZE	(MLX5_TTC_GROUP1_SIZE +\
17				 MLX5_TTC_GROUP2_SIZE +\
18				 MLX5_TTC_GROUP3_SIZE)
19
20#define MLX5_INNER_TTC_NUM_GROUPS	3
21#define MLX5_INNER_TTC_GROUP1_SIZE	BIT(3)
22#define MLX5_INNER_TTC_GROUP2_SIZE	BIT(1)
23#define MLX5_INNER_TTC_GROUP3_SIZE	BIT(0)
24#define MLX5_INNER_TTC_TABLE_SIZE	(MLX5_INNER_TTC_GROUP1_SIZE +\
25					 MLX5_INNER_TTC_GROUP2_SIZE +\
26					 MLX5_INNER_TTC_GROUP3_SIZE)
27
28/* L3/L4 traffic type classifier */
29struct mlx5_ttc_table {
30	int num_groups;
31	struct mlx5_flow_table *t;
32	struct mlx5_flow_group **g;
33	struct mlx5_ttc_rule rules[MLX5_NUM_TT];
34	struct mlx5_flow_handle *tunnel_rules[MLX5_NUM_TUNNEL_TT];
35};
36
37struct mlx5_flow_table *mlx5_get_ttc_flow_table(struct mlx5_ttc_table *ttc)
38{
39	return ttc->t;
40}
41
42static void mlx5_cleanup_ttc_rules(struct mlx5_ttc_table *ttc)
43{
44	int i;
45
46	for (i = 0; i < MLX5_NUM_TT; i++) {
47		if (!IS_ERR_OR_NULL(ttc->rules[i].rule)) {
48			mlx5_del_flow_rules(ttc->rules[i].rule);
49			ttc->rules[i].rule = NULL;
50		}
51	}
52
53	for (i = 0; i < MLX5_NUM_TUNNEL_TT; i++) {
54		if (!IS_ERR_OR_NULL(ttc->tunnel_rules[i])) {
55			mlx5_del_flow_rules(ttc->tunnel_rules[i]);
56			ttc->tunnel_rules[i] = NULL;
57		}
58	}
59}
60
61struct mlx5_etype_proto {
62	u16 etype;
63	u8 proto;
64};
65
66static struct mlx5_etype_proto ttc_rules[] = {
67	[MLX5_TT_IPV4_TCP] = {
68		.etype = ETH_P_IP,
69		.proto = IPPROTO_TCP,
70	},
71	[MLX5_TT_IPV6_TCP] = {
72		.etype = ETH_P_IPV6,
73		.proto = IPPROTO_TCP,
74	},
75	[MLX5_TT_IPV4_UDP] = {
76		.etype = ETH_P_IP,
77		.proto = IPPROTO_UDP,
78	},
79	[MLX5_TT_IPV6_UDP] = {
80		.etype = ETH_P_IPV6,
81		.proto = IPPROTO_UDP,
82	},
83	[MLX5_TT_IPV4_IPSEC_AH] = {
84		.etype = ETH_P_IP,
85		.proto = IPPROTO_AH,
86	},
87	[MLX5_TT_IPV6_IPSEC_AH] = {
88		.etype = ETH_P_IPV6,
89		.proto = IPPROTO_AH,
90	},
91	[MLX5_TT_IPV4_IPSEC_ESP] = {
92		.etype = ETH_P_IP,
93		.proto = IPPROTO_ESP,
94	},
95	[MLX5_TT_IPV6_IPSEC_ESP] = {
96		.etype = ETH_P_IPV6,
97		.proto = IPPROTO_ESP,
98	},
99	[MLX5_TT_IPV4] = {
100		.etype = ETH_P_IP,
101		.proto = 0,
102	},
103	[MLX5_TT_IPV6] = {
104		.etype = ETH_P_IPV6,
105		.proto = 0,
106	},
107	[MLX5_TT_ANY] = {
108		.etype = 0,
109		.proto = 0,
110	},
111};
112
113static struct mlx5_etype_proto ttc_tunnel_rules[] = {
114	[MLX5_TT_IPV4_GRE] = {
115		.etype = ETH_P_IP,
116		.proto = IPPROTO_GRE,
117	},
118	[MLX5_TT_IPV6_GRE] = {
119		.etype = ETH_P_IPV6,
120		.proto = IPPROTO_GRE,
121	},
122	[MLX5_TT_IPV4_IPIP] = {
123		.etype = ETH_P_IP,
124		.proto = IPPROTO_IPIP,
125	},
126	[MLX5_TT_IPV6_IPIP] = {
127		.etype = ETH_P_IPV6,
128		.proto = IPPROTO_IPIP,
129	},
130	[MLX5_TT_IPV4_IPV6] = {
131		.etype = ETH_P_IP,
132		.proto = IPPROTO_IPV6,
133	},
134	[MLX5_TT_IPV6_IPV6] = {
135		.etype = ETH_P_IPV6,
136		.proto = IPPROTO_IPV6,
137	},
138
139};
140
141u8 mlx5_get_proto_by_tunnel_type(enum mlx5_tunnel_types tt)
142{
143	return ttc_tunnel_rules[tt].proto;
144}
145
146static bool mlx5_tunnel_proto_supported_rx(struct mlx5_core_dev *mdev,
147					   u8 proto_type)
148{
149	switch (proto_type) {
150	case IPPROTO_GRE:
151		return MLX5_CAP_ETH(mdev, tunnel_stateless_gre);
152	case IPPROTO_IPIP:
153	case IPPROTO_IPV6:
154		return (MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip) ||
155			MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip_rx));
156	default:
157		return false;
158	}
159}
160
161static bool mlx5_tunnel_any_rx_proto_supported(struct mlx5_core_dev *mdev)
162{
163	int tt;
164
165	for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) {
166		if (mlx5_tunnel_proto_supported_rx(mdev,
167						   ttc_tunnel_rules[tt].proto))
168			return true;
169	}
170	return false;
171}
172
173bool mlx5_tunnel_inner_ft_supported(struct mlx5_core_dev *mdev)
174{
175	return (mlx5_tunnel_any_rx_proto_supported(mdev) &&
176		MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
177					  ft_field_support.inner_ip_version));
178}
179
180static u8 mlx5_etype_to_ipv(u16 ethertype)
181{
182	if (ethertype == ETH_P_IP)
183		return 4;
184
185	if (ethertype == ETH_P_IPV6)
186		return 6;
187
188	return 0;
189}
190
191static struct mlx5_flow_handle *
192mlx5_generate_ttc_rule(struct mlx5_core_dev *dev, struct mlx5_flow_table *ft,
193		       struct mlx5_flow_destination *dest, u16 etype, u8 proto)
194{
195	int match_ipv_outer =
196		MLX5_CAP_FLOWTABLE_NIC_RX(dev,
197					  ft_field_support.outer_ip_version);
198	MLX5_DECLARE_FLOW_ACT(flow_act);
199	struct mlx5_flow_handle *rule;
200	struct mlx5_flow_spec *spec;
201	int err = 0;
202	u8 ipv;
203
204	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
205	if (!spec)
206		return ERR_PTR(-ENOMEM);
207
208	if (proto) {
209		spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
210		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
211		MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, proto);
212	}
213
214	ipv = mlx5_etype_to_ipv(etype);
215	if (match_ipv_outer && ipv) {
216		spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
217		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
218		MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, ipv);
219	} else if (etype) {
220		spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
221		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype);
222		MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, etype);
223	}
224
225	rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, 1);
226	if (IS_ERR(rule)) {
227		err = PTR_ERR(rule);
228		mlx5_core_err(dev, "%s: add rule failed\n", __func__);
229	}
230
231	kvfree(spec);
232	return err ? ERR_PTR(err) : rule;
233}
234
235static int mlx5_generate_ttc_table_rules(struct mlx5_core_dev *dev,
236					 struct ttc_params *params,
237					 struct mlx5_ttc_table *ttc)
238{
239	struct mlx5_flow_handle **trules;
240	struct mlx5_ttc_rule *rules;
241	struct mlx5_flow_table *ft;
242	int tt;
243	int err;
244
245	ft = ttc->t;
246	rules = ttc->rules;
247	for (tt = 0; tt < MLX5_NUM_TT; tt++) {
248		struct mlx5_ttc_rule *rule = &rules[tt];
249
250		if (test_bit(tt, params->ignore_dests))
251			continue;
252		rule->rule = mlx5_generate_ttc_rule(dev, ft, &params->dests[tt],
253						    ttc_rules[tt].etype,
254						    ttc_rules[tt].proto);
255		if (IS_ERR(rule->rule)) {
256			err = PTR_ERR(rule->rule);
257			rule->rule = NULL;
258			goto del_rules;
259		}
260		rule->default_dest = params->dests[tt];
261	}
262
263	if (!params->inner_ttc || !mlx5_tunnel_inner_ft_supported(dev))
264		return 0;
265
266	trules    = ttc->tunnel_rules;
267	for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) {
268		if (!mlx5_tunnel_proto_supported_rx(dev,
269						    ttc_tunnel_rules[tt].proto))
270			continue;
271		if (test_bit(tt, params->ignore_tunnel_dests))
272			continue;
273		trules[tt] = mlx5_generate_ttc_rule(dev, ft,
274						    &params->tunnel_dests[tt],
275						    ttc_tunnel_rules[tt].etype,
276						    ttc_tunnel_rules[tt].proto);
277		if (IS_ERR(trules[tt])) {
278			err = PTR_ERR(trules[tt]);
279			trules[tt] = NULL;
280			goto del_rules;
281		}
282	}
283
284	return 0;
285
286del_rules:
287	mlx5_cleanup_ttc_rules(ttc);
288	return err;
289}
290
291static int mlx5_create_ttc_table_groups(struct mlx5_ttc_table *ttc,
292					bool use_ipv)
293{
294	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
295	int ix = 0;
296	u32 *in;
297	int err;
298	u8 *mc;
299
300	ttc->g = kcalloc(MLX5_TTC_NUM_GROUPS, sizeof(*ttc->g), GFP_KERNEL);
301	if (!ttc->g)
302		return -ENOMEM;
303	in = kvzalloc(inlen, GFP_KERNEL);
304	if (!in) {
305		kfree(ttc->g);
306		ttc->g = NULL;
307		return -ENOMEM;
308	}
309
310	/* L4 Group */
311	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
312	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
313	if (use_ipv)
314		MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_version);
315	else
316		MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
317	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
318	MLX5_SET_CFG(in, start_flow_index, ix);
319	ix += MLX5_TTC_GROUP1_SIZE;
320	MLX5_SET_CFG(in, end_flow_index, ix - 1);
321	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
322	if (IS_ERR(ttc->g[ttc->num_groups]))
323		goto err;
324	ttc->num_groups++;
325
326	/* L3 Group */
327	MLX5_SET(fte_match_param, mc, outer_headers.ip_protocol, 0);
328	MLX5_SET_CFG(in, start_flow_index, ix);
329	ix += MLX5_TTC_GROUP2_SIZE;
330	MLX5_SET_CFG(in, end_flow_index, ix - 1);
331	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
332	if (IS_ERR(ttc->g[ttc->num_groups]))
333		goto err;
334	ttc->num_groups++;
335
336	/* Any Group */
337	memset(in, 0, inlen);
338	MLX5_SET_CFG(in, start_flow_index, ix);
339	ix += MLX5_TTC_GROUP3_SIZE;
340	MLX5_SET_CFG(in, end_flow_index, ix - 1);
341	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
342	if (IS_ERR(ttc->g[ttc->num_groups]))
343		goto err;
344	ttc->num_groups++;
345
346	kvfree(in);
347	return 0;
348
349err:
350	err = PTR_ERR(ttc->g[ttc->num_groups]);
351	ttc->g[ttc->num_groups] = NULL;
352	kvfree(in);
353
354	return err;
355}
356
357static struct mlx5_flow_handle *
358mlx5_generate_inner_ttc_rule(struct mlx5_core_dev *dev,
359			     struct mlx5_flow_table *ft,
360			     struct mlx5_flow_destination *dest,
361			     u16 etype, u8 proto)
362{
363	MLX5_DECLARE_FLOW_ACT(flow_act);
364	struct mlx5_flow_handle *rule;
365	struct mlx5_flow_spec *spec;
366	int err = 0;
367	u8 ipv;
368
369	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
370	if (!spec)
371		return ERR_PTR(-ENOMEM);
372
373	ipv = mlx5_etype_to_ipv(etype);
374	if (etype && ipv) {
375		spec->match_criteria_enable = MLX5_MATCH_INNER_HEADERS;
376		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, inner_headers.ip_version);
377		MLX5_SET(fte_match_param, spec->match_value, inner_headers.ip_version, ipv);
378	}
379
380	if (proto) {
381		spec->match_criteria_enable = MLX5_MATCH_INNER_HEADERS;
382		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, inner_headers.ip_protocol);
383		MLX5_SET(fte_match_param, spec->match_value, inner_headers.ip_protocol, proto);
384	}
385
386	rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, 1);
387	if (IS_ERR(rule)) {
388		err = PTR_ERR(rule);
389		mlx5_core_err(dev, "%s: add inner TTC rule failed\n", __func__);
390	}
391
392	kvfree(spec);
393	return err ? ERR_PTR(err) : rule;
394}
395
396static int mlx5_generate_inner_ttc_table_rules(struct mlx5_core_dev *dev,
397					       struct ttc_params *params,
398					       struct mlx5_ttc_table *ttc)
399{
400	struct mlx5_ttc_rule *rules;
401	struct mlx5_flow_table *ft;
402	int err;
403	int tt;
404
405	ft = ttc->t;
406	rules = ttc->rules;
407
408	for (tt = 0; tt < MLX5_NUM_TT; tt++) {
409		struct mlx5_ttc_rule *rule = &rules[tt];
410
411		if (test_bit(tt, params->ignore_dests))
412			continue;
413		rule->rule = mlx5_generate_inner_ttc_rule(dev, ft,
414							  &params->dests[tt],
415							  ttc_rules[tt].etype,
416							  ttc_rules[tt].proto);
417		if (IS_ERR(rule->rule)) {
418			err = PTR_ERR(rule->rule);
419			rule->rule = NULL;
420			goto del_rules;
421		}
422		rule->default_dest = params->dests[tt];
423	}
424
425	return 0;
426
427del_rules:
428
429	mlx5_cleanup_ttc_rules(ttc);
430	return err;
431}
432
433static int mlx5_create_inner_ttc_table_groups(struct mlx5_ttc_table *ttc)
434{
435	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
436	int ix = 0;
437	u32 *in;
438	int err;
439	u8 *mc;
440
441	ttc->g = kcalloc(MLX5_INNER_TTC_NUM_GROUPS, sizeof(*ttc->g),
442			 GFP_KERNEL);
443	if (!ttc->g)
444		return -ENOMEM;
445	in = kvzalloc(inlen, GFP_KERNEL);
446	if (!in) {
447		kfree(ttc->g);
448		ttc->g = NULL;
449		return -ENOMEM;
450	}
451
452	/* L4 Group */
453	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
454	MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_protocol);
455	MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_version);
456	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS);
457	MLX5_SET_CFG(in, start_flow_index, ix);
458	ix += MLX5_INNER_TTC_GROUP1_SIZE;
459	MLX5_SET_CFG(in, end_flow_index, ix - 1);
460	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
461	if (IS_ERR(ttc->g[ttc->num_groups]))
462		goto err;
463	ttc->num_groups++;
464
465	/* L3 Group */
466	MLX5_SET(fte_match_param, mc, inner_headers.ip_protocol, 0);
467	MLX5_SET_CFG(in, start_flow_index, ix);
468	ix += MLX5_INNER_TTC_GROUP2_SIZE;
469	MLX5_SET_CFG(in, end_flow_index, ix - 1);
470	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
471	if (IS_ERR(ttc->g[ttc->num_groups]))
472		goto err;
473	ttc->num_groups++;
474
475	/* Any Group */
476	memset(in, 0, inlen);
477	MLX5_SET_CFG(in, start_flow_index, ix);
478	ix += MLX5_INNER_TTC_GROUP3_SIZE;
479	MLX5_SET_CFG(in, end_flow_index, ix - 1);
480	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
481	if (IS_ERR(ttc->g[ttc->num_groups]))
482		goto err;
483	ttc->num_groups++;
484
485	kvfree(in);
486	return 0;
487
488err:
489	err = PTR_ERR(ttc->g[ttc->num_groups]);
490	ttc->g[ttc->num_groups] = NULL;
491	kvfree(in);
492
493	return err;
494}
495
496struct mlx5_ttc_table *mlx5_create_inner_ttc_table(struct mlx5_core_dev *dev,
497						   struct ttc_params *params)
498{
499	struct mlx5_ttc_table *ttc;
500	int err;
501
502	ttc = kvzalloc(sizeof(*ttc), GFP_KERNEL);
503	if (!ttc)
504		return ERR_PTR(-ENOMEM);
505
506	WARN_ON_ONCE(params->ft_attr.max_fte);
507	params->ft_attr.max_fte = MLX5_INNER_TTC_TABLE_SIZE;
508	ttc->t = mlx5_create_flow_table(params->ns, &params->ft_attr);
509	if (IS_ERR(ttc->t)) {
510		err = PTR_ERR(ttc->t);
511		kvfree(ttc);
512		return ERR_PTR(err);
513	}
514
515	err = mlx5_create_inner_ttc_table_groups(ttc);
516	if (err)
517		goto destroy_ft;
518
519	err = mlx5_generate_inner_ttc_table_rules(dev, params, ttc);
520	if (err)
521		goto destroy_ft;
522
523	return ttc;
524
525destroy_ft:
526	mlx5_destroy_ttc_table(ttc);
527	return ERR_PTR(err);
528}
529
530void mlx5_destroy_ttc_table(struct mlx5_ttc_table *ttc)
531{
532	int i;
533
534	mlx5_cleanup_ttc_rules(ttc);
535	for (i = ttc->num_groups - 1; i >= 0; i--) {
536		if (!IS_ERR_OR_NULL(ttc->g[i]))
537			mlx5_destroy_flow_group(ttc->g[i]);
538		ttc->g[i] = NULL;
539	}
540
541	kfree(ttc->g);
542	mlx5_destroy_flow_table(ttc->t);
543	kvfree(ttc);
544}
545
546struct mlx5_ttc_table *mlx5_create_ttc_table(struct mlx5_core_dev *dev,
547					     struct ttc_params *params)
548{
549	bool match_ipv_outer =
550		MLX5_CAP_FLOWTABLE_NIC_RX(dev,
551					  ft_field_support.outer_ip_version);
552	struct mlx5_ttc_table *ttc;
553	int err;
554
555	ttc = kvzalloc(sizeof(*ttc), GFP_KERNEL);
556	if (!ttc)
557		return ERR_PTR(-ENOMEM);
558
559	WARN_ON_ONCE(params->ft_attr.max_fte);
560	params->ft_attr.max_fte = MLX5_TTC_TABLE_SIZE;
561	ttc->t = mlx5_create_flow_table(params->ns, &params->ft_attr);
562	if (IS_ERR(ttc->t)) {
563		err = PTR_ERR(ttc->t);
564		kvfree(ttc);
565		return ERR_PTR(err);
566	}
567
568	err = mlx5_create_ttc_table_groups(ttc, match_ipv_outer);
569	if (err)
570		goto destroy_ft;
571
572	err = mlx5_generate_ttc_table_rules(dev, params, ttc);
573	if (err)
574		goto destroy_ft;
575
576	return ttc;
577
578destroy_ft:
579	mlx5_destroy_ttc_table(ttc);
580	return ERR_PTR(err);
581}
582
583int mlx5_ttc_fwd_dest(struct mlx5_ttc_table *ttc, enum mlx5_traffic_types type,
584		      struct mlx5_flow_destination *new_dest)
585{
586	return mlx5_modify_rule_destination(ttc->rules[type].rule, new_dest,
587					    NULL);
588}
589
590struct mlx5_flow_destination
591mlx5_ttc_get_default_dest(struct mlx5_ttc_table *ttc,
592			  enum mlx5_traffic_types type)
593{
594	struct mlx5_flow_destination *dest = &ttc->rules[type].default_dest;
595
596	WARN_ONCE(dest->type != MLX5_FLOW_DESTINATION_TYPE_TIR,
597		  "TTC[%d] default dest is not setup yet", type);
598
599	return *dest;
600}
601
602int mlx5_ttc_fwd_default_dest(struct mlx5_ttc_table *ttc,
603			      enum mlx5_traffic_types type)
604{
605	struct mlx5_flow_destination dest = mlx5_ttc_get_default_dest(ttc, type);
606
607	return mlx5_ttc_fwd_dest(ttc, type, &dest);
608}
609