1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2// Copyright (c) 2019 Mellanox Technologies.
3
4#include <linux/mlx5/fs.h>
5#include "eswitch.h"
6#include "en_tc.h"
7#include "fs_core.h"
8
9struct mlx5_termtbl_handle {
10	struct hlist_node termtbl_hlist;
11
12	struct mlx5_flow_table *termtbl;
13	struct mlx5_flow_act flow_act;
14	struct mlx5_flow_destination dest;
15
16	struct mlx5_flow_handle *rule;
17	int ref_count;
18};
19
20static u32
21mlx5_eswitch_termtbl_hash(struct mlx5_flow_act *flow_act,
22			  struct mlx5_flow_destination *dest)
23{
24	u32 hash;
25
26	hash = jhash_1word(flow_act->action, 0);
27	hash = jhash((const void *)&flow_act->vlan,
28		     sizeof(flow_act->vlan), hash);
29	hash = jhash((const void *)&dest->vport.num,
30		     sizeof(dest->vport.num), hash);
31	hash = jhash((const void *)&dest->vport.vhca_id,
32		     sizeof(dest->vport.num), hash);
33	if (flow_act->pkt_reformat)
34		hash = jhash(flow_act->pkt_reformat,
35			     sizeof(*flow_act->pkt_reformat),
36			     hash);
37	return hash;
38}
39
40static int
41mlx5_eswitch_termtbl_cmp(struct mlx5_flow_act *flow_act1,
42			 struct mlx5_flow_destination *dest1,
43			 struct mlx5_flow_act *flow_act2,
44			 struct mlx5_flow_destination *dest2)
45{
46	int ret;
47
48	ret = flow_act1->action != flow_act2->action ||
49	      dest1->vport.num != dest2->vport.num ||
50	      dest1->vport.vhca_id != dest2->vport.vhca_id ||
51	      memcmp(&flow_act1->vlan, &flow_act2->vlan,
52		     sizeof(flow_act1->vlan));
53	if (ret)
54		return ret;
55
56	if (flow_act1->pkt_reformat && flow_act2->pkt_reformat)
57		return memcmp(flow_act1->pkt_reformat, flow_act2->pkt_reformat,
58			      sizeof(*flow_act1->pkt_reformat));
59
60	return !(flow_act1->pkt_reformat == flow_act2->pkt_reformat);
61}
62
63static int
64mlx5_eswitch_termtbl_create(struct mlx5_core_dev *dev,
65			    struct mlx5_termtbl_handle *tt,
66			    struct mlx5_flow_act *flow_act)
67{
68	struct mlx5_flow_table_attr ft_attr = {};
69	struct mlx5_flow_namespace *root_ns;
70	int err, err2;
71
72	root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
73	if (!root_ns) {
74		esw_warn(dev, "Failed to get FDB flow namespace\n");
75		return -EOPNOTSUPP;
76	}
77
78	/* As this is the terminating action then the termination table is the
79	 * same prio as the slow path
80	 */
81	ft_attr.flags = MLX5_FLOW_TABLE_TERMINATION | MLX5_FLOW_TABLE_UNMANAGED |
82			MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
83	ft_attr.prio = FDB_TC_OFFLOAD;
84	ft_attr.max_fte = 1;
85	ft_attr.level = 1;
86	ft_attr.autogroup.max_num_groups = 1;
87	tt->termtbl = mlx5_create_auto_grouped_flow_table(root_ns, &ft_attr);
88	if (IS_ERR(tt->termtbl)) {
89		err = PTR_ERR(tt->termtbl);
90		esw_warn(dev, "Failed to create termination table, err %pe\n", tt->termtbl);
91		return err;
92	}
93
94	tt->rule = mlx5_add_flow_rules(tt->termtbl, NULL, flow_act,
95				       &tt->dest, 1);
96	if (IS_ERR(tt->rule)) {
97		err = PTR_ERR(tt->rule);
98		esw_warn(dev, "Failed to create termination table rule, err %pe\n", tt->rule);
99		goto add_flow_err;
100	}
101	return 0;
102
103add_flow_err:
104	err2 = mlx5_destroy_flow_table(tt->termtbl);
105	if (err2)
106		esw_warn(dev, "Failed to destroy termination table, err %d\n", err2);
107
108	return err;
109}
110
111static struct mlx5_termtbl_handle *
112mlx5_eswitch_termtbl_get_create(struct mlx5_eswitch *esw,
113				struct mlx5_flow_act *flow_act,
114				struct mlx5_flow_destination *dest,
115				struct mlx5_esw_flow_attr *attr)
116{
117	struct mlx5_termtbl_handle *tt;
118	bool found = false;
119	u32 hash_key;
120	int err;
121
122	mutex_lock(&esw->offloads.termtbl_mutex);
123	hash_key = mlx5_eswitch_termtbl_hash(flow_act, dest);
124	hash_for_each_possible(esw->offloads.termtbl_tbl, tt,
125			       termtbl_hlist, hash_key) {
126		if (!mlx5_eswitch_termtbl_cmp(&tt->flow_act, &tt->dest,
127					      flow_act, dest)) {
128			found = true;
129			break;
130		}
131	}
132	if (found)
133		goto tt_add_ref;
134
135	tt = kzalloc(sizeof(*tt), GFP_KERNEL);
136	if (!tt) {
137		err = -ENOMEM;
138		goto tt_create_err;
139	}
140
141	tt->dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
142	tt->dest.vport.num = dest->vport.num;
143	tt->dest.vport.vhca_id = dest->vport.vhca_id;
144	tt->dest.vport.flags = dest->vport.flags;
145	memcpy(&tt->flow_act, flow_act, sizeof(*flow_act));
146
147	err = mlx5_eswitch_termtbl_create(esw->dev, tt, flow_act);
148	if (err)
149		goto tt_create_err;
150
151	hash_add(esw->offloads.termtbl_tbl, &tt->termtbl_hlist, hash_key);
152tt_add_ref:
153	tt->ref_count++;
154	mutex_unlock(&esw->offloads.termtbl_mutex);
155	return tt;
156tt_create_err:
157	kfree(tt);
158	mutex_unlock(&esw->offloads.termtbl_mutex);
159	return ERR_PTR(err);
160}
161
162void
163mlx5_eswitch_termtbl_put(struct mlx5_eswitch *esw,
164			 struct mlx5_termtbl_handle *tt)
165{
166	mutex_lock(&esw->offloads.termtbl_mutex);
167	if (--tt->ref_count == 0)
168		hash_del(&tt->termtbl_hlist);
169	mutex_unlock(&esw->offloads.termtbl_mutex);
170
171	if (!tt->ref_count) {
172		mlx5_del_flow_rules(tt->rule);
173		mlx5_destroy_flow_table(tt->termtbl);
174		kfree(tt);
175	}
176}
177
178static void
179mlx5_eswitch_termtbl_actions_move(struct mlx5_flow_act *src,
180				  struct mlx5_flow_act *dst)
181{
182	if (src->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) {
183		src->action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH;
184		dst->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH;
185		memcpy(&dst->vlan[0], &src->vlan[0], sizeof(src->vlan[0]));
186		memset(&src->vlan[0], 0, sizeof(src->vlan[0]));
187
188		if (src->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) {
189			src->action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2;
190			dst->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2;
191			memcpy(&dst->vlan[1], &src->vlan[1], sizeof(src->vlan[1]));
192			memset(&src->vlan[1], 0, sizeof(src->vlan[1]));
193		}
194	}
195}
196
197static bool mlx5_eswitch_offload_is_uplink_port(const struct mlx5_eswitch *esw,
198						const struct mlx5_flow_spec *spec)
199{
200	u16 port_mask, port_value;
201
202	if (MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source))
203		return spec->flow_context.flow_source ==
204					MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK;
205
206	port_mask = MLX5_GET(fte_match_param, spec->match_criteria,
207			     misc_parameters.source_port);
208	port_value = MLX5_GET(fte_match_param, spec->match_value,
209			      misc_parameters.source_port);
210	return (port_mask & port_value) == MLX5_VPORT_UPLINK;
211}
212
213bool
214mlx5_eswitch_termtbl_required(struct mlx5_eswitch *esw,
215			      struct mlx5_flow_attr *attr,
216			      struct mlx5_flow_act *flow_act,
217			      struct mlx5_flow_spec *spec)
218{
219	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
220	int i;
221
222	if (!MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, termination_table) ||
223	    !MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level) ||
224	    mlx5e_tc_attr_flags_skip(attr->flags) ||
225	    (!mlx5_eswitch_offload_is_uplink_port(esw, spec) && !esw_attr->int_port))
226		return false;
227
228	/* push vlan on RX */
229	if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH &&
230	    !(mlx5_fs_get_capabilities(esw->dev, MLX5_FLOW_NAMESPACE_FDB) &
231	      MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX))
232		return true;
233
234	/* hairpin */
235	for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
236		if (!esw_attr->dest_int_port && esw_attr->dests[i].vport_valid &&
237		    esw_attr->dests[i].vport == MLX5_VPORT_UPLINK)
238			return true;
239
240	return false;
241}
242
243struct mlx5_flow_handle *
244mlx5_eswitch_add_termtbl_rule(struct mlx5_eswitch *esw,
245			      struct mlx5_flow_table *fdb,
246			      struct mlx5_flow_spec *spec,
247			      struct mlx5_esw_flow_attr *attr,
248			      struct mlx5_flow_act *flow_act,
249			      struct mlx5_flow_destination *dest,
250			      int num_dest)
251{
252	struct mlx5_flow_act term_tbl_act = {};
253	struct mlx5_flow_handle *rule = NULL;
254	bool term_table_created = false;
255	int num_vport_dests = 0;
256	int i, curr_dest;
257
258	mlx5_eswitch_termtbl_actions_move(flow_act, &term_tbl_act);
259	term_tbl_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
260
261	for (i = 0; i < num_dest; i++) {
262		struct mlx5_termtbl_handle *tt;
263
264		/* only vport destinations can be terminated */
265		if (dest[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT)
266			continue;
267
268		if (attr->dests[num_vport_dests].flags & MLX5_ESW_DEST_ENCAP) {
269			term_tbl_act.action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
270			term_tbl_act.pkt_reformat = attr->dests[num_vport_dests].pkt_reformat;
271		} else {
272			term_tbl_act.action &= ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
273			term_tbl_act.pkt_reformat = NULL;
274		}
275
276		/* get the terminating table for the action list */
277		tt = mlx5_eswitch_termtbl_get_create(esw, &term_tbl_act,
278						     &dest[i], attr);
279		if (IS_ERR(tt)) {
280			esw_warn(esw->dev, "Failed to get termination table, err %pe\n", tt);
281			goto revert_changes;
282		}
283		attr->dests[num_vport_dests].termtbl = tt;
284		num_vport_dests++;
285
286		/* link the destination with the termination table */
287		dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
288		dest[i].ft = tt->termtbl;
289		term_table_created = true;
290	}
291
292	/* at least one destination should reference a termination table */
293	if (!term_table_created)
294		goto revert_changes;
295
296	/* create the FTE */
297	flow_act->action &= ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
298	flow_act->pkt_reformat = NULL;
299	flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
300	rule = mlx5_add_flow_rules(fdb, spec, flow_act, dest, num_dest);
301	if (IS_ERR(rule))
302		goto revert_changes;
303
304	goto out;
305
306revert_changes:
307	/* revert the changes that were made to the original flow_act
308	 * and fall-back to the original rule actions
309	 */
310	mlx5_eswitch_termtbl_actions_move(&term_tbl_act, flow_act);
311
312	for (curr_dest = 0; curr_dest < num_vport_dests; curr_dest++) {
313		struct mlx5_termtbl_handle *tt = attr->dests[curr_dest].termtbl;
314
315		attr->dests[curr_dest].termtbl = NULL;
316
317		/* search for the destination associated with the
318		 * current term table
319		 */
320		for (i = 0; i < num_dest; i++) {
321			if (dest[i].ft != tt->termtbl)
322				continue;
323
324			memset(&dest[i], 0, sizeof(dest[i]));
325			dest[i].type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
326			dest[i].vport.num = tt->dest.vport.num;
327			dest[i].vport.vhca_id = tt->dest.vport.vhca_id;
328			mlx5_eswitch_termtbl_put(esw, tt);
329			break;
330		}
331	}
332	rule = mlx5_add_flow_rules(fdb, spec, flow_act, dest, num_dest);
333out:
334	return rule;
335}
336