1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2// Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3
4#include <linux/rhashtable.h>
5#include <net/flow_offload.h>
6#include "en/tc_priv.h"
7#include "act_stats.h"
8#include "en/fs.h"
9
10struct mlx5e_tc_act_stats_handle {
11	struct rhashtable ht;
12	spinlock_t ht_lock; /* protects hashtable */
13};
14
15struct mlx5e_tc_act_stats {
16	unsigned long		tc_act_cookie;
17
18	struct mlx5_fc		*counter;
19	u64			lastpackets;
20	u64			lastbytes;
21
22	struct rhash_head	hash;
23	struct rcu_head		rcu_head;
24};
25
26static const struct rhashtable_params act_counters_ht_params = {
27	.head_offset = offsetof(struct mlx5e_tc_act_stats, hash),
28	.key_offset = offsetof(struct mlx5e_tc_act_stats, tc_act_cookie),
29	.key_len = sizeof_field(struct mlx5e_tc_act_stats, tc_act_cookie),
30	.automatic_shrinking = true,
31};
32
33struct mlx5e_tc_act_stats_handle *
34mlx5e_tc_act_stats_create(void)
35{
36	struct mlx5e_tc_act_stats_handle *handle;
37	int err;
38
39	handle = kvzalloc(sizeof(*handle), GFP_KERNEL);
40	if (!handle)
41		return ERR_PTR(-ENOMEM);
42
43	err = rhashtable_init(&handle->ht, &act_counters_ht_params);
44	if (err)
45		goto err;
46
47	spin_lock_init(&handle->ht_lock);
48	return handle;
49err:
50	kvfree(handle);
51	return ERR_PTR(err);
52}
53
54void mlx5e_tc_act_stats_free(struct mlx5e_tc_act_stats_handle *handle)
55{
56	rhashtable_destroy(&handle->ht);
57	kvfree(handle);
58}
59
60static int
61mlx5e_tc_act_stats_add(struct mlx5e_tc_act_stats_handle *handle,
62		       unsigned long act_cookie,
63		       struct mlx5_fc *counter)
64{
65	struct mlx5e_tc_act_stats *act_stats, *old_act_stats;
66	struct rhashtable *ht = &handle->ht;
67	u64 lastused;
68	int err = 0;
69
70	act_stats = kvzalloc(sizeof(*act_stats), GFP_KERNEL);
71	if (!act_stats)
72		return -ENOMEM;
73
74	act_stats->tc_act_cookie = act_cookie;
75	act_stats->counter = counter;
76
77	mlx5_fc_query_cached_raw(counter,
78				 &act_stats->lastbytes,
79				 &act_stats->lastpackets, &lastused);
80
81	rcu_read_lock();
82	old_act_stats = rhashtable_lookup_get_insert_fast(ht,
83							  &act_stats->hash,
84							  act_counters_ht_params);
85	if (IS_ERR(old_act_stats)) {
86		err = PTR_ERR(old_act_stats);
87		goto err_hash_insert;
88	} else if (old_act_stats) {
89		err = -EEXIST;
90		goto err_hash_insert;
91	}
92	rcu_read_unlock();
93
94	return 0;
95
96err_hash_insert:
97	rcu_read_unlock();
98	kvfree(act_stats);
99	return err;
100}
101
102void
103mlx5e_tc_act_stats_del_flow(struct mlx5e_tc_act_stats_handle *handle,
104			    struct mlx5e_tc_flow *flow)
105{
106	struct mlx5_flow_attr *attr;
107	struct mlx5e_tc_act_stats *act_stats;
108	int i;
109
110	if (!flow_flag_test(flow, USE_ACT_STATS))
111		return;
112
113	list_for_each_entry(attr, &flow->attrs, list) {
114		for (i = 0; i < attr->tc_act_cookies_count; i++) {
115			struct rhashtable *ht = &handle->ht;
116
117			spin_lock(&handle->ht_lock);
118			act_stats = rhashtable_lookup_fast(ht,
119							   &attr->tc_act_cookies[i],
120							   act_counters_ht_params);
121			if (act_stats &&
122			    rhashtable_remove_fast(ht, &act_stats->hash,
123						   act_counters_ht_params) == 0)
124				kvfree_rcu(act_stats, rcu_head);
125
126			spin_unlock(&handle->ht_lock);
127		}
128	}
129}
130
131int
132mlx5e_tc_act_stats_add_flow(struct mlx5e_tc_act_stats_handle *handle,
133			    struct mlx5e_tc_flow *flow)
134{
135	struct mlx5_fc *curr_counter = NULL;
136	unsigned long last_cookie = 0;
137	struct mlx5_flow_attr *attr;
138	int err;
139	int i;
140
141	if (!flow_flag_test(flow, USE_ACT_STATS))
142		return 0;
143
144	list_for_each_entry(attr, &flow->attrs, list) {
145		if (attr->counter)
146			curr_counter = attr->counter;
147
148		for (i = 0; i < attr->tc_act_cookies_count; i++) {
149			/* jump over identical ids (e.g. pedit)*/
150			if (last_cookie == attr->tc_act_cookies[i])
151				continue;
152
153			err = mlx5e_tc_act_stats_add(handle, attr->tc_act_cookies[i], curr_counter);
154			if (err)
155				goto out_err;
156			last_cookie = attr->tc_act_cookies[i];
157		}
158	}
159
160	return 0;
161out_err:
162	mlx5e_tc_act_stats_del_flow(handle, flow);
163	return err;
164}
165
166int
167mlx5e_tc_act_stats_fill_stats(struct mlx5e_tc_act_stats_handle *handle,
168			      struct flow_offload_action *fl_act)
169{
170	struct rhashtable *ht = &handle->ht;
171	struct mlx5e_tc_act_stats *item;
172	u64 pkts, bytes, lastused;
173	int err = 0;
174
175	rcu_read_lock();
176	item = rhashtable_lookup(ht, &fl_act->cookie, act_counters_ht_params);
177	if (!item) {
178		rcu_read_unlock();
179		err = -ENOENT;
180		goto err_out;
181	}
182
183	mlx5_fc_query_cached_raw(item->counter,
184				 &bytes, &pkts, &lastused);
185
186	flow_stats_update(&fl_act->stats,
187			  bytes - item->lastbytes,
188			  pkts - item->lastpackets,
189			  0, lastused, FLOW_ACTION_HW_STATS_DELAYED);
190
191	item->lastpackets = pkts;
192	item->lastbytes = bytes;
193	rcu_read_unlock();
194
195	return 0;
196
197err_out:
198	return err;
199}
200