1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2// Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 4#include <linux/rhashtable.h> 5#include <net/flow_offload.h> 6#include "en/tc_priv.h" 7#include "act_stats.h" 8#include "en/fs.h" 9 10struct mlx5e_tc_act_stats_handle { 11 struct rhashtable ht; 12 spinlock_t ht_lock; /* protects hashtable */ 13}; 14 15struct mlx5e_tc_act_stats { 16 unsigned long tc_act_cookie; 17 18 struct mlx5_fc *counter; 19 u64 lastpackets; 20 u64 lastbytes; 21 22 struct rhash_head hash; 23 struct rcu_head rcu_head; 24}; 25 26static const struct rhashtable_params act_counters_ht_params = { 27 .head_offset = offsetof(struct mlx5e_tc_act_stats, hash), 28 .key_offset = offsetof(struct mlx5e_tc_act_stats, tc_act_cookie), 29 .key_len = sizeof_field(struct mlx5e_tc_act_stats, tc_act_cookie), 30 .automatic_shrinking = true, 31}; 32 33struct mlx5e_tc_act_stats_handle * 34mlx5e_tc_act_stats_create(void) 35{ 36 struct mlx5e_tc_act_stats_handle *handle; 37 int err; 38 39 handle = kvzalloc(sizeof(*handle), GFP_KERNEL); 40 if (!handle) 41 return ERR_PTR(-ENOMEM); 42 43 err = rhashtable_init(&handle->ht, &act_counters_ht_params); 44 if (err) 45 goto err; 46 47 spin_lock_init(&handle->ht_lock); 48 return handle; 49err: 50 kvfree(handle); 51 return ERR_PTR(err); 52} 53 54void mlx5e_tc_act_stats_free(struct mlx5e_tc_act_stats_handle *handle) 55{ 56 rhashtable_destroy(&handle->ht); 57 kvfree(handle); 58} 59 60static int 61mlx5e_tc_act_stats_add(struct mlx5e_tc_act_stats_handle *handle, 62 unsigned long act_cookie, 63 struct mlx5_fc *counter) 64{ 65 struct mlx5e_tc_act_stats *act_stats, *old_act_stats; 66 struct rhashtable *ht = &handle->ht; 67 u64 lastused; 68 int err = 0; 69 70 act_stats = kvzalloc(sizeof(*act_stats), GFP_KERNEL); 71 if (!act_stats) 72 return -ENOMEM; 73 74 act_stats->tc_act_cookie = act_cookie; 75 act_stats->counter = counter; 76 77 mlx5_fc_query_cached_raw(counter, 78 &act_stats->lastbytes, 79 &act_stats->lastpackets, &lastused); 80 81 rcu_read_lock(); 82 old_act_stats = rhashtable_lookup_get_insert_fast(ht, 83 &act_stats->hash, 84 act_counters_ht_params); 85 if (IS_ERR(old_act_stats)) { 86 err = PTR_ERR(old_act_stats); 87 goto err_hash_insert; 88 } else if (old_act_stats) { 89 err = -EEXIST; 90 goto err_hash_insert; 91 } 92 rcu_read_unlock(); 93 94 return 0; 95 96err_hash_insert: 97 rcu_read_unlock(); 98 kvfree(act_stats); 99 return err; 100} 101 102void 103mlx5e_tc_act_stats_del_flow(struct mlx5e_tc_act_stats_handle *handle, 104 struct mlx5e_tc_flow *flow) 105{ 106 struct mlx5_flow_attr *attr; 107 struct mlx5e_tc_act_stats *act_stats; 108 int i; 109 110 if (!flow_flag_test(flow, USE_ACT_STATS)) 111 return; 112 113 list_for_each_entry(attr, &flow->attrs, list) { 114 for (i = 0; i < attr->tc_act_cookies_count; i++) { 115 struct rhashtable *ht = &handle->ht; 116 117 spin_lock(&handle->ht_lock); 118 act_stats = rhashtable_lookup_fast(ht, 119 &attr->tc_act_cookies[i], 120 act_counters_ht_params); 121 if (act_stats && 122 rhashtable_remove_fast(ht, &act_stats->hash, 123 act_counters_ht_params) == 0) 124 kvfree_rcu(act_stats, rcu_head); 125 126 spin_unlock(&handle->ht_lock); 127 } 128 } 129} 130 131int 132mlx5e_tc_act_stats_add_flow(struct mlx5e_tc_act_stats_handle *handle, 133 struct mlx5e_tc_flow *flow) 134{ 135 struct mlx5_fc *curr_counter = NULL; 136 unsigned long last_cookie = 0; 137 struct mlx5_flow_attr *attr; 138 int err; 139 int i; 140 141 if (!flow_flag_test(flow, USE_ACT_STATS)) 142 return 0; 143 144 list_for_each_entry(attr, &flow->attrs, list) { 145 if (attr->counter) 146 curr_counter = attr->counter; 147 148 for (i = 0; i < attr->tc_act_cookies_count; i++) { 149 /* jump over identical ids (e.g. pedit)*/ 150 if (last_cookie == attr->tc_act_cookies[i]) 151 continue; 152 153 err = mlx5e_tc_act_stats_add(handle, attr->tc_act_cookies[i], curr_counter); 154 if (err) 155 goto out_err; 156 last_cookie = attr->tc_act_cookies[i]; 157 } 158 } 159 160 return 0; 161out_err: 162 mlx5e_tc_act_stats_del_flow(handle, flow); 163 return err; 164} 165 166int 167mlx5e_tc_act_stats_fill_stats(struct mlx5e_tc_act_stats_handle *handle, 168 struct flow_offload_action *fl_act) 169{ 170 struct rhashtable *ht = &handle->ht; 171 struct mlx5e_tc_act_stats *item; 172 u64 pkts, bytes, lastused; 173 int err = 0; 174 175 rcu_read_lock(); 176 item = rhashtable_lookup(ht, &fl_act->cookie, act_counters_ht_params); 177 if (!item) { 178 rcu_read_unlock(); 179 err = -ENOENT; 180 goto err_out; 181 } 182 183 mlx5_fc_query_cached_raw(item->counter, 184 &bytes, &pkts, &lastused); 185 186 flow_stats_update(&fl_act->stats, 187 bytes - item->lastbytes, 188 pkts - item->lastpackets, 189 0, lastused, FLOW_ACTION_HW_STATS_DELAYED); 190 191 item->lastpackets = pkts; 192 item->lastbytes = bytes; 193 rcu_read_unlock(); 194 195 return 0; 196 197err_out: 198 return err; 199} 200