1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2/* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */ 3 4#include "rqt.h" 5#include <linux/mlx5/transobj.h> 6 7static bool verify_num_vhca_ids(struct mlx5_core_dev *mdev, u32 *vhca_ids, 8 unsigned int size) 9{ 10 unsigned int max_num_vhca_id = MLX5_CAP_GEN_2(mdev, max_rqt_vhca_id); 11 int i; 12 13 /* Verify that all vhca_ids are in range [0, max_num_vhca_ids - 1] */ 14 for (i = 0; i < size; i++) 15 if (vhca_ids[i] >= max_num_vhca_id) 16 return false; 17 return true; 18} 19 20static bool rqt_verify_vhca_ids(struct mlx5_core_dev *mdev, u32 *vhca_ids, 21 unsigned int size) 22{ 23 if (!vhca_ids) 24 return true; 25 26 if (!MLX5_CAP_GEN(mdev, cross_vhca_rqt)) 27 return false; 28 if (!verify_num_vhca_ids(mdev, vhca_ids, size)) 29 return false; 30 31 return true; 32} 33 34void mlx5e_rss_params_indir_init_uniform(struct mlx5e_rss_params_indir *indir, 35 unsigned int num_channels) 36{ 37 unsigned int i; 38 39 for (i = 0; i < indir->actual_table_size; i++) 40 indir->table[i] = i % num_channels; 41} 42 43static void fill_rqn_list(void *rqtc, u32 *rqns, u32 *vhca_ids, unsigned int size) 44{ 45 unsigned int i; 46 47 if (vhca_ids) { 48 MLX5_SET(rqtc, rqtc, rq_vhca_id_format, 1); 49 for (i = 0; i < size; i++) { 50 MLX5_SET(rqtc, rqtc, rq_vhca[i].rq_num, rqns[i]); 51 MLX5_SET(rqtc, rqtc, rq_vhca[i].rq_vhca_id, vhca_ids[i]); 52 } 53 } else { 54 for (i = 0; i < size; i++) 55 MLX5_SET(rqtc, rqtc, rq_num[i], rqns[i]); 56 } 57} 58static int mlx5e_rqt_init(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, 59 u16 max_size, u32 *init_rqns, u32 *init_vhca_ids, u16 init_size) 60{ 61 int entry_sz; 62 void *rqtc; 63 int inlen; 64 int err; 65 u32 *in; 66 67 if (!rqt_verify_vhca_ids(mdev, init_vhca_ids, init_size)) 68 return -EOPNOTSUPP; 69 70 rqt->mdev = mdev; 71 rqt->size = max_size; 72 73 entry_sz = init_vhca_ids ? MLX5_ST_SZ_BYTES(rq_vhca) : MLX5_ST_SZ_BYTES(rq_num); 74 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + entry_sz * init_size; 75 in = kvzalloc(inlen, GFP_KERNEL); 76 if (!in) 77 return -ENOMEM; 78 79 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); 80 81 MLX5_SET(rqtc, rqtc, rqt_max_size, rqt->size); 82 MLX5_SET(rqtc, rqtc, rqt_actual_size, init_size); 83 84 fill_rqn_list(rqtc, init_rqns, init_vhca_ids, init_size); 85 86 err = mlx5_core_create_rqt(rqt->mdev, in, inlen, &rqt->rqtn); 87 88 kvfree(in); 89 return err; 90} 91 92int mlx5e_rqt_init_direct(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, 93 bool indir_enabled, u32 init_rqn, u32 indir_table_size) 94{ 95 u16 max_size = indir_enabled ? indir_table_size : 1; 96 97 return mlx5e_rqt_init(rqt, mdev, max_size, &init_rqn, NULL, 1); 98} 99 100static int mlx5e_bits_invert(unsigned long a, int size) 101{ 102 int inv = 0; 103 int i; 104 105 for (i = 0; i < size; i++) 106 inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i; 107 108 return inv; 109} 110 111static int mlx5e_calc_indir_rqns(u32 *rss_rqns, u32 *rqns, u32 *rss_vhca_ids, u32 *vhca_ids, 112 unsigned int num_rqns, 113 u8 hfunc, struct mlx5e_rss_params_indir *indir) 114{ 115 unsigned int i; 116 117 for (i = 0; i < indir->actual_table_size; i++) { 118 unsigned int ix = i; 119 120 if (hfunc == ETH_RSS_HASH_XOR) 121 ix = mlx5e_bits_invert(ix, ilog2(indir->actual_table_size)); 122 123 ix = indir->table[ix]; 124 125 if (WARN_ON(ix >= num_rqns)) 126 /* Could be a bug in the driver or in the kernel part of 127 * ethtool: indir table refers to non-existent RQs. 128 */ 129 return -EINVAL; 130 rss_rqns[i] = rqns[ix]; 131 if (vhca_ids) 132 rss_vhca_ids[i] = vhca_ids[ix]; 133 } 134 135 return 0; 136} 137 138int mlx5e_rqt_init_indir(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, 139 u32 *rqns, u32 *vhca_ids, unsigned int num_rqns, 140 u8 hfunc, struct mlx5e_rss_params_indir *indir) 141{ 142 u32 *rss_rqns, *rss_vhca_ids = NULL; 143 int err; 144 145 rss_rqns = kvmalloc_array(indir->actual_table_size, sizeof(*rss_rqns), GFP_KERNEL); 146 if (!rss_rqns) 147 return -ENOMEM; 148 149 if (vhca_ids) { 150 rss_vhca_ids = kvmalloc_array(indir->actual_table_size, sizeof(*rss_vhca_ids), 151 GFP_KERNEL); 152 if (!rss_vhca_ids) { 153 kvfree(rss_rqns); 154 return -ENOMEM; 155 } 156 } 157 158 err = mlx5e_calc_indir_rqns(rss_rqns, rqns, rss_vhca_ids, vhca_ids, num_rqns, hfunc, indir); 159 if (err) 160 goto out; 161 162 err = mlx5e_rqt_init(rqt, mdev, indir->max_table_size, rss_rqns, rss_vhca_ids, 163 indir->actual_table_size); 164 165out: 166 kvfree(rss_vhca_ids); 167 kvfree(rss_rqns); 168 return err; 169} 170 171#define MLX5E_UNIFORM_SPREAD_RQT_FACTOR 2 172 173u32 mlx5e_rqt_size(struct mlx5_core_dev *mdev, unsigned int num_channels) 174{ 175 u32 rqt_size = max_t(u32, MLX5E_INDIR_MIN_RQT_SIZE, 176 roundup_pow_of_two(num_channels * MLX5E_UNIFORM_SPREAD_RQT_FACTOR)); 177 u32 max_cap_rqt_size = 1 << MLX5_CAP_GEN(mdev, log_max_rqt_size); 178 179 return min_t(u32, rqt_size, max_cap_rqt_size); 180} 181 182#define MLX5E_MAX_RQT_SIZE_ALLOWED_WITH_XOR8_HASH 256 183 184unsigned int mlx5e_rqt_max_num_channels_allowed_for_xor8(void) 185{ 186 return MLX5E_MAX_RQT_SIZE_ALLOWED_WITH_XOR8_HASH / MLX5E_UNIFORM_SPREAD_RQT_FACTOR; 187} 188 189void mlx5e_rqt_destroy(struct mlx5e_rqt *rqt) 190{ 191 mlx5_core_destroy_rqt(rqt->mdev, rqt->rqtn); 192} 193 194static int mlx5e_rqt_redirect(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids, 195 unsigned int size) 196{ 197 int entry_sz; 198 void *rqtc; 199 int inlen; 200 u32 *in; 201 int err; 202 203 if (!rqt_verify_vhca_ids(rqt->mdev, vhca_ids, size)) 204 return -EINVAL; 205 206 entry_sz = vhca_ids ? MLX5_ST_SZ_BYTES(rq_vhca) : MLX5_ST_SZ_BYTES(rq_num); 207 inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + entry_sz * size; 208 in = kvzalloc(inlen, GFP_KERNEL); 209 if (!in) 210 return -ENOMEM; 211 212 rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx); 213 214 MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1); 215 MLX5_SET(rqtc, rqtc, rqt_actual_size, size); 216 217 fill_rqn_list(rqtc, rqns, vhca_ids, size); 218 219 err = mlx5_core_modify_rqt(rqt->mdev, rqt->rqtn, in, inlen); 220 221 kvfree(in); 222 return err; 223} 224 225int mlx5e_rqt_redirect_direct(struct mlx5e_rqt *rqt, u32 rqn, u32 *vhca_id) 226{ 227 return mlx5e_rqt_redirect(rqt, &rqn, vhca_id, 1); 228} 229 230int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids, 231 unsigned int num_rqns, 232 u8 hfunc, struct mlx5e_rss_params_indir *indir) 233{ 234 u32 *rss_rqns, *rss_vhca_ids = NULL; 235 int err; 236 237 if (!rqt_verify_vhca_ids(rqt->mdev, vhca_ids, num_rqns)) 238 return -EINVAL; 239 240 if (WARN_ON(rqt->size != indir->max_table_size)) 241 return -EINVAL; 242 243 rss_rqns = kvmalloc_array(indir->actual_table_size, sizeof(*rss_rqns), GFP_KERNEL); 244 if (!rss_rqns) 245 return -ENOMEM; 246 247 if (vhca_ids) { 248 rss_vhca_ids = kvmalloc_array(indir->actual_table_size, sizeof(*rss_vhca_ids), 249 GFP_KERNEL); 250 if (!rss_vhca_ids) { 251 kvfree(rss_rqns); 252 return -ENOMEM; 253 } 254 } 255 256 err = mlx5e_calc_indir_rqns(rss_rqns, rqns, rss_vhca_ids, vhca_ids, num_rqns, hfunc, indir); 257 if (err) 258 goto out; 259 260 err = mlx5e_rqt_redirect(rqt, rss_rqns, rss_vhca_ids, indir->actual_table_size); 261 262out: 263 kvfree(rss_vhca_ids); 264 kvfree(rss_rqns); 265 return err; 266} 267