1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2// Copyright (c) 2019 Mellanox Technologies 3 4#include <linux/mlx5/driver.h> 5#include <linux/mlx5/device.h> 6 7#include "mlx5_core.h" 8#include "lib/mlx5.h" 9 10struct mlx5_dm { 11 /* protect access to icm bitmask */ 12 spinlock_t lock; 13 unsigned long *steering_sw_icm_alloc_blocks; 14 unsigned long *header_modify_sw_icm_alloc_blocks; 15 unsigned long *header_modify_pattern_sw_icm_alloc_blocks; 16 unsigned long *header_encap_sw_icm_alloc_blocks; 17}; 18 19struct mlx5_dm *mlx5_dm_create(struct mlx5_core_dev *dev) 20{ 21 u64 header_modify_pattern_icm_blocks = 0; 22 u64 header_sw_encap_icm_blocks = 0; 23 u64 header_modify_icm_blocks = 0; 24 u64 steering_icm_blocks = 0; 25 struct mlx5_dm *dm; 26 bool support_v2; 27 28 if (!(MLX5_CAP_GEN_64(dev, general_obj_types) & MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM)) 29 return NULL; 30 31 dm = kzalloc(sizeof(*dm), GFP_KERNEL); 32 if (!dm) 33 return ERR_PTR(-ENOMEM); 34 35 spin_lock_init(&dm->lock); 36 37 if (MLX5_CAP64_DEV_MEM(dev, steering_sw_icm_start_address)) { 38 steering_icm_blocks = 39 BIT(MLX5_CAP_DEV_MEM(dev, log_steering_sw_icm_size) - 40 MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); 41 42 dm->steering_sw_icm_alloc_blocks = 43 bitmap_zalloc(steering_icm_blocks, GFP_KERNEL); 44 if (!dm->steering_sw_icm_alloc_blocks) 45 goto err_steering; 46 } 47 48 if (MLX5_CAP64_DEV_MEM(dev, header_modify_sw_icm_start_address)) { 49 header_modify_icm_blocks = 50 BIT(MLX5_CAP_DEV_MEM(dev, log_header_modify_sw_icm_size) - 51 MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); 52 53 dm->header_modify_sw_icm_alloc_blocks = 54 bitmap_zalloc(header_modify_icm_blocks, GFP_KERNEL); 55 if (!dm->header_modify_sw_icm_alloc_blocks) 56 goto err_modify_hdr; 57 } 58 59 if (MLX5_CAP_DEV_MEM(dev, log_indirect_encap_sw_icm_size)) { 60 header_sw_encap_icm_blocks = 61 BIT(MLX5_CAP_DEV_MEM(dev, log_indirect_encap_sw_icm_size) - 62 MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); 63 64 dm->header_encap_sw_icm_alloc_blocks = 65 bitmap_zalloc(header_sw_encap_icm_blocks, GFP_KERNEL); 66 if (!dm->header_encap_sw_icm_alloc_blocks) 67 goto err_pattern; 68 } 69 70 support_v2 = MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner_v2) && 71 MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner_v2) && 72 MLX5_CAP64_DEV_MEM(dev, header_modify_pattern_sw_icm_start_address); 73 74 if (support_v2) { 75 header_modify_pattern_icm_blocks = 76 BIT(MLX5_CAP_DEV_MEM(dev, log_header_modify_pattern_sw_icm_size) - 77 MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); 78 79 dm->header_modify_pattern_sw_icm_alloc_blocks = 80 bitmap_zalloc(header_modify_pattern_icm_blocks, GFP_KERNEL); 81 if (!dm->header_modify_pattern_sw_icm_alloc_blocks) 82 goto err_sw_encap; 83 } 84 85 return dm; 86 87err_sw_encap: 88 bitmap_free(dm->header_encap_sw_icm_alloc_blocks); 89 90err_pattern: 91 bitmap_free(dm->header_modify_sw_icm_alloc_blocks); 92 93err_modify_hdr: 94 bitmap_free(dm->steering_sw_icm_alloc_blocks); 95 96err_steering: 97 kfree(dm); 98 99 return ERR_PTR(-ENOMEM); 100} 101 102void mlx5_dm_cleanup(struct mlx5_core_dev *dev) 103{ 104 struct mlx5_dm *dm = dev->dm; 105 106 if (!dev->dm) 107 return; 108 109 if (dm->steering_sw_icm_alloc_blocks) { 110 WARN_ON(!bitmap_empty(dm->steering_sw_icm_alloc_blocks, 111 BIT(MLX5_CAP_DEV_MEM(dev, log_steering_sw_icm_size) - 112 MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)))); 113 bitmap_free(dm->steering_sw_icm_alloc_blocks); 114 } 115 116 if (dm->header_modify_sw_icm_alloc_blocks) { 117 WARN_ON(!bitmap_empty(dm->header_modify_sw_icm_alloc_blocks, 118 BIT(MLX5_CAP_DEV_MEM(dev, 119 log_header_modify_sw_icm_size) - 120 MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)))); 121 bitmap_free(dm->header_modify_sw_icm_alloc_blocks); 122 } 123 124 if (dm->header_encap_sw_icm_alloc_blocks) { 125 WARN_ON(!bitmap_empty(dm->header_encap_sw_icm_alloc_blocks, 126 BIT(MLX5_CAP_DEV_MEM(dev, 127 log_indirect_encap_sw_icm_size) - 128 MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)))); 129 bitmap_free(dm->header_encap_sw_icm_alloc_blocks); 130 } 131 132 if (dm->header_modify_pattern_sw_icm_alloc_blocks) { 133 WARN_ON(!bitmap_empty(dm->header_modify_pattern_sw_icm_alloc_blocks, 134 BIT(MLX5_CAP_DEV_MEM(dev, 135 log_header_modify_pattern_sw_icm_size) - 136 MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)))); 137 bitmap_free(dm->header_modify_pattern_sw_icm_alloc_blocks); 138 } 139 140 kfree(dm); 141} 142 143int mlx5_dm_sw_icm_alloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type, 144 u64 length, u32 log_alignment, u16 uid, 145 phys_addr_t *addr, u32 *obj_id) 146{ 147 u32 num_blocks = DIV_ROUND_UP_ULL(length, MLX5_SW_ICM_BLOCK_SIZE(dev)); 148 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {}; 149 u32 in[MLX5_ST_SZ_DW(create_sw_icm_in)] = {}; 150 struct mlx5_dm *dm = dev->dm; 151 unsigned long *block_map; 152 u64 icm_start_addr; 153 u32 log_icm_size; 154 u64 align_mask; 155 u32 max_blocks; 156 u64 block_idx; 157 void *sw_icm; 158 int ret; 159 160 if (!dev->dm) 161 return -EOPNOTSUPP; 162 163 if (!length || (length & (length - 1)) || 164 length & (MLX5_SW_ICM_BLOCK_SIZE(dev) - 1)) 165 return -EINVAL; 166 167 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, 168 MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 169 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_OBJ_TYPE_SW_ICM); 170 MLX5_SET(general_obj_in_cmd_hdr, in, uid, uid); 171 172 switch (type) { 173 case MLX5_SW_ICM_TYPE_STEERING: 174 icm_start_addr = MLX5_CAP64_DEV_MEM(dev, steering_sw_icm_start_address); 175 log_icm_size = MLX5_CAP_DEV_MEM(dev, log_steering_sw_icm_size); 176 block_map = dm->steering_sw_icm_alloc_blocks; 177 break; 178 case MLX5_SW_ICM_TYPE_HEADER_MODIFY: 179 icm_start_addr = MLX5_CAP64_DEV_MEM(dev, header_modify_sw_icm_start_address); 180 log_icm_size = MLX5_CAP_DEV_MEM(dev, 181 log_header_modify_sw_icm_size); 182 block_map = dm->header_modify_sw_icm_alloc_blocks; 183 break; 184 case MLX5_SW_ICM_TYPE_HEADER_MODIFY_PATTERN: 185 icm_start_addr = MLX5_CAP64_DEV_MEM(dev, 186 header_modify_pattern_sw_icm_start_address); 187 log_icm_size = MLX5_CAP_DEV_MEM(dev, 188 log_header_modify_pattern_sw_icm_size); 189 block_map = dm->header_modify_pattern_sw_icm_alloc_blocks; 190 break; 191 case MLX5_SW_ICM_TYPE_SW_ENCAP: 192 icm_start_addr = MLX5_CAP64_DEV_MEM(dev, 193 indirect_encap_sw_icm_start_address); 194 log_icm_size = MLX5_CAP_DEV_MEM(dev, 195 log_indirect_encap_sw_icm_size); 196 block_map = dm->header_encap_sw_icm_alloc_blocks; 197 break; 198 default: 199 return -EINVAL; 200 } 201 202 if (!block_map) 203 return -EOPNOTSUPP; 204 205 max_blocks = BIT(log_icm_size - MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); 206 207 if (log_alignment < MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)) 208 log_alignment = MLX5_LOG_SW_ICM_BLOCK_SIZE(dev); 209 align_mask = BIT(log_alignment - MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)) - 1; 210 211 spin_lock(&dm->lock); 212 block_idx = bitmap_find_next_zero_area(block_map, max_blocks, 0, 213 num_blocks, align_mask); 214 215 if (block_idx < max_blocks) 216 bitmap_set(block_map, 217 block_idx, num_blocks); 218 219 spin_unlock(&dm->lock); 220 221 if (block_idx >= max_blocks) 222 return -ENOMEM; 223 224 sw_icm = MLX5_ADDR_OF(create_sw_icm_in, in, sw_icm); 225 icm_start_addr += block_idx << MLX5_LOG_SW_ICM_BLOCK_SIZE(dev); 226 MLX5_SET64(sw_icm, sw_icm, sw_icm_start_addr, 227 icm_start_addr); 228 MLX5_SET(sw_icm, sw_icm, log_sw_icm_size, ilog2(length)); 229 230 ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 231 if (ret) { 232 spin_lock(&dm->lock); 233 bitmap_clear(block_map, 234 block_idx, num_blocks); 235 spin_unlock(&dm->lock); 236 237 return ret; 238 } 239 240 *addr = icm_start_addr; 241 *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 242 243 return 0; 244} 245EXPORT_SYMBOL_GPL(mlx5_dm_sw_icm_alloc); 246 247int mlx5_dm_sw_icm_dealloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type, 248 u64 length, u16 uid, phys_addr_t addr, u32 obj_id) 249{ 250 u32 num_blocks = DIV_ROUND_UP_ULL(length, MLX5_SW_ICM_BLOCK_SIZE(dev)); 251 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {}; 252 u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; 253 struct mlx5_dm *dm = dev->dm; 254 unsigned long *block_map; 255 u64 icm_start_addr; 256 u64 start_idx; 257 int err; 258 259 if (!dev->dm) 260 return -EOPNOTSUPP; 261 262 switch (type) { 263 case MLX5_SW_ICM_TYPE_STEERING: 264 icm_start_addr = MLX5_CAP64_DEV_MEM(dev, steering_sw_icm_start_address); 265 block_map = dm->steering_sw_icm_alloc_blocks; 266 break; 267 case MLX5_SW_ICM_TYPE_HEADER_MODIFY: 268 icm_start_addr = MLX5_CAP64_DEV_MEM(dev, header_modify_sw_icm_start_address); 269 block_map = dm->header_modify_sw_icm_alloc_blocks; 270 break; 271 case MLX5_SW_ICM_TYPE_HEADER_MODIFY_PATTERN: 272 icm_start_addr = MLX5_CAP64_DEV_MEM(dev, 273 header_modify_pattern_sw_icm_start_address); 274 block_map = dm->header_modify_pattern_sw_icm_alloc_blocks; 275 break; 276 case MLX5_SW_ICM_TYPE_SW_ENCAP: 277 icm_start_addr = MLX5_CAP64_DEV_MEM(dev, 278 indirect_encap_sw_icm_start_address); 279 block_map = dm->header_encap_sw_icm_alloc_blocks; 280 break; 281 default: 282 return -EINVAL; 283 } 284 285 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, 286 MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 287 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_OBJ_TYPE_SW_ICM); 288 MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id); 289 MLX5_SET(general_obj_in_cmd_hdr, in, uid, uid); 290 291 err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 292 if (err) 293 return err; 294 295 start_idx = (addr - icm_start_addr) >> MLX5_LOG_SW_ICM_BLOCK_SIZE(dev); 296 spin_lock(&dm->lock); 297 bitmap_clear(block_map, 298 start_idx, num_blocks); 299 spin_unlock(&dm->lock); 300 301 return 0; 302} 303EXPORT_SYMBOL_GPL(mlx5_dm_sw_icm_dealloc); 304