1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2// Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 4#include "dr_types.h" 5#include "mlx5_ifc_dr_ste_v1.h" 6 7enum dr_ptrn_modify_hdr_action_id { 8 DR_PTRN_MODIFY_HDR_ACTION_ID_NOP = 0x00, 9 DR_PTRN_MODIFY_HDR_ACTION_ID_COPY = 0x05, 10 DR_PTRN_MODIFY_HDR_ACTION_ID_SET = 0x06, 11 DR_PTRN_MODIFY_HDR_ACTION_ID_ADD = 0x07, 12 DR_PTRN_MODIFY_HDR_ACTION_ID_INSERT_INLINE = 0x0a, 13}; 14 15struct mlx5dr_ptrn_mgr { 16 struct mlx5dr_domain *dmn; 17 struct mlx5dr_icm_pool *ptrn_icm_pool; 18 /* cache for modify_header ptrn */ 19 struct list_head ptrn_list; 20 struct mutex modify_hdr_mutex; /* protect the pattern cache */ 21}; 22 23/* Cache structure and functions */ 24static bool dr_ptrn_compare_modify_hdr(size_t cur_num_of_actions, 25 __be64 cur_hw_actions[], 26 size_t num_of_actions, 27 __be64 hw_actions[]) 28{ 29 int i; 30 31 if (cur_num_of_actions != num_of_actions) 32 return false; 33 34 for (i = 0; i < num_of_actions; i++) { 35 u8 action_id = 36 MLX5_GET(ste_double_action_set_v1, &hw_actions[i], action_id); 37 38 if (action_id == DR_PTRN_MODIFY_HDR_ACTION_ID_COPY) { 39 if (hw_actions[i] != cur_hw_actions[i]) 40 return false; 41 } else { 42 if ((__force __be32)hw_actions[i] != 43 (__force __be32)cur_hw_actions[i]) 44 return false; 45 } 46 } 47 48 return true; 49} 50 51static struct mlx5dr_ptrn_obj * 52dr_ptrn_find_cached_pattern(struct mlx5dr_ptrn_mgr *mgr, 53 size_t num_of_actions, 54 __be64 hw_actions[]) 55{ 56 struct mlx5dr_ptrn_obj *cached_pattern; 57 struct mlx5dr_ptrn_obj *tmp; 58 59 list_for_each_entry_safe(cached_pattern, tmp, &mgr->ptrn_list, list) { 60 if (dr_ptrn_compare_modify_hdr(cached_pattern->num_of_actions, 61 (__be64 *)cached_pattern->data, 62 num_of_actions, 63 hw_actions)) { 64 /* Put this pattern in the head of the list, 65 * as we will probably use it more. 66 */ 67 list_del_init(&cached_pattern->list); 68 list_add(&cached_pattern->list, &mgr->ptrn_list); 69 return cached_pattern; 70 } 71 } 72 73 return NULL; 74} 75 76static struct mlx5dr_ptrn_obj * 77dr_ptrn_alloc_pattern(struct mlx5dr_ptrn_mgr *mgr, 78 u16 num_of_actions, u8 *data) 79{ 80 struct mlx5dr_ptrn_obj *pattern; 81 struct mlx5dr_icm_chunk *chunk; 82 u32 chunk_size; 83 u32 index; 84 85 chunk_size = ilog2(roundup_pow_of_two(num_of_actions)); 86 /* HW modify action index granularity is at least 64B */ 87 chunk_size = max_t(u32, chunk_size, DR_CHUNK_SIZE_8); 88 89 chunk = mlx5dr_icm_alloc_chunk(mgr->ptrn_icm_pool, chunk_size); 90 if (!chunk) 91 return NULL; 92 93 index = (mlx5dr_icm_pool_get_chunk_icm_addr(chunk) - 94 mgr->dmn->info.caps.hdr_modify_pattern_icm_addr) / 95 DR_ACTION_CACHE_LINE_SIZE; 96 97 pattern = kzalloc(sizeof(*pattern), GFP_KERNEL); 98 if (!pattern) 99 goto free_chunk; 100 101 pattern->data = kzalloc(num_of_actions * DR_MODIFY_ACTION_SIZE * 102 sizeof(*pattern->data), GFP_KERNEL); 103 if (!pattern->data) 104 goto free_pattern; 105 106 memcpy(pattern->data, data, num_of_actions * DR_MODIFY_ACTION_SIZE); 107 pattern->chunk = chunk; 108 pattern->index = index; 109 pattern->num_of_actions = num_of_actions; 110 111 list_add(&pattern->list, &mgr->ptrn_list); 112 refcount_set(&pattern->refcount, 1); 113 114 return pattern; 115 116free_pattern: 117 kfree(pattern); 118free_chunk: 119 mlx5dr_icm_free_chunk(chunk); 120 return NULL; 121} 122 123static void 124dr_ptrn_free_pattern(struct mlx5dr_ptrn_obj *pattern) 125{ 126 list_del(&pattern->list); 127 mlx5dr_icm_free_chunk(pattern->chunk); 128 kfree(pattern->data); 129 kfree(pattern); 130} 131 132struct mlx5dr_ptrn_obj * 133mlx5dr_ptrn_cache_get_pattern(struct mlx5dr_ptrn_mgr *mgr, 134 u16 num_of_actions, 135 u8 *data) 136{ 137 struct mlx5dr_ptrn_obj *pattern; 138 u64 *hw_actions; 139 u8 action_id; 140 int i; 141 142 mutex_lock(&mgr->modify_hdr_mutex); 143 pattern = dr_ptrn_find_cached_pattern(mgr, 144 num_of_actions, 145 (__be64 *)data); 146 if (!pattern) { 147 /* Alloc and add new pattern to cache */ 148 pattern = dr_ptrn_alloc_pattern(mgr, num_of_actions, data); 149 if (!pattern) 150 goto out_unlock; 151 152 hw_actions = (u64 *)pattern->data; 153 /* Here we mask the pattern data to create a valid pattern 154 * since we do an OR operation between the arg and pattern 155 */ 156 for (i = 0; i < num_of_actions; i++) { 157 action_id = MLX5_GET(ste_double_action_set_v1, &hw_actions[i], action_id); 158 159 if (action_id == DR_PTRN_MODIFY_HDR_ACTION_ID_SET || 160 action_id == DR_PTRN_MODIFY_HDR_ACTION_ID_ADD || 161 action_id == DR_PTRN_MODIFY_HDR_ACTION_ID_INSERT_INLINE) 162 MLX5_SET(ste_double_action_set_v1, &hw_actions[i], inline_data, 0); 163 } 164 165 if (mlx5dr_send_postsend_pattern(mgr->dmn, pattern->chunk, 166 num_of_actions, pattern->data)) { 167 refcount_dec(&pattern->refcount); 168 goto free_pattern; 169 } 170 } else { 171 refcount_inc(&pattern->refcount); 172 } 173 174 mutex_unlock(&mgr->modify_hdr_mutex); 175 176 return pattern; 177 178free_pattern: 179 dr_ptrn_free_pattern(pattern); 180out_unlock: 181 mutex_unlock(&mgr->modify_hdr_mutex); 182 return NULL; 183} 184 185void 186mlx5dr_ptrn_cache_put_pattern(struct mlx5dr_ptrn_mgr *mgr, 187 struct mlx5dr_ptrn_obj *pattern) 188{ 189 mutex_lock(&mgr->modify_hdr_mutex); 190 191 if (refcount_dec_and_test(&pattern->refcount)) 192 dr_ptrn_free_pattern(pattern); 193 194 mutex_unlock(&mgr->modify_hdr_mutex); 195} 196 197struct mlx5dr_ptrn_mgr *mlx5dr_ptrn_mgr_create(struct mlx5dr_domain *dmn) 198{ 199 struct mlx5dr_ptrn_mgr *mgr; 200 201 if (!mlx5dr_domain_is_support_ptrn_arg(dmn)) 202 return NULL; 203 204 mgr = kzalloc(sizeof(*mgr), GFP_KERNEL); 205 if (!mgr) 206 return NULL; 207 208 mgr->dmn = dmn; 209 mgr->ptrn_icm_pool = mlx5dr_icm_pool_create(dmn, DR_ICM_TYPE_MODIFY_HDR_PTRN); 210 if (!mgr->ptrn_icm_pool) { 211 mlx5dr_err(dmn, "Couldn't get modify-header-pattern memory\n"); 212 goto free_mgr; 213 } 214 215 INIT_LIST_HEAD(&mgr->ptrn_list); 216 mutex_init(&mgr->modify_hdr_mutex); 217 218 return mgr; 219 220free_mgr: 221 kfree(mgr); 222 return NULL; 223} 224 225void mlx5dr_ptrn_mgr_destroy(struct mlx5dr_ptrn_mgr *mgr) 226{ 227 struct mlx5dr_ptrn_obj *pattern; 228 struct mlx5dr_ptrn_obj *tmp; 229 230 if (!mgr) 231 return; 232 233 WARN_ON(!list_empty(&mgr->ptrn_list)); 234 235 list_for_each_entry_safe(pattern, tmp, &mgr->ptrn_list, list) { 236 list_del(&pattern->list); 237 kfree(pattern->data); 238 kfree(pattern); 239 } 240 241 mlx5dr_icm_pool_destroy(mgr->ptrn_icm_pool); 242 mutex_destroy(&mgr->modify_hdr_mutex); 243 kfree(mgr); 244} 245