1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright (C) 2018-2022 Marvell International Ltd. 4 * 5 * Support functions for managing command queues used for 6 * various hardware blocks. 7 */ 8 9#include <errno.h> 10#include <log.h> 11#include <time.h> 12#include <linux/delay.h> 13 14#include <mach/cvmx-regs.h> 15#include <mach/cvmx-csr.h> 16#include <mach/cvmx-bootmem.h> 17#include <mach/octeon-model.h> 18#include <mach/cvmx-fuse.h> 19#include <mach/octeon-feature.h> 20#include <mach/cvmx-qlm.h> 21#include <mach/octeon_qlm.h> 22#include <mach/cvmx-pcie.h> 23#include <mach/cvmx-coremask.h> 24 25#include <mach/cvmx-fpa.h> 26#include <mach/cvmx-cmd-queue.h> 27 28#include <mach/cvmx-agl-defs.h> 29#include <mach/cvmx-bgxx-defs.h> 30#include <mach/cvmx-ciu-defs.h> 31#include <mach/cvmx-gmxx-defs.h> 32#include <mach/cvmx-gserx-defs.h> 33#include <mach/cvmx-ilk-defs.h> 34#include <mach/cvmx-ipd-defs.h> 35#include <mach/cvmx-pcsx-defs.h> 36#include <mach/cvmx-pcsxx-defs.h> 37#include <mach/cvmx-pki-defs.h> 38#include <mach/cvmx-pko-defs.h> 39#include <mach/cvmx-xcv-defs.h> 40 41#include <mach/cvmx-hwpko.h> 42#include <mach/cvmx-ilk.h> 43#include <mach/cvmx-pki.h> 44#include <mach/cvmx-pko3.h> 45#include <mach/cvmx-pko3-queue.h> 46#include <mach/cvmx-pko3-resources.h> 47 48#include <mach/cvmx-helper.h> 49#include <mach/cvmx-helper-board.h> 50#include <mach/cvmx-helper-cfg.h> 51 52#include <mach/cvmx-helper-bgx.h> 53#include <mach/cvmx-helper-cfg.h> 54#include <mach/cvmx-helper-util.h> 55#include <mach/cvmx-helper-pki.h> 56 57#include <mach/cvmx-helper-util.h> 58#include <mach/cvmx-dpi-defs.h> 59#include <mach/cvmx-npei-defs.h> 60#include <mach/cvmx-pexp-defs.h> 61 62/** 63 * This application uses this pointer to access the global queue 64 * state. It points to a bootmem named block. 65 */ 66__cvmx_cmd_queue_all_state_t *__cvmx_cmd_queue_state_ptrs[CVMX_MAX_NODES]; 67 68/** 69 * @INTERNAL 70 * Initialize the Global queue state pointer. 71 * 72 * @return CVMX_CMD_QUEUE_SUCCESS or a failure code 73 */ 74cvmx_cmd_queue_result_t __cvmx_cmd_queue_init_state_ptr(unsigned int node) 75{ 76 const char *alloc_name = "cvmx_cmd_queues\0\0"; 77 char s[4] = "_0"; 78 const struct cvmx_bootmem_named_block_desc *block_desc = NULL; 79 unsigned int size; 80 u64 paddr_min = 0, paddr_max = 0; 81 void *ptr; 82 83 if (cvmx_likely(__cvmx_cmd_queue_state_ptrs[node])) 84 return CVMX_CMD_QUEUE_SUCCESS; 85 86 /* Add node# to block name */ 87 if (node > 0) { 88 s[1] += node; 89 strcat((char *)alloc_name, s); 90 } 91 92 /* Find the named block in case it has been created already */ 93 block_desc = cvmx_bootmem_find_named_block(alloc_name); 94 if (block_desc) { 95 __cvmx_cmd_queue_state_ptrs[node] = 96 (__cvmx_cmd_queue_all_state_t *)cvmx_phys_to_ptr( 97 block_desc->base_addr); 98 return CVMX_CMD_QUEUE_SUCCESS; 99 } 100 101 size = sizeof(*__cvmx_cmd_queue_state_ptrs[node]); 102 103 /* Rest f the code is to allocate a new named block */ 104 105 /* Atomically allocate named block once, and zero it by default */ 106 ptr = cvmx_bootmem_alloc_named_range_once(size, paddr_min, paddr_max, 107 128, alloc_name, NULL); 108 109 if (ptr) { 110 __cvmx_cmd_queue_state_ptrs[node] = 111 (__cvmx_cmd_queue_all_state_t *)ptr; 112 } else { 113 debug("ERROR: %s: Unable to get named block %s.\n", __func__, 114 alloc_name); 115 return CVMX_CMD_QUEUE_NO_MEMORY; 116 } 117 return CVMX_CMD_QUEUE_SUCCESS; 118} 119 120/** 121 * Initialize a command queue for use. The initial FPA buffer is 122 * allocated and the hardware unit is configured to point to the 123 * new command queue. 124 * 125 * @param queue_id Hardware command queue to initialize. 126 * @param max_depth Maximum outstanding commands that can be queued. 127 * @param fpa_pool FPA pool the command queues should come from. 128 * @param pool_size Size of each buffer in the FPA pool (bytes) 129 * 130 * @return CVMX_CMD_QUEUE_SUCCESS or a failure code 131 */ 132cvmx_cmd_queue_result_t cvmx_cmd_queue_initialize(cvmx_cmd_queue_id_t queue_id, 133 int max_depth, int fpa_pool, 134 int pool_size) 135{ 136 __cvmx_cmd_queue_state_t *qstate; 137 cvmx_cmd_queue_result_t result; 138 unsigned int node; 139 unsigned int index; 140 int fpa_pool_min, fpa_pool_max; 141 union cvmx_fpa_ctl_status status; 142 void *buffer; 143 144 node = __cvmx_cmd_queue_get_node(queue_id); 145 146 index = __cvmx_cmd_queue_get_index(queue_id); 147 if (index >= NUM_ELEMENTS(__cvmx_cmd_queue_state_ptrs[node]->state)) { 148 printf("ERROR: %s: queue %#x out of range\n", __func__, 149 queue_id); 150 return CVMX_CMD_QUEUE_INVALID_PARAM; 151 } 152 153 result = __cvmx_cmd_queue_init_state_ptr(node); 154 if (result != CVMX_CMD_QUEUE_SUCCESS) 155 return result; 156 157 qstate = __cvmx_cmd_queue_get_state(queue_id); 158 if (!qstate) 159 return CVMX_CMD_QUEUE_INVALID_PARAM; 160 161 /* 162 * We artificially limit max_depth to 1<<20 words. It is an 163 * arbitrary limit. 164 */ 165 if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH) { 166 if (max_depth < 0 || max_depth > 1 << 20) 167 return CVMX_CMD_QUEUE_INVALID_PARAM; 168 } else if (max_depth != 0) { 169 return CVMX_CMD_QUEUE_INVALID_PARAM; 170 } 171 172 /* CVMX_FPA_NUM_POOLS maps to cvmx_fpa3_num_auras for FPA3 */ 173 fpa_pool_min = node << 10; 174 fpa_pool_max = fpa_pool_min + CVMX_FPA_NUM_POOLS; 175 176 if (fpa_pool < fpa_pool_min || fpa_pool >= fpa_pool_max) 177 return CVMX_CMD_QUEUE_INVALID_PARAM; 178 179 if (pool_size < 128 || pool_size > (1 << 17)) 180 return CVMX_CMD_QUEUE_INVALID_PARAM; 181 182 if (pool_size & 3) 183 debug("WARNING: %s: pool_size %d not multiple of 8\n", __func__, 184 pool_size); 185 186 /* See if someone else has already initialized the queue */ 187 if (qstate->base_paddr) { 188 int depth; 189 static const char emsg[] = /* Common error message part */ 190 "Queue already initialized with different "; 191 192 depth = (max_depth + qstate->pool_size_m1 - 1) / 193 qstate->pool_size_m1; 194 if (depth != qstate->max_depth) { 195 depth = qstate->max_depth * qstate->pool_size_m1; 196 debug("ERROR: %s: %s max_depth (%d).\n", __func__, emsg, 197 depth); 198 return CVMX_CMD_QUEUE_INVALID_PARAM; 199 } 200 if (fpa_pool != qstate->fpa_pool) { 201 debug("ERROR: %s: %s FPA pool (%d).\n", __func__, emsg, 202 (int)qstate->fpa_pool); 203 return CVMX_CMD_QUEUE_INVALID_PARAM; 204 } 205 if ((pool_size >> 3) - 1 != qstate->pool_size_m1) { 206 debug("ERROR: %s: %s FPA pool size (%u).\n", __func__, 207 emsg, (qstate->pool_size_m1 + 1) << 3); 208 return CVMX_CMD_QUEUE_INVALID_PARAM; 209 } 210 return CVMX_CMD_QUEUE_ALREADY_SETUP; 211 } 212 213 if (!(octeon_has_feature(OCTEON_FEATURE_FPA3))) { 214 status.u64 = csr_rd(CVMX_FPA_CTL_STATUS); 215 if (!status.s.enb) { 216 debug("ERROR: %s: FPA is not enabled.\n", 217 __func__); 218 return CVMX_CMD_QUEUE_NO_MEMORY; 219 } 220 } 221 buffer = cvmx_fpa_alloc(fpa_pool); 222 if (!buffer) { 223 debug("ERROR: %s: allocating first buffer.\n", __func__); 224 return CVMX_CMD_QUEUE_NO_MEMORY; 225 } 226 227 index = (pool_size >> 3) - 1; 228 qstate->pool_size_m1 = index; 229 qstate->max_depth = (max_depth + index - 1) / index; 230 qstate->index = 0; 231 qstate->fpa_pool = fpa_pool; 232 qstate->base_paddr = cvmx_ptr_to_phys(buffer); 233 234 /* Initialize lock */ 235 __cvmx_cmd_queue_lock_init(queue_id); 236 return CVMX_CMD_QUEUE_SUCCESS; 237} 238 239/** 240 * Return the command buffer to be written to. The purpose of this 241 * function is to allow CVMX routine access to the low level buffer 242 * for initial hardware setup. User applications should not call this 243 * function directly. 244 * 245 * @param queue_id Command queue to query 246 * 247 * @return Command buffer or NULL on failure 248 */ 249void *cvmx_cmd_queue_buffer(cvmx_cmd_queue_id_t queue_id) 250{ 251 __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id); 252 253 if (qptr && qptr->base_paddr) 254 return cvmx_phys_to_ptr((u64)qptr->base_paddr); 255 else 256 return NULL; 257} 258 259static u64 *__cvmx_cmd_queue_add_blk(__cvmx_cmd_queue_state_t *qptr) 260{ 261 u64 *cmd_ptr; 262 u64 *new_buffer; 263 u64 new_paddr; 264 265 /* Get base vaddr of current (full) block */ 266 cmd_ptr = (u64 *)cvmx_phys_to_ptr((u64)qptr->base_paddr); 267 268 /* Allocate a new block from the per-queue pool */ 269 new_buffer = (u64 *)cvmx_fpa_alloc(qptr->fpa_pool); 270 271 /* Check for allocation failure */ 272 if (cvmx_unlikely(!new_buffer)) 273 return NULL; 274 275 /* Zero out the new block link pointer, 276 * in case this block will be filled to the rim 277 */ 278 new_buffer[qptr->pool_size_m1] = ~0ull; 279 280 /* Get physical address of the new buffer */ 281 new_paddr = cvmx_ptr_to_phys(new_buffer); 282 283 /* Store the physical link address at the end of current full block */ 284 cmd_ptr[qptr->pool_size_m1] = new_paddr; 285 286 /* Store the physical address in the queue state structure */ 287 qptr->base_paddr = new_paddr; 288 qptr->index = 0; 289 290 /* Return the virtual base of the new block */ 291 return new_buffer; 292} 293 294/** 295 * @INTERNAL 296 * Add command words into a queue, handles all the corener cases 297 * where only some of the words might fit into the current block, 298 * and a new block may need to be allocated. 299 * Locking and argument checks are done in the front-end in-line 300 * functions that call this one for the rare corner cases. 301 */ 302cvmx_cmd_queue_result_t 303__cvmx_cmd_queue_write_raw(cvmx_cmd_queue_id_t queue_id, 304 __cvmx_cmd_queue_state_t *qptr, int cmd_count, 305 const u64 *cmds) 306{ 307 u64 *cmd_ptr; 308 unsigned int index; 309 310 cmd_ptr = (u64 *)cvmx_phys_to_ptr((u64)qptr->base_paddr); 311 index = qptr->index; 312 313 /* Enforce queue depth limit, if enabled, once per block */ 314 if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && cvmx_unlikely(qptr->max_depth)) { 315 unsigned int depth = cvmx_cmd_queue_length(queue_id); 316 317 depth /= qptr->pool_size_m1; 318 319 if (cvmx_unlikely(depth > qptr->max_depth)) 320 return CVMX_CMD_QUEUE_FULL; 321 } 322 323 /* 324 * If the block allocation fails, even the words that we wrote 325 * to the current block will not count because the 'index' will 326 * not be comitted. 327 * The loop is run 'count + 1' times to take care of the tail 328 * case, where the buffer is full to the rim, so the link 329 * pointer must be filled with a valid address. 330 */ 331 while (cmd_count >= 0) { 332 if (index >= qptr->pool_size_m1) { 333 /* Block is full, get another one and proceed */ 334 cmd_ptr = __cvmx_cmd_queue_add_blk(qptr); 335 336 /* Baul on allocation error w/o comitting anything */ 337 if (cvmx_unlikely(!cmd_ptr)) 338 return CVMX_CMD_QUEUE_NO_MEMORY; 339 340 /* Reset index for start of new block */ 341 index = 0; 342 } 343 /* Exit Loop on 'count + 1' iterations */ 344 if (cmd_count <= 0) 345 break; 346 /* Store commands into queue block while there is space */ 347 cmd_ptr[index++] = *cmds++; 348 cmd_count--; 349 } /* while cmd_count */ 350 351 /* Commit added words if all is well */ 352 qptr->index = index; 353 354 return CVMX_CMD_QUEUE_SUCCESS; 355} 356