1210284Sjmallett/***********************license start*************** 2232812Sjmallett * Copyright (c) 2003-2010 Cavium Inc. (support@cavium.com). All rights 3215990Sjmallett * reserved. 4210284Sjmallett * 5210284Sjmallett * 6215990Sjmallett * Redistribution and use in source and binary forms, with or without 7215990Sjmallett * modification, are permitted provided that the following conditions are 8215990Sjmallett * met: 9210284Sjmallett * 10215990Sjmallett * * Redistributions of source code must retain the above copyright 11215990Sjmallett * notice, this list of conditions and the following disclaimer. 12210284Sjmallett * 13215990Sjmallett * * Redistributions in binary form must reproduce the above 14215990Sjmallett * copyright notice, this list of conditions and the following 15215990Sjmallett * disclaimer in the documentation and/or other materials provided 16215990Sjmallett * with the distribution. 17215990Sjmallett 18232812Sjmallett * * Neither the name of Cavium Inc. nor the names of 19215990Sjmallett * its contributors may be used to endorse or promote products 20215990Sjmallett * derived from this software without specific prior written 21215990Sjmallett * permission. 22215990Sjmallett 23215990Sjmallett * This Software, including technical data, may be subject to U.S. export control 24215990Sjmallett * laws, including the U.S. Export Administration Act and its associated 25215990Sjmallett * regulations, and may be subject to export or import regulations in other 26215990Sjmallett * countries. 27215990Sjmallett 28215990Sjmallett * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" 29232812Sjmallett * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR 30215990Sjmallett * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO 31215990Sjmallett * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR 32215990Sjmallett * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM 33215990Sjmallett * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, 34215990Sjmallett * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF 35215990Sjmallett * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR 36215990Sjmallett * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR 37215990Sjmallett * PERFORMANCE OF THE SOFTWARE LIES WITH YOU. 38210284Sjmallett ***********************license end**************************************/ 39210284Sjmallett 40210284Sjmallett 41210284Sjmallett 42210284Sjmallett 43210284Sjmallett 44210284Sjmallett 45215990Sjmallett 46210284Sjmallett/** 47210284Sjmallett * @file 48210284Sjmallett * 49210284Sjmallett * Support functions for managing command queues used for 50210284Sjmallett * various hardware blocks. 51210284Sjmallett * 52210284Sjmallett * The common command queue infrastructure abstracts out the 53210284Sjmallett * software necessary for adding to Octeon's chained queue 54210284Sjmallett * structures. These structures are used for commands to the 55210284Sjmallett * PKO, ZIP, DFA, RAID, and DMA engine blocks. Although each 56210284Sjmallett * hardware unit takes commands and CSRs of different types, 57210284Sjmallett * they all use basic linked command buffers to store the 58210284Sjmallett * pending request. In general, users of the CVMX API don't 59210284Sjmallett * call cvmx-cmd-queue functions directly. Instead the hardware 60210284Sjmallett * unit specific wrapper should be used. The wrappers perform 61210284Sjmallett * unit specific validation and CSR writes to submit the 62210284Sjmallett * commands. 63210284Sjmallett * 64210284Sjmallett * Even though most software will never directly interact with 65215990Sjmallett * cvmx-cmd-queue, knowledge of its internal workings can help 66210284Sjmallett * in diagnosing performance problems and help with debugging. 67210284Sjmallett * 68210284Sjmallett * Command queue pointers are stored in a global named block 69210284Sjmallett * called "cvmx_cmd_queues". Except for the PKO queues, each 70210284Sjmallett * hardware queue is stored in its own cache line to reduce SMP 71210284Sjmallett * contention on spin locks. The PKO queues are stored such that 72210284Sjmallett * every 16th queue is next to each other in memory. This scheme 73210284Sjmallett * allows for queues being in separate cache lines when there 74210284Sjmallett * are low number of queues per port. With 16 queues per port, 75210284Sjmallett * the first queue for each port is in the same cache area. The 76210284Sjmallett * second queues for each port are in another area, etc. This 77210284Sjmallett * allows software to implement very efficient lockless PKO with 78210284Sjmallett * 16 queues per port using a minimum of cache lines per core. 79210284Sjmallett * All queues for a given core will be isolated in the same 80210284Sjmallett * cache area. 81210284Sjmallett * 82210284Sjmallett * In addition to the memory pointer layout, cvmx-cmd-queue 83210284Sjmallett * provides an optimized fair ll/sc locking mechanism for the 84210284Sjmallett * queues. The lock uses a "ticket / now serving" model to 85210284Sjmallett * maintain fair order on contended locks. In addition, it uses 86210284Sjmallett * predicted locking time to limit cache contention. When a core 87210284Sjmallett * know it must wait in line for a lock, it spins on the 88210284Sjmallett * internal cycle counter to completely eliminate any causes of 89210284Sjmallett * bus traffic. 90210284Sjmallett * 91232812Sjmallett * <hr> $Revision: 70030 $ <hr> 92210284Sjmallett */ 93210284Sjmallett 94210284Sjmallett#ifndef __CVMX_CMD_QUEUE_H__ 95210284Sjmallett#define __CVMX_CMD_QUEUE_H__ 96210284Sjmallett 97215990Sjmallett#if !defined(CVMX_BUILD_FOR_LINUX_KERNEL) && !defined(CVMX_BUILD_FOR_FREEBSD_KERNEL) 98210284Sjmallett#include "executive-config.h" 99210284Sjmallett#include "cvmx-config.h" 100210311Sjmallett#endif 101215990Sjmallett 102210284Sjmallett#include "cvmx-fpa.h" 103210284Sjmallett 104210284Sjmallett#ifdef __cplusplus 105210284Sjmallettextern "C" { 106210284Sjmallett#endif 107210284Sjmallett 108210284Sjmallett/** 109210284Sjmallett * By default we disable the max depth support. Most programs 110210284Sjmallett * don't use it and it slows down the command queue processing 111210284Sjmallett * significantly. 112210284Sjmallett */ 113210284Sjmallett#ifndef CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH 114210284Sjmallett#define CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH 0 115210284Sjmallett#endif 116210284Sjmallett 117210284Sjmallett/** 118210284Sjmallett * Enumeration representing all hardware blocks that use command 119210284Sjmallett * queues. Each hardware block has up to 65536 sub identifiers for 120210284Sjmallett * multiple command queues. Not all chips support all hardware 121210284Sjmallett * units. 122210284Sjmallett */ 123210284Sjmalletttypedef enum 124210284Sjmallett{ 125210284Sjmallett CVMX_CMD_QUEUE_PKO_BASE = 0x00000, 126210284Sjmallett#define CVMX_CMD_QUEUE_PKO(queue) ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_PKO_BASE + (0xffff&(queue)))) 127210284Sjmallett CVMX_CMD_QUEUE_ZIP = 0x10000, 128232812Sjmallett#define CVMX_CMD_QUEUE_ZIP_QUE(queue) ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_ZIP + (0xffff&(queue)))) 129210284Sjmallett CVMX_CMD_QUEUE_DFA = 0x20000, 130210284Sjmallett CVMX_CMD_QUEUE_RAID = 0x30000, 131210284Sjmallett CVMX_CMD_QUEUE_DMA_BASE = 0x40000, 132210284Sjmallett#define CVMX_CMD_QUEUE_DMA(queue) ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_DMA_BASE + (0xffff&(queue)))) 133210284Sjmallett CVMX_CMD_QUEUE_END = 0x50000, 134210284Sjmallett} cvmx_cmd_queue_id_t; 135210284Sjmallett 136210284Sjmallett/** 137215990Sjmallett * Command write operations can fail if the command queue needs 138210284Sjmallett * a new buffer and the associated FPA pool is empty. It can also 139210284Sjmallett * fail if the number of queued command words reaches the maximum 140210284Sjmallett * set at initialization. 141210284Sjmallett */ 142210284Sjmalletttypedef enum 143210284Sjmallett{ 144210284Sjmallett CVMX_CMD_QUEUE_SUCCESS = 0, 145210284Sjmallett CVMX_CMD_QUEUE_NO_MEMORY = -1, 146210284Sjmallett CVMX_CMD_QUEUE_FULL = -2, 147210284Sjmallett CVMX_CMD_QUEUE_INVALID_PARAM = -3, 148210284Sjmallett CVMX_CMD_QUEUE_ALREADY_SETUP = -4, 149210284Sjmallett} cvmx_cmd_queue_result_t; 150210284Sjmallett 151210284Sjmalletttypedef struct 152210284Sjmallett{ 153210284Sjmallett uint8_t now_serving; /**< You have lock when this is your ticket */ 154210284Sjmallett uint64_t unused1 : 24; 155210284Sjmallett uint32_t max_depth; /**< Maximum outstanding command words */ 156210284Sjmallett uint64_t fpa_pool : 3; /**< FPA pool buffers come from */ 157210284Sjmallett uint64_t base_ptr_div128: 29; /**< Top of command buffer pointer shifted 7 */ 158210284Sjmallett uint64_t unused2 : 6; 159210284Sjmallett uint64_t pool_size_m1 : 13; /**< FPA buffer size in 64bit words minus 1 */ 160215990Sjmallett uint64_t index : 13; /**< Number of commands already used in buffer */ 161210284Sjmallett} __cvmx_cmd_queue_state_t; 162210284Sjmallett 163210284Sjmallett/** 164215990Sjmallett * This structure contains the global state of all command queues. 165210284Sjmallett * It is stored in a bootmem named block and shared by all 166215990Sjmallett * applications running on Octeon. Tickets are stored in a different 167215990Sjmallett * cache line that queue information to reduce the contention on the 168210284Sjmallett * ll/sc used to get a ticket. If this is not the case, the update 169210284Sjmallett * of queue state causes the ll/sc to fail quite often. 170210284Sjmallett */ 171210284Sjmalletttypedef struct 172210284Sjmallett{ 173210284Sjmallett uint64_t ticket[(CVMX_CMD_QUEUE_END>>16) * 256]; 174210284Sjmallett __cvmx_cmd_queue_state_t state[(CVMX_CMD_QUEUE_END>>16) * 256]; 175210284Sjmallett} __cvmx_cmd_queue_all_state_t; 176210284Sjmallett 177210311Sjmallettextern CVMX_SHARED __cvmx_cmd_queue_all_state_t *__cvmx_cmd_queue_state_ptr; 178210311Sjmallett 179210284Sjmallett/** 180210284Sjmallett * Initialize a command queue for use. The initial FPA buffer is 181210284Sjmallett * allocated and the hardware unit is configured to point to the 182210284Sjmallett * new command queue. 183210284Sjmallett * 184210284Sjmallett * @param queue_id Hardware command queue to initialize. 185210284Sjmallett * @param max_depth Maximum outstanding commands that can be queued. 186210284Sjmallett * @param fpa_pool FPA pool the command queues should come from. 187210284Sjmallett * @param pool_size Size of each buffer in the FPA pool (bytes) 188210284Sjmallett * 189210284Sjmallett * @return CVMX_CMD_QUEUE_SUCCESS or a failure code 190210284Sjmallett */ 191210284Sjmallettcvmx_cmd_queue_result_t cvmx_cmd_queue_initialize(cvmx_cmd_queue_id_t queue_id, int max_depth, int fpa_pool, int pool_size); 192210284Sjmallett 193210284Sjmallett/** 194210284Sjmallett * Shutdown a queue a free it's command buffers to the FPA. The 195210284Sjmallett * hardware connected to the queue must be stopped before this 196210284Sjmallett * function is called. 197210284Sjmallett * 198210284Sjmallett * @param queue_id Queue to shutdown 199210284Sjmallett * 200210284Sjmallett * @return CVMX_CMD_QUEUE_SUCCESS or a failure code 201210284Sjmallett */ 202210284Sjmallettcvmx_cmd_queue_result_t cvmx_cmd_queue_shutdown(cvmx_cmd_queue_id_t queue_id); 203210284Sjmallett 204210284Sjmallett/** 205210284Sjmallett * Return the number of command words pending in the queue. This 206210284Sjmallett * function may be relatively slow for some hardware units. 207210284Sjmallett * 208210284Sjmallett * @param queue_id Hardware command queue to query 209210284Sjmallett * 210210284Sjmallett * @return Number of outstanding commands 211210284Sjmallett */ 212210284Sjmallettint cvmx_cmd_queue_length(cvmx_cmd_queue_id_t queue_id); 213210284Sjmallett 214210284Sjmallett/** 215210284Sjmallett * Return the command buffer to be written to. The purpose of this 216215990Sjmallett * function is to allow CVMX routine access to the low level buffer 217210284Sjmallett * for initial hardware setup. User applications should not call this 218210284Sjmallett * function directly. 219210284Sjmallett * 220210284Sjmallett * @param queue_id Command queue to query 221210284Sjmallett * 222210284Sjmallett * @return Command buffer or NULL on failure 223210284Sjmallett */ 224210284Sjmallettvoid *cvmx_cmd_queue_buffer(cvmx_cmd_queue_id_t queue_id); 225210284Sjmallett 226210284Sjmallett/** 227210284Sjmallett * @INTERNAL 228210284Sjmallett * Get the index into the state arrays for the supplied queue id. 229210284Sjmallett * 230210284Sjmallett * @param queue_id Queue ID to get an index for 231210284Sjmallett * 232210284Sjmallett * @return Index into the state arrays 233210284Sjmallett */ 234210284Sjmallettstatic inline int __cvmx_cmd_queue_get_index(cvmx_cmd_queue_id_t queue_id) 235210284Sjmallett{ 236210284Sjmallett /* Warning: This code currently only works with devices that have 256 queues 237232812Sjmallett or less. Devices with more than 16 queues are laid out in memory to allow 238210284Sjmallett cores quick access to every 16th queue. This reduces cache thrashing 239210284Sjmallett when you are running 16 queues per port to support lockless operation */ 240210284Sjmallett int unit = queue_id>>16; 241210284Sjmallett int q = (queue_id >> 4) & 0xf; 242210284Sjmallett int core = queue_id & 0xf; 243210284Sjmallett return unit*256 + core*16 + q; 244210284Sjmallett} 245210284Sjmallett 246210284Sjmallett 247210284Sjmallett/** 248210284Sjmallett * @INTERNAL 249210284Sjmallett * Lock the supplied queue so nobody else is updating it at the same 250210284Sjmallett * time as us. 251210284Sjmallett * 252210284Sjmallett * @param queue_id Queue ID to lock 253210284Sjmallett * @param qptr Pointer to the queue's global state 254210284Sjmallett */ 255210284Sjmallettstatic inline void __cvmx_cmd_queue_lock(cvmx_cmd_queue_id_t queue_id, __cvmx_cmd_queue_state_t *qptr) 256210284Sjmallett{ 257210284Sjmallett int tmp; 258210284Sjmallett int my_ticket; 259210284Sjmallett CVMX_PREFETCH(qptr, 0); 260210284Sjmallett asm volatile ( 261210284Sjmallett ".set push\n" 262210284Sjmallett ".set noreorder\n" 263210284Sjmallett "1:\n" 264210284Sjmallett "ll %[my_ticket], %[ticket_ptr]\n" /* Atomic add one to ticket_ptr */ 265210284Sjmallett "li %[ticket], 1\n" /* and store the original value */ 266210284Sjmallett "baddu %[ticket], %[my_ticket]\n" /* in my_ticket */ 267210284Sjmallett "sc %[ticket], %[ticket_ptr]\n" 268210284Sjmallett "beqz %[ticket], 1b\n" 269210284Sjmallett " nop\n" 270210284Sjmallett "lbu %[ticket], %[now_serving]\n" /* Load the current now_serving ticket */ 271210284Sjmallett "2:\n" 272210284Sjmallett "beq %[ticket], %[my_ticket], 4f\n" /* Jump out if now_serving == my_ticket */ 273210284Sjmallett " subu %[ticket], %[my_ticket], %[ticket]\n" /* Find out how many tickets are in front of me */ 274210284Sjmallett "subu %[ticket], 1\n" /* Use tickets in front of me minus one to delay */ 275210284Sjmallett "cins %[ticket], %[ticket], 5, 7\n" /* Delay will be ((tickets in front)-1)*32 loops */ 276210284Sjmallett "3:\n" 277210284Sjmallett "bnez %[ticket], 3b\n" /* Loop here until our ticket might be up */ 278210284Sjmallett " subu %[ticket], 1\n" 279210284Sjmallett "b 2b\n" /* Jump back up to check out ticket again */ 280210284Sjmallett " lbu %[ticket], %[now_serving]\n" /* Load the current now_serving ticket */ 281210284Sjmallett "4:\n" 282210284Sjmallett ".set pop\n" 283210284Sjmallett : [ticket_ptr] "=m" (__cvmx_cmd_queue_state_ptr->ticket[__cvmx_cmd_queue_get_index(queue_id)]), 284210284Sjmallett [now_serving] "=m" (qptr->now_serving), 285210284Sjmallett [ticket] "=r" (tmp), 286210284Sjmallett [my_ticket] "=r" (my_ticket) 287210284Sjmallett ); 288210284Sjmallett} 289210284Sjmallett 290210284Sjmallett 291210284Sjmallett/** 292210284Sjmallett * @INTERNAL 293210284Sjmallett * Unlock the queue, flushing all writes. 294210284Sjmallett * 295210284Sjmallett * @param qptr Queue to unlock 296210284Sjmallett */ 297210284Sjmallettstatic inline void __cvmx_cmd_queue_unlock(__cvmx_cmd_queue_state_t *qptr) 298210284Sjmallett{ 299215990Sjmallett uint8_t ns; 300215990Sjmallett 301215990Sjmallett ns = qptr->now_serving + 1; 302215990Sjmallett CVMX_SYNCWS; /* Order queue manipulation with respect to the unlock. */ 303215990Sjmallett qptr->now_serving = ns; 304215990Sjmallett CVMX_SYNCWS; /* nudge out the unlock. */ 305210284Sjmallett} 306210284Sjmallett 307210284Sjmallett 308210284Sjmallett/** 309210284Sjmallett * @INTERNAL 310210284Sjmallett * Get the queue state structure for the given queue id 311210284Sjmallett * 312210284Sjmallett * @param queue_id Queue id to get 313210284Sjmallett * 314210284Sjmallett * @return Queue structure or NULL on failure 315210284Sjmallett */ 316210284Sjmallettstatic inline __cvmx_cmd_queue_state_t *__cvmx_cmd_queue_get_state(cvmx_cmd_queue_id_t queue_id) 317210284Sjmallett{ 318210284Sjmallett if (CVMX_ENABLE_PARAMETER_CHECKING) 319210284Sjmallett { 320210284Sjmallett if (cvmx_unlikely(queue_id >= CVMX_CMD_QUEUE_END)) 321210284Sjmallett return NULL; 322210284Sjmallett if (cvmx_unlikely((queue_id & 0xffff) >= 256)) 323210284Sjmallett return NULL; 324210284Sjmallett } 325210284Sjmallett return &__cvmx_cmd_queue_state_ptr->state[__cvmx_cmd_queue_get_index(queue_id)]; 326210284Sjmallett} 327210284Sjmallett 328210284Sjmallett 329210284Sjmallett/** 330210284Sjmallett * Write an arbitrary number of command words to a command queue. 331215990Sjmallett * This is a generic function; the fixed number of command word 332210284Sjmallett * functions yield higher performance. 333210284Sjmallett * 334210284Sjmallett * @param queue_id Hardware command queue to write to 335210284Sjmallett * @param use_locking 336210284Sjmallett * Use internal locking to ensure exclusive access for queue 337210284Sjmallett * updates. If you don't use this locking you must ensure 338210284Sjmallett * exclusivity some other way. Locking is strongly recommended. 339210284Sjmallett * @param cmd_count Number of command words to write 340215990Sjmallett * @param cmds Array of commands to write 341210284Sjmallett * 342210284Sjmallett * @return CVMX_CMD_QUEUE_SUCCESS or a failure code 343210284Sjmallett */ 344210284Sjmallettstatic inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write(cvmx_cmd_queue_id_t queue_id, int use_locking, int cmd_count, uint64_t *cmds) 345210284Sjmallett{ 346210284Sjmallett __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id); 347210284Sjmallett 348210284Sjmallett if (CVMX_ENABLE_PARAMETER_CHECKING) 349210284Sjmallett { 350210284Sjmallett if (cvmx_unlikely(qptr == NULL)) 351210284Sjmallett return CVMX_CMD_QUEUE_INVALID_PARAM; 352210284Sjmallett if (cvmx_unlikely((cmd_count < 1) || (cmd_count > 32))) 353210284Sjmallett return CVMX_CMD_QUEUE_INVALID_PARAM; 354210284Sjmallett if (cvmx_unlikely(cmds == NULL)) 355210284Sjmallett return CVMX_CMD_QUEUE_INVALID_PARAM; 356210284Sjmallett } 357210284Sjmallett 358210284Sjmallett /* Make sure nobody else is updating the same queue */ 359210284Sjmallett if (cvmx_likely(use_locking)) 360210284Sjmallett __cvmx_cmd_queue_lock(queue_id, qptr); 361210284Sjmallett 362210284Sjmallett /* If a max queue length was specified then make sure we don't 363210284Sjmallett exceed it. If any part of the command would be below the limit 364210284Sjmallett we allow it */ 365210284Sjmallett if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && cvmx_unlikely(qptr->max_depth)) 366210284Sjmallett { 367210284Sjmallett if (cvmx_unlikely(cvmx_cmd_queue_length(queue_id) > (int)qptr->max_depth)) 368210284Sjmallett { 369210284Sjmallett if (cvmx_likely(use_locking)) 370210284Sjmallett __cvmx_cmd_queue_unlock(qptr); 371210284Sjmallett return CVMX_CMD_QUEUE_FULL; 372210284Sjmallett } 373210284Sjmallett } 374210284Sjmallett 375210284Sjmallett /* Normally there is plenty of room in the current buffer for the command */ 376210284Sjmallett if (cvmx_likely(qptr->index + cmd_count < qptr->pool_size_m1)) 377210284Sjmallett { 378210284Sjmallett uint64_t *ptr = (uint64_t *)cvmx_phys_to_ptr((uint64_t)qptr->base_ptr_div128<<7); 379210284Sjmallett ptr += qptr->index; 380210284Sjmallett qptr->index += cmd_count; 381210284Sjmallett while (cmd_count--) 382210284Sjmallett *ptr++ = *cmds++; 383210284Sjmallett } 384210284Sjmallett else 385210284Sjmallett { 386210284Sjmallett uint64_t *ptr; 387210284Sjmallett int count; 388215990Sjmallett /* We need a new command buffer. Fail if there isn't one available */ 389210284Sjmallett uint64_t *new_buffer = (uint64_t *)cvmx_fpa_alloc(qptr->fpa_pool); 390210284Sjmallett if (cvmx_unlikely(new_buffer == NULL)) 391210284Sjmallett { 392210284Sjmallett if (cvmx_likely(use_locking)) 393210284Sjmallett __cvmx_cmd_queue_unlock(qptr); 394210284Sjmallett return CVMX_CMD_QUEUE_NO_MEMORY; 395210284Sjmallett } 396210284Sjmallett ptr = (uint64_t *)cvmx_phys_to_ptr((uint64_t)qptr->base_ptr_div128<<7); 397210284Sjmallett /* Figure out how many command words will fit in this buffer. One 398210284Sjmallett location will be needed for the next buffer pointer */ 399210284Sjmallett count = qptr->pool_size_m1 - qptr->index; 400210284Sjmallett ptr += qptr->index; 401210284Sjmallett cmd_count-=count; 402210284Sjmallett while (count--) 403210284Sjmallett *ptr++ = *cmds++; 404210284Sjmallett *ptr = cvmx_ptr_to_phys(new_buffer); 405210284Sjmallett /* The current buffer is full and has a link to the next buffer. Time 406210284Sjmallett to write the rest of the commands into the new buffer */ 407210284Sjmallett qptr->base_ptr_div128 = *ptr >> 7; 408210284Sjmallett qptr->index = cmd_count; 409210284Sjmallett ptr = new_buffer; 410210284Sjmallett while (cmd_count--) 411210284Sjmallett *ptr++ = *cmds++; 412210284Sjmallett } 413210284Sjmallett 414210284Sjmallett /* All updates are complete. Release the lock and return */ 415210284Sjmallett if (cvmx_likely(use_locking)) 416210284Sjmallett __cvmx_cmd_queue_unlock(qptr); 417210284Sjmallett return CVMX_CMD_QUEUE_SUCCESS; 418210284Sjmallett} 419210284Sjmallett 420210284Sjmallett 421210284Sjmallett/** 422210284Sjmallett * Simple function to write two command words to a command 423210284Sjmallett * queue. 424210284Sjmallett * 425210284Sjmallett * @param queue_id Hardware command queue to write to 426210284Sjmallett * @param use_locking 427210284Sjmallett * Use internal locking to ensure exclusive access for queue 428210284Sjmallett * updates. If you don't use this locking you must ensure 429210284Sjmallett * exclusivity some other way. Locking is strongly recommended. 430210284Sjmallett * @param cmd1 Command 431210284Sjmallett * @param cmd2 Command 432210284Sjmallett * 433210284Sjmallett * @return CVMX_CMD_QUEUE_SUCCESS or a failure code 434210284Sjmallett */ 435210284Sjmallettstatic inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write2(cvmx_cmd_queue_id_t queue_id, int use_locking, uint64_t cmd1, uint64_t cmd2) 436210284Sjmallett{ 437210284Sjmallett __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id); 438210284Sjmallett 439210284Sjmallett if (CVMX_ENABLE_PARAMETER_CHECKING) 440210284Sjmallett { 441210284Sjmallett if (cvmx_unlikely(qptr == NULL)) 442210284Sjmallett return CVMX_CMD_QUEUE_INVALID_PARAM; 443210284Sjmallett } 444210284Sjmallett 445210284Sjmallett /* Make sure nobody else is updating the same queue */ 446210284Sjmallett if (cvmx_likely(use_locking)) 447210284Sjmallett __cvmx_cmd_queue_lock(queue_id, qptr); 448210284Sjmallett 449210284Sjmallett /* If a max queue length was specified then make sure we don't 450210284Sjmallett exceed it. If any part of the command would be below the limit 451210284Sjmallett we allow it */ 452210284Sjmallett if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && cvmx_unlikely(qptr->max_depth)) 453210284Sjmallett { 454210284Sjmallett if (cvmx_unlikely(cvmx_cmd_queue_length(queue_id) > (int)qptr->max_depth)) 455210284Sjmallett { 456210284Sjmallett if (cvmx_likely(use_locking)) 457210284Sjmallett __cvmx_cmd_queue_unlock(qptr); 458210284Sjmallett return CVMX_CMD_QUEUE_FULL; 459210284Sjmallett } 460210284Sjmallett } 461210284Sjmallett 462210284Sjmallett /* Normally there is plenty of room in the current buffer for the command */ 463210284Sjmallett if (cvmx_likely(qptr->index + 2 < qptr->pool_size_m1)) 464210284Sjmallett { 465210284Sjmallett uint64_t *ptr = (uint64_t *)cvmx_phys_to_ptr((uint64_t)qptr->base_ptr_div128<<7); 466210284Sjmallett ptr += qptr->index; 467210284Sjmallett qptr->index += 2; 468210284Sjmallett ptr[0] = cmd1; 469210284Sjmallett ptr[1] = cmd2; 470210284Sjmallett } 471210284Sjmallett else 472210284Sjmallett { 473210284Sjmallett uint64_t *ptr; 474210284Sjmallett /* Figure out how many command words will fit in this buffer. One 475210284Sjmallett location will be needed for the next buffer pointer */ 476210284Sjmallett int count = qptr->pool_size_m1 - qptr->index; 477215990Sjmallett /* We need a new command buffer. Fail if there isn't one available */ 478210284Sjmallett uint64_t *new_buffer = (uint64_t *)cvmx_fpa_alloc(qptr->fpa_pool); 479210284Sjmallett if (cvmx_unlikely(new_buffer == NULL)) 480210284Sjmallett { 481210284Sjmallett if (cvmx_likely(use_locking)) 482210284Sjmallett __cvmx_cmd_queue_unlock(qptr); 483210284Sjmallett return CVMX_CMD_QUEUE_NO_MEMORY; 484210284Sjmallett } 485210284Sjmallett count--; 486210284Sjmallett ptr = (uint64_t *)cvmx_phys_to_ptr((uint64_t)qptr->base_ptr_div128<<7); 487210284Sjmallett ptr += qptr->index; 488210284Sjmallett *ptr++ = cmd1; 489210284Sjmallett if (cvmx_likely(count)) 490210284Sjmallett *ptr++ = cmd2; 491210284Sjmallett *ptr = cvmx_ptr_to_phys(new_buffer); 492210284Sjmallett /* The current buffer is full and has a link to the next buffer. Time 493210284Sjmallett to write the rest of the commands into the new buffer */ 494210284Sjmallett qptr->base_ptr_div128 = *ptr >> 7; 495210284Sjmallett qptr->index = 0; 496210284Sjmallett if (cvmx_unlikely(count == 0)) 497210284Sjmallett { 498210284Sjmallett qptr->index = 1; 499210284Sjmallett new_buffer[0] = cmd2; 500210284Sjmallett } 501210284Sjmallett } 502210284Sjmallett 503210284Sjmallett /* All updates are complete. Release the lock and return */ 504210284Sjmallett if (cvmx_likely(use_locking)) 505210284Sjmallett __cvmx_cmd_queue_unlock(qptr); 506210284Sjmallett return CVMX_CMD_QUEUE_SUCCESS; 507210284Sjmallett} 508210284Sjmallett 509210284Sjmallett 510210284Sjmallett/** 511210284Sjmallett * Simple function to write three command words to a command 512210284Sjmallett * queue. 513210284Sjmallett * 514210284Sjmallett * @param queue_id Hardware command queue to write to 515210284Sjmallett * @param use_locking 516210284Sjmallett * Use internal locking to ensure exclusive access for queue 517210284Sjmallett * updates. If you don't use this locking you must ensure 518210284Sjmallett * exclusivity some other way. Locking is strongly recommended. 519210284Sjmallett * @param cmd1 Command 520210284Sjmallett * @param cmd2 Command 521210284Sjmallett * @param cmd3 Command 522210284Sjmallett * 523210284Sjmallett * @return CVMX_CMD_QUEUE_SUCCESS or a failure code 524210284Sjmallett */ 525210284Sjmallettstatic inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write3(cvmx_cmd_queue_id_t queue_id, int use_locking, uint64_t cmd1, uint64_t cmd2, uint64_t cmd3) 526210284Sjmallett{ 527210284Sjmallett __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id); 528210284Sjmallett 529210284Sjmallett if (CVMX_ENABLE_PARAMETER_CHECKING) 530210284Sjmallett { 531210284Sjmallett if (cvmx_unlikely(qptr == NULL)) 532210284Sjmallett return CVMX_CMD_QUEUE_INVALID_PARAM; 533210284Sjmallett } 534210284Sjmallett 535210284Sjmallett /* Make sure nobody else is updating the same queue */ 536210284Sjmallett if (cvmx_likely(use_locking)) 537210284Sjmallett __cvmx_cmd_queue_lock(queue_id, qptr); 538210284Sjmallett 539210284Sjmallett /* If a max queue length was specified then make sure we don't 540210284Sjmallett exceed it. If any part of the command would be below the limit 541210284Sjmallett we allow it */ 542210284Sjmallett if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && cvmx_unlikely(qptr->max_depth)) 543210284Sjmallett { 544210284Sjmallett if (cvmx_unlikely(cvmx_cmd_queue_length(queue_id) > (int)qptr->max_depth)) 545210284Sjmallett { 546210284Sjmallett if (cvmx_likely(use_locking)) 547210284Sjmallett __cvmx_cmd_queue_unlock(qptr); 548210284Sjmallett return CVMX_CMD_QUEUE_FULL; 549210284Sjmallett } 550210284Sjmallett } 551210284Sjmallett 552210284Sjmallett /* Normally there is plenty of room in the current buffer for the command */ 553210284Sjmallett if (cvmx_likely(qptr->index + 3 < qptr->pool_size_m1)) 554210284Sjmallett { 555210284Sjmallett uint64_t *ptr = (uint64_t *)cvmx_phys_to_ptr((uint64_t)qptr->base_ptr_div128<<7); 556210284Sjmallett ptr += qptr->index; 557210284Sjmallett qptr->index += 3; 558210284Sjmallett ptr[0] = cmd1; 559210284Sjmallett ptr[1] = cmd2; 560210284Sjmallett ptr[2] = cmd3; 561210284Sjmallett } 562210284Sjmallett else 563210284Sjmallett { 564210284Sjmallett uint64_t *ptr; 565210284Sjmallett /* Figure out how many command words will fit in this buffer. One 566210284Sjmallett location will be needed for the next buffer pointer */ 567210284Sjmallett int count = qptr->pool_size_m1 - qptr->index; 568215990Sjmallett /* We need a new command buffer. Fail if there isn't one available */ 569210284Sjmallett uint64_t *new_buffer = (uint64_t *)cvmx_fpa_alloc(qptr->fpa_pool); 570210284Sjmallett if (cvmx_unlikely(new_buffer == NULL)) 571210284Sjmallett { 572210284Sjmallett if (cvmx_likely(use_locking)) 573210284Sjmallett __cvmx_cmd_queue_unlock(qptr); 574210284Sjmallett return CVMX_CMD_QUEUE_NO_MEMORY; 575210284Sjmallett } 576210284Sjmallett count--; 577210284Sjmallett ptr = (uint64_t *)cvmx_phys_to_ptr((uint64_t)qptr->base_ptr_div128<<7); 578210284Sjmallett ptr += qptr->index; 579210284Sjmallett *ptr++ = cmd1; 580210284Sjmallett if (count) 581210284Sjmallett { 582210284Sjmallett *ptr++ = cmd2; 583210284Sjmallett if (count > 1) 584210284Sjmallett *ptr++ = cmd3; 585210284Sjmallett } 586210284Sjmallett *ptr = cvmx_ptr_to_phys(new_buffer); 587210284Sjmallett /* The current buffer is full and has a link to the next buffer. Time 588210284Sjmallett to write the rest of the commands into the new buffer */ 589210284Sjmallett qptr->base_ptr_div128 = *ptr >> 7; 590210284Sjmallett qptr->index = 0; 591210284Sjmallett ptr = new_buffer; 592210284Sjmallett if (count == 0) 593210284Sjmallett { 594210284Sjmallett *ptr++ = cmd2; 595210284Sjmallett qptr->index++; 596210284Sjmallett } 597210284Sjmallett if (count < 2) 598210284Sjmallett { 599210284Sjmallett *ptr++ = cmd3; 600210284Sjmallett qptr->index++; 601210284Sjmallett } 602210284Sjmallett } 603210284Sjmallett 604210284Sjmallett /* All updates are complete. Release the lock and return */ 605210284Sjmallett if (cvmx_likely(use_locking)) 606210284Sjmallett __cvmx_cmd_queue_unlock(qptr); 607210284Sjmallett return CVMX_CMD_QUEUE_SUCCESS; 608210284Sjmallett} 609210284Sjmallett 610210284Sjmallett#ifdef __cplusplus 611210284Sjmallett} 612210284Sjmallett#endif 613210284Sjmallett 614210284Sjmallett#endif /* __CVMX_CMD_QUEUE_H__ */ 615