cvmx-cmd-queue.h revision 210311
1210284Sjmallett/***********************license start*************** 2210284Sjmallett * Copyright (c) 2003-2008 Cavium Networks (support@cavium.com). All rights 3210284Sjmallett * reserved. 4210284Sjmallett * 5210284Sjmallett * 6210284Sjmallett * Redistribution and use in source and binary forms, with or without 7210284Sjmallett * modification, are permitted provided that the following conditions are 8210284Sjmallett * met: 9210284Sjmallett * 10210284Sjmallett * * Redistributions of source code must retain the above copyright 11210284Sjmallett * notice, this list of conditions and the following disclaimer. 12210284Sjmallett * 13210284Sjmallett * * Redistributions in binary form must reproduce the above 14210284Sjmallett * copyright notice, this list of conditions and the following 15210284Sjmallett * disclaimer in the documentation and/or other materials provided 16210284Sjmallett * with the distribution. 17210284Sjmallett * 18210284Sjmallett * * Neither the name of Cavium Networks nor the names of 19210284Sjmallett * its contributors may be used to endorse or promote products 20210284Sjmallett * derived from this software without specific prior written 21210284Sjmallett * permission. 22210284Sjmallett * 23210284Sjmallett * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" 24210284Sjmallett * AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS 25210284Sjmallett * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH 26210284Sjmallett * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY 27210284Sjmallett * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT 28210284Sjmallett * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES 29210284Sjmallett * OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR 30210284Sjmallett * PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET 31210284Sjmallett * POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT 32210284Sjmallett * OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. 33210284Sjmallett * 34210284Sjmallett * 35210284Sjmallett * For any questions regarding licensing please contact marketing@caviumnetworks.com 36210284Sjmallett * 37210284Sjmallett ***********************license end**************************************/ 38210284Sjmallett 39210284Sjmallett 40210284Sjmallett 41210284Sjmallett 42210284Sjmallett 43210284Sjmallett 44210284Sjmallett/** 45210284Sjmallett * @file 46210284Sjmallett * 47210284Sjmallett * Support functions for managing command queues used for 48210284Sjmallett * various hardware blocks. 49210284Sjmallett * 50210284Sjmallett * The common command queue infrastructure abstracts out the 51210284Sjmallett * software necessary for adding to Octeon's chained queue 52210284Sjmallett * structures. These structures are used for commands to the 53210284Sjmallett * PKO, ZIP, DFA, RAID, and DMA engine blocks. Although each 54210284Sjmallett * hardware unit takes commands and CSRs of different types, 55210284Sjmallett * they all use basic linked command buffers to store the 56210284Sjmallett * pending request. In general, users of the CVMX API don't 57210284Sjmallett * call cvmx-cmd-queue functions directly. Instead the hardware 58210284Sjmallett * unit specific wrapper should be used. The wrappers perform 59210284Sjmallett * unit specific validation and CSR writes to submit the 60210284Sjmallett * commands. 61210284Sjmallett * 62210284Sjmallett * Even though most software will never directly interact with 63210284Sjmallett * cvmx-cmd-queue, knowledge of its internal working can help 64210284Sjmallett * in diagnosing performance problems and help with debugging. 65210284Sjmallett * 66210284Sjmallett * Command queue pointers are stored in a global named block 67210284Sjmallett * called "cvmx_cmd_queues". Except for the PKO queues, each 68210284Sjmallett * hardware queue is stored in its own cache line to reduce SMP 69210284Sjmallett * contention on spin locks. The PKO queues are stored such that 70210284Sjmallett * every 16th queue is next to each other in memory. This scheme 71210284Sjmallett * allows for queues being in separate cache lines when there 72210284Sjmallett * are low number of queues per port. With 16 queues per port, 73210284Sjmallett * the first queue for each port is in the same cache area. The 74210284Sjmallett * second queues for each port are in another area, etc. This 75210284Sjmallett * allows software to implement very efficient lockless PKO with 76210284Sjmallett * 16 queues per port using a minimum of cache lines per core. 77210284Sjmallett * All queues for a given core will be isolated in the same 78210284Sjmallett * cache area. 79210284Sjmallett * 80210284Sjmallett * In addition to the memory pointer layout, cvmx-cmd-queue 81210284Sjmallett * provides an optimized fair ll/sc locking mechanism for the 82210284Sjmallett * queues. The lock uses a "ticket / now serving" model to 83210284Sjmallett * maintain fair order on contended locks. In addition, it uses 84210284Sjmallett * predicted locking time to limit cache contention. When a core 85210284Sjmallett * know it must wait in line for a lock, it spins on the 86210284Sjmallett * internal cycle counter to completely eliminate any causes of 87210284Sjmallett * bus traffic. 88210284Sjmallett * 89210284Sjmallett * <hr> $Revision: 42150 $ <hr> 90210284Sjmallett */ 91210284Sjmallett 92210284Sjmallett#ifndef __CVMX_CMD_QUEUE_H__ 93210284Sjmallett#define __CVMX_CMD_QUEUE_H__ 94210284Sjmallett 95210311Sjmallett#ifndef CVMX_DONT_INCLUDE_CONFIG 96210284Sjmallett#include "executive-config.h" 97210284Sjmallett#include "cvmx-config.h" 98210311Sjmallett#endif 99210284Sjmallett#include "cvmx-fpa.h" 100210284Sjmallett 101210284Sjmallett#ifdef __cplusplus 102210284Sjmallettextern "C" { 103210284Sjmallett#endif 104210284Sjmallett 105210284Sjmallett/** 106210284Sjmallett * By default we disable the max depth support. Most programs 107210284Sjmallett * don't use it and it slows down the command queue processing 108210284Sjmallett * significantly. 109210284Sjmallett */ 110210284Sjmallett#ifndef CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH 111210284Sjmallett#define CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH 0 112210284Sjmallett#endif 113210284Sjmallett 114210284Sjmallett/** 115210284Sjmallett * Enumeration representing all hardware blocks that use command 116210284Sjmallett * queues. Each hardware block has up to 65536 sub identifiers for 117210284Sjmallett * multiple command queues. Not all chips support all hardware 118210284Sjmallett * units. 119210284Sjmallett */ 120210284Sjmalletttypedef enum 121210284Sjmallett{ 122210284Sjmallett CVMX_CMD_QUEUE_PKO_BASE = 0x00000, 123210284Sjmallett#define CVMX_CMD_QUEUE_PKO(queue) ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_PKO_BASE + (0xffff&(queue)))) 124210284Sjmallett CVMX_CMD_QUEUE_ZIP = 0x10000, 125210284Sjmallett CVMX_CMD_QUEUE_DFA = 0x20000, 126210284Sjmallett CVMX_CMD_QUEUE_RAID = 0x30000, 127210284Sjmallett CVMX_CMD_QUEUE_DMA_BASE = 0x40000, 128210284Sjmallett#define CVMX_CMD_QUEUE_DMA(queue) ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_DMA_BASE + (0xffff&(queue)))) 129210284Sjmallett CVMX_CMD_QUEUE_END = 0x50000, 130210284Sjmallett} cvmx_cmd_queue_id_t; 131210284Sjmallett 132210284Sjmallett/** 133210284Sjmallett * Command write operations can fail if the comamnd queue needs 134210284Sjmallett * a new buffer and the associated FPA pool is empty. It can also 135210284Sjmallett * fail if the number of queued command words reaches the maximum 136210284Sjmallett * set at initialization. 137210284Sjmallett */ 138210284Sjmalletttypedef enum 139210284Sjmallett{ 140210284Sjmallett CVMX_CMD_QUEUE_SUCCESS = 0, 141210284Sjmallett CVMX_CMD_QUEUE_NO_MEMORY = -1, 142210284Sjmallett CVMX_CMD_QUEUE_FULL = -2, 143210284Sjmallett CVMX_CMD_QUEUE_INVALID_PARAM = -3, 144210284Sjmallett CVMX_CMD_QUEUE_ALREADY_SETUP = -4, 145210284Sjmallett} cvmx_cmd_queue_result_t; 146210284Sjmallett 147210284Sjmalletttypedef struct 148210284Sjmallett{ 149210284Sjmallett uint8_t now_serving; /**< You have lock when this is your ticket */ 150210284Sjmallett uint64_t unused1 : 24; 151210284Sjmallett uint32_t max_depth; /**< Maximum outstanding command words */ 152210284Sjmallett uint64_t fpa_pool : 3; /**< FPA pool buffers come from */ 153210284Sjmallett uint64_t base_ptr_div128: 29; /**< Top of command buffer pointer shifted 7 */ 154210284Sjmallett uint64_t unused2 : 6; 155210284Sjmallett uint64_t pool_size_m1 : 13; /**< FPA buffer size in 64bit words minus 1 */ 156210284Sjmallett uint64_t index : 13; /**< Number of comamnds already used in buffer */ 157210284Sjmallett} __cvmx_cmd_queue_state_t; 158210284Sjmallett 159210284Sjmallett/** 160210284Sjmallett * This structure contains the global state of all comamnd queues. 161210284Sjmallett * It is stored in a bootmem named block and shared by all 162210284Sjmallett * applications running on Octeon. Tickets are stored in a differnet 163210284Sjmallett * cahce line that queue information to reduce the contention on the 164210284Sjmallett * ll/sc used to get a ticket. If this is not the case, the update 165210284Sjmallett * of queue state causes the ll/sc to fail quite often. 166210284Sjmallett */ 167210284Sjmalletttypedef struct 168210284Sjmallett{ 169210284Sjmallett uint64_t ticket[(CVMX_CMD_QUEUE_END>>16) * 256]; 170210284Sjmallett __cvmx_cmd_queue_state_t state[(CVMX_CMD_QUEUE_END>>16) * 256]; 171210284Sjmallett} __cvmx_cmd_queue_all_state_t; 172210284Sjmallett 173210311Sjmallettextern CVMX_SHARED __cvmx_cmd_queue_all_state_t *__cvmx_cmd_queue_state_ptr; 174210311Sjmallett 175210284Sjmallett/** 176210284Sjmallett * Initialize a command queue for use. The initial FPA buffer is 177210284Sjmallett * allocated and the hardware unit is configured to point to the 178210284Sjmallett * new command queue. 179210284Sjmallett * 180210284Sjmallett * @param queue_id Hardware command queue to initialize. 181210284Sjmallett * @param max_depth Maximum outstanding commands that can be queued. 182210284Sjmallett * @param fpa_pool FPA pool the command queues should come from. 183210284Sjmallett * @param pool_size Size of each buffer in the FPA pool (bytes) 184210284Sjmallett * 185210284Sjmallett * @return CVMX_CMD_QUEUE_SUCCESS or a failure code 186210284Sjmallett */ 187210284Sjmallettcvmx_cmd_queue_result_t cvmx_cmd_queue_initialize(cvmx_cmd_queue_id_t queue_id, int max_depth, int fpa_pool, int pool_size); 188210284Sjmallett 189210284Sjmallett/** 190210284Sjmallett * Shutdown a queue a free it's command buffers to the FPA. The 191210284Sjmallett * hardware connected to the queue must be stopped before this 192210284Sjmallett * function is called. 193210284Sjmallett * 194210284Sjmallett * @param queue_id Queue to shutdown 195210284Sjmallett * 196210284Sjmallett * @return CVMX_CMD_QUEUE_SUCCESS or a failure code 197210284Sjmallett */ 198210284Sjmallettcvmx_cmd_queue_result_t cvmx_cmd_queue_shutdown(cvmx_cmd_queue_id_t queue_id); 199210284Sjmallett 200210284Sjmallett/** 201210284Sjmallett * Return the number of command words pending in the queue. This 202210284Sjmallett * function may be relatively slow for some hardware units. 203210284Sjmallett * 204210284Sjmallett * @param queue_id Hardware command queue to query 205210284Sjmallett * 206210284Sjmallett * @return Number of outstanding commands 207210284Sjmallett */ 208210284Sjmallettint cvmx_cmd_queue_length(cvmx_cmd_queue_id_t queue_id); 209210284Sjmallett 210210284Sjmallett/** 211210284Sjmallett * Return the command buffer to be written to. The purpose of this 212210284Sjmallett * function is to allow CVMX routine access t othe low level buffer 213210284Sjmallett * for initial hardware setup. User applications should not call this 214210284Sjmallett * function directly. 215210284Sjmallett * 216210284Sjmallett * @param queue_id Command queue to query 217210284Sjmallett * 218210284Sjmallett * @return Command buffer or NULL on failure 219210284Sjmallett */ 220210284Sjmallettvoid *cvmx_cmd_queue_buffer(cvmx_cmd_queue_id_t queue_id); 221210284Sjmallett 222210284Sjmallett/** 223210284Sjmallett * @INTERNAL 224210284Sjmallett * Get the index into the state arrays for the supplied queue id. 225210284Sjmallett * 226210284Sjmallett * @param queue_id Queue ID to get an index for 227210284Sjmallett * 228210284Sjmallett * @return Index into the state arrays 229210284Sjmallett */ 230210284Sjmallettstatic inline int __cvmx_cmd_queue_get_index(cvmx_cmd_queue_id_t queue_id) 231210284Sjmallett{ 232210284Sjmallett /* Warning: This code currently only works with devices that have 256 queues 233210284Sjmallett or less. Devices with more than 16 queues are layed out in memory to allow 234210284Sjmallett cores quick access to every 16th queue. This reduces cache thrashing 235210284Sjmallett when you are running 16 queues per port to support lockless operation */ 236210284Sjmallett int unit = queue_id>>16; 237210284Sjmallett int q = (queue_id >> 4) & 0xf; 238210284Sjmallett int core = queue_id & 0xf; 239210284Sjmallett return unit*256 + core*16 + q; 240210284Sjmallett} 241210284Sjmallett 242210284Sjmallett 243210284Sjmallett/** 244210284Sjmallett * @INTERNAL 245210284Sjmallett * Lock the supplied queue so nobody else is updating it at the same 246210284Sjmallett * time as us. 247210284Sjmallett * 248210284Sjmallett * @param queue_id Queue ID to lock 249210284Sjmallett * @param qptr Pointer to the queue's global state 250210284Sjmallett */ 251210284Sjmallettstatic inline void __cvmx_cmd_queue_lock(cvmx_cmd_queue_id_t queue_id, __cvmx_cmd_queue_state_t *qptr) 252210284Sjmallett{ 253210284Sjmallett int tmp; 254210284Sjmallett int my_ticket; 255210284Sjmallett CVMX_PREFETCH(qptr, 0); 256210284Sjmallett asm volatile ( 257210284Sjmallett ".set push\n" 258210284Sjmallett ".set noreorder\n" 259210284Sjmallett "1:\n" 260210284Sjmallett "ll %[my_ticket], %[ticket_ptr]\n" /* Atomic add one to ticket_ptr */ 261210284Sjmallett "li %[ticket], 1\n" /* and store the original value */ 262210284Sjmallett "baddu %[ticket], %[my_ticket]\n" /* in my_ticket */ 263210284Sjmallett "sc %[ticket], %[ticket_ptr]\n" 264210284Sjmallett "beqz %[ticket], 1b\n" 265210284Sjmallett " nop\n" 266210284Sjmallett "lbu %[ticket], %[now_serving]\n" /* Load the current now_serving ticket */ 267210284Sjmallett "2:\n" 268210284Sjmallett "beq %[ticket], %[my_ticket], 4f\n" /* Jump out if now_serving == my_ticket */ 269210284Sjmallett " subu %[ticket], %[my_ticket], %[ticket]\n" /* Find out how many tickets are in front of me */ 270210284Sjmallett "subu %[ticket], 1\n" /* Use tickets in front of me minus one to delay */ 271210284Sjmallett "cins %[ticket], %[ticket], 5, 7\n" /* Delay will be ((tickets in front)-1)*32 loops */ 272210284Sjmallett "3:\n" 273210284Sjmallett "bnez %[ticket], 3b\n" /* Loop here until our ticket might be up */ 274210284Sjmallett " subu %[ticket], 1\n" 275210284Sjmallett "b 2b\n" /* Jump back up to check out ticket again */ 276210284Sjmallett " lbu %[ticket], %[now_serving]\n" /* Load the current now_serving ticket */ 277210284Sjmallett "4:\n" 278210284Sjmallett ".set pop\n" 279210284Sjmallett : [ticket_ptr] "=m" (__cvmx_cmd_queue_state_ptr->ticket[__cvmx_cmd_queue_get_index(queue_id)]), 280210284Sjmallett [now_serving] "=m" (qptr->now_serving), 281210284Sjmallett [ticket] "=r" (tmp), 282210284Sjmallett [my_ticket] "=r" (my_ticket) 283210284Sjmallett ); 284210284Sjmallett} 285210284Sjmallett 286210284Sjmallett 287210284Sjmallett/** 288210284Sjmallett * @INTERNAL 289210284Sjmallett * Unlock the queue, flushing all writes. 290210284Sjmallett * 291210284Sjmallett * @param qptr Queue to unlock 292210284Sjmallett */ 293210284Sjmallettstatic inline void __cvmx_cmd_queue_unlock(__cvmx_cmd_queue_state_t *qptr) 294210284Sjmallett{ 295210284Sjmallett qptr->now_serving++; 296210284Sjmallett CVMX_SYNCWS; 297210284Sjmallett} 298210284Sjmallett 299210284Sjmallett 300210284Sjmallett/** 301210284Sjmallett * @INTERNAL 302210284Sjmallett * Get the queue state structure for the given queue id 303210284Sjmallett * 304210284Sjmallett * @param queue_id Queue id to get 305210284Sjmallett * 306210284Sjmallett * @return Queue structure or NULL on failure 307210284Sjmallett */ 308210284Sjmallettstatic inline __cvmx_cmd_queue_state_t *__cvmx_cmd_queue_get_state(cvmx_cmd_queue_id_t queue_id) 309210284Sjmallett{ 310210284Sjmallett if (CVMX_ENABLE_PARAMETER_CHECKING) 311210284Sjmallett { 312210284Sjmallett if (cvmx_unlikely(queue_id >= CVMX_CMD_QUEUE_END)) 313210284Sjmallett return NULL; 314210284Sjmallett if (cvmx_unlikely((queue_id & 0xffff) >= 256)) 315210284Sjmallett return NULL; 316210284Sjmallett } 317210284Sjmallett return &__cvmx_cmd_queue_state_ptr->state[__cvmx_cmd_queue_get_index(queue_id)]; 318210284Sjmallett} 319210284Sjmallett 320210284Sjmallett 321210284Sjmallett/** 322210284Sjmallett * Write an arbitrary number of command words to a command queue. 323210284Sjmallett * This is a generic function; the fixed number of comamnd word 324210284Sjmallett * functions yield higher performance. 325210284Sjmallett * 326210284Sjmallett * @param queue_id Hardware command queue to write to 327210284Sjmallett * @param use_locking 328210284Sjmallett * Use internal locking to ensure exclusive access for queue 329210284Sjmallett * updates. If you don't use this locking you must ensure 330210284Sjmallett * exclusivity some other way. Locking is strongly recommended. 331210284Sjmallett * @param cmd_count Number of command words to write 332210284Sjmallett * @param cmds Array of comamnds to write 333210284Sjmallett * 334210284Sjmallett * @return CVMX_CMD_QUEUE_SUCCESS or a failure code 335210284Sjmallett */ 336210284Sjmallettstatic inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write(cvmx_cmd_queue_id_t queue_id, int use_locking, int cmd_count, uint64_t *cmds) 337210284Sjmallett{ 338210284Sjmallett __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id); 339210284Sjmallett 340210284Sjmallett if (CVMX_ENABLE_PARAMETER_CHECKING) 341210284Sjmallett { 342210284Sjmallett if (cvmx_unlikely(qptr == NULL)) 343210284Sjmallett return CVMX_CMD_QUEUE_INVALID_PARAM; 344210284Sjmallett if (cvmx_unlikely((cmd_count < 1) || (cmd_count > 32))) 345210284Sjmallett return CVMX_CMD_QUEUE_INVALID_PARAM; 346210284Sjmallett if (cvmx_unlikely(cmds == NULL)) 347210284Sjmallett return CVMX_CMD_QUEUE_INVALID_PARAM; 348210284Sjmallett } 349210284Sjmallett 350210284Sjmallett /* Make sure nobody else is updating the same queue */ 351210284Sjmallett if (cvmx_likely(use_locking)) 352210284Sjmallett __cvmx_cmd_queue_lock(queue_id, qptr); 353210284Sjmallett 354210284Sjmallett /* If a max queue length was specified then make sure we don't 355210284Sjmallett exceed it. If any part of the command would be below the limit 356210284Sjmallett we allow it */ 357210284Sjmallett if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && cvmx_unlikely(qptr->max_depth)) 358210284Sjmallett { 359210284Sjmallett if (cvmx_unlikely(cvmx_cmd_queue_length(queue_id) > (int)qptr->max_depth)) 360210284Sjmallett { 361210284Sjmallett if (cvmx_likely(use_locking)) 362210284Sjmallett __cvmx_cmd_queue_unlock(qptr); 363210284Sjmallett return CVMX_CMD_QUEUE_FULL; 364210284Sjmallett } 365210284Sjmallett } 366210284Sjmallett 367210284Sjmallett /* Normally there is plenty of room in the current buffer for the command */ 368210284Sjmallett if (cvmx_likely(qptr->index + cmd_count < qptr->pool_size_m1)) 369210284Sjmallett { 370210284Sjmallett uint64_t *ptr = (uint64_t *)cvmx_phys_to_ptr((uint64_t)qptr->base_ptr_div128<<7); 371210284Sjmallett ptr += qptr->index; 372210284Sjmallett qptr->index += cmd_count; 373210284Sjmallett while (cmd_count--) 374210284Sjmallett *ptr++ = *cmds++; 375210284Sjmallett } 376210284Sjmallett else 377210284Sjmallett { 378210284Sjmallett uint64_t *ptr; 379210284Sjmallett int count; 380210284Sjmallett /* We need a new comamnd buffer. Fail if there isn't one available */ 381210284Sjmallett uint64_t *new_buffer = (uint64_t *)cvmx_fpa_alloc(qptr->fpa_pool); 382210284Sjmallett if (cvmx_unlikely(new_buffer == NULL)) 383210284Sjmallett { 384210284Sjmallett if (cvmx_likely(use_locking)) 385210284Sjmallett __cvmx_cmd_queue_unlock(qptr); 386210284Sjmallett return CVMX_CMD_QUEUE_NO_MEMORY; 387210284Sjmallett } 388210284Sjmallett ptr = (uint64_t *)cvmx_phys_to_ptr((uint64_t)qptr->base_ptr_div128<<7); 389210284Sjmallett /* Figure out how many command words will fit in this buffer. One 390210284Sjmallett location will be needed for the next buffer pointer */ 391210284Sjmallett count = qptr->pool_size_m1 - qptr->index; 392210284Sjmallett ptr += qptr->index; 393210284Sjmallett cmd_count-=count; 394210284Sjmallett while (count--) 395210284Sjmallett *ptr++ = *cmds++; 396210284Sjmallett *ptr = cvmx_ptr_to_phys(new_buffer); 397210284Sjmallett /* The current buffer is full and has a link to the next buffer. Time 398210284Sjmallett to write the rest of the commands into the new buffer */ 399210284Sjmallett qptr->base_ptr_div128 = *ptr >> 7; 400210284Sjmallett qptr->index = cmd_count; 401210284Sjmallett ptr = new_buffer; 402210284Sjmallett while (cmd_count--) 403210284Sjmallett *ptr++ = *cmds++; 404210284Sjmallett } 405210284Sjmallett 406210284Sjmallett /* All updates are complete. Release the lock and return */ 407210284Sjmallett if (cvmx_likely(use_locking)) 408210284Sjmallett __cvmx_cmd_queue_unlock(qptr); 409210284Sjmallett return CVMX_CMD_QUEUE_SUCCESS; 410210284Sjmallett} 411210284Sjmallett 412210284Sjmallett 413210284Sjmallett/** 414210284Sjmallett * Simple function to write two command words to a command 415210284Sjmallett * queue. 416210284Sjmallett * 417210284Sjmallett * @param queue_id Hardware command queue to write to 418210284Sjmallett * @param use_locking 419210284Sjmallett * Use internal locking to ensure exclusive access for queue 420210284Sjmallett * updates. If you don't use this locking you must ensure 421210284Sjmallett * exclusivity some other way. Locking is strongly recommended. 422210284Sjmallett * @param cmd1 Command 423210284Sjmallett * @param cmd2 Command 424210284Sjmallett * 425210284Sjmallett * @return CVMX_CMD_QUEUE_SUCCESS or a failure code 426210284Sjmallett */ 427210284Sjmallettstatic inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write2(cvmx_cmd_queue_id_t queue_id, int use_locking, uint64_t cmd1, uint64_t cmd2) 428210284Sjmallett{ 429210284Sjmallett __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id); 430210284Sjmallett 431210284Sjmallett if (CVMX_ENABLE_PARAMETER_CHECKING) 432210284Sjmallett { 433210284Sjmallett if (cvmx_unlikely(qptr == NULL)) 434210284Sjmallett return CVMX_CMD_QUEUE_INVALID_PARAM; 435210284Sjmallett } 436210284Sjmallett 437210284Sjmallett /* Make sure nobody else is updating the same queue */ 438210284Sjmallett if (cvmx_likely(use_locking)) 439210284Sjmallett __cvmx_cmd_queue_lock(queue_id, qptr); 440210284Sjmallett 441210284Sjmallett /* If a max queue length was specified then make sure we don't 442210284Sjmallett exceed it. If any part of the command would be below the limit 443210284Sjmallett we allow it */ 444210284Sjmallett if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && cvmx_unlikely(qptr->max_depth)) 445210284Sjmallett { 446210284Sjmallett if (cvmx_unlikely(cvmx_cmd_queue_length(queue_id) > (int)qptr->max_depth)) 447210284Sjmallett { 448210284Sjmallett if (cvmx_likely(use_locking)) 449210284Sjmallett __cvmx_cmd_queue_unlock(qptr); 450210284Sjmallett return CVMX_CMD_QUEUE_FULL; 451210284Sjmallett } 452210284Sjmallett } 453210284Sjmallett 454210284Sjmallett /* Normally there is plenty of room in the current buffer for the command */ 455210284Sjmallett if (cvmx_likely(qptr->index + 2 < qptr->pool_size_m1)) 456210284Sjmallett { 457210284Sjmallett uint64_t *ptr = (uint64_t *)cvmx_phys_to_ptr((uint64_t)qptr->base_ptr_div128<<7); 458210284Sjmallett ptr += qptr->index; 459210284Sjmallett qptr->index += 2; 460210284Sjmallett ptr[0] = cmd1; 461210284Sjmallett ptr[1] = cmd2; 462210284Sjmallett } 463210284Sjmallett else 464210284Sjmallett { 465210284Sjmallett uint64_t *ptr; 466210284Sjmallett /* Figure out how many command words will fit in this buffer. One 467210284Sjmallett location will be needed for the next buffer pointer */ 468210284Sjmallett int count = qptr->pool_size_m1 - qptr->index; 469210284Sjmallett /* We need a new comamnd buffer. Fail if there isn't one available */ 470210284Sjmallett uint64_t *new_buffer = (uint64_t *)cvmx_fpa_alloc(qptr->fpa_pool); 471210284Sjmallett if (cvmx_unlikely(new_buffer == NULL)) 472210284Sjmallett { 473210284Sjmallett if (cvmx_likely(use_locking)) 474210284Sjmallett __cvmx_cmd_queue_unlock(qptr); 475210284Sjmallett return CVMX_CMD_QUEUE_NO_MEMORY; 476210284Sjmallett } 477210284Sjmallett count--; 478210284Sjmallett ptr = (uint64_t *)cvmx_phys_to_ptr((uint64_t)qptr->base_ptr_div128<<7); 479210284Sjmallett ptr += qptr->index; 480210284Sjmallett *ptr++ = cmd1; 481210284Sjmallett if (cvmx_likely(count)) 482210284Sjmallett *ptr++ = cmd2; 483210284Sjmallett *ptr = cvmx_ptr_to_phys(new_buffer); 484210284Sjmallett /* The current buffer is full and has a link to the next buffer. Time 485210284Sjmallett to write the rest of the commands into the new buffer */ 486210284Sjmallett qptr->base_ptr_div128 = *ptr >> 7; 487210284Sjmallett qptr->index = 0; 488210284Sjmallett if (cvmx_unlikely(count == 0)) 489210284Sjmallett { 490210284Sjmallett qptr->index = 1; 491210284Sjmallett new_buffer[0] = cmd2; 492210284Sjmallett } 493210284Sjmallett } 494210284Sjmallett 495210284Sjmallett /* All updates are complete. Release the lock and return */ 496210284Sjmallett if (cvmx_likely(use_locking)) 497210284Sjmallett __cvmx_cmd_queue_unlock(qptr); 498210284Sjmallett return CVMX_CMD_QUEUE_SUCCESS; 499210284Sjmallett} 500210284Sjmallett 501210284Sjmallett 502210284Sjmallett/** 503210284Sjmallett * Simple function to write three command words to a command 504210284Sjmallett * queue. 505210284Sjmallett * 506210284Sjmallett * @param queue_id Hardware command queue to write to 507210284Sjmallett * @param use_locking 508210284Sjmallett * Use internal locking to ensure exclusive access for queue 509210284Sjmallett * updates. If you don't use this locking you must ensure 510210284Sjmallett * exclusivity some other way. Locking is strongly recommended. 511210284Sjmallett * @param cmd1 Command 512210284Sjmallett * @param cmd2 Command 513210284Sjmallett * @param cmd3 Command 514210284Sjmallett * 515210284Sjmallett * @return CVMX_CMD_QUEUE_SUCCESS or a failure code 516210284Sjmallett */ 517210284Sjmallettstatic inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write3(cvmx_cmd_queue_id_t queue_id, int use_locking, uint64_t cmd1, uint64_t cmd2, uint64_t cmd3) 518210284Sjmallett{ 519210284Sjmallett __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id); 520210284Sjmallett 521210284Sjmallett if (CVMX_ENABLE_PARAMETER_CHECKING) 522210284Sjmallett { 523210284Sjmallett if (cvmx_unlikely(qptr == NULL)) 524210284Sjmallett return CVMX_CMD_QUEUE_INVALID_PARAM; 525210284Sjmallett } 526210284Sjmallett 527210284Sjmallett /* Make sure nobody else is updating the same queue */ 528210284Sjmallett if (cvmx_likely(use_locking)) 529210284Sjmallett __cvmx_cmd_queue_lock(queue_id, qptr); 530210284Sjmallett 531210284Sjmallett /* If a max queue length was specified then make sure we don't 532210284Sjmallett exceed it. If any part of the command would be below the limit 533210284Sjmallett we allow it */ 534210284Sjmallett if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && cvmx_unlikely(qptr->max_depth)) 535210284Sjmallett { 536210284Sjmallett if (cvmx_unlikely(cvmx_cmd_queue_length(queue_id) > (int)qptr->max_depth)) 537210284Sjmallett { 538210284Sjmallett if (cvmx_likely(use_locking)) 539210284Sjmallett __cvmx_cmd_queue_unlock(qptr); 540210284Sjmallett return CVMX_CMD_QUEUE_FULL; 541210284Sjmallett } 542210284Sjmallett } 543210284Sjmallett 544210284Sjmallett /* Normally there is plenty of room in the current buffer for the command */ 545210284Sjmallett if (cvmx_likely(qptr->index + 3 < qptr->pool_size_m1)) 546210284Sjmallett { 547210284Sjmallett uint64_t *ptr = (uint64_t *)cvmx_phys_to_ptr((uint64_t)qptr->base_ptr_div128<<7); 548210284Sjmallett ptr += qptr->index; 549210284Sjmallett qptr->index += 3; 550210284Sjmallett ptr[0] = cmd1; 551210284Sjmallett ptr[1] = cmd2; 552210284Sjmallett ptr[2] = cmd3; 553210284Sjmallett } 554210284Sjmallett else 555210284Sjmallett { 556210284Sjmallett uint64_t *ptr; 557210284Sjmallett /* Figure out how many command words will fit in this buffer. One 558210284Sjmallett location will be needed for the next buffer pointer */ 559210284Sjmallett int count = qptr->pool_size_m1 - qptr->index; 560210284Sjmallett /* We need a new comamnd buffer. Fail if there isn't one available */ 561210284Sjmallett uint64_t *new_buffer = (uint64_t *)cvmx_fpa_alloc(qptr->fpa_pool); 562210284Sjmallett if (cvmx_unlikely(new_buffer == NULL)) 563210284Sjmallett { 564210284Sjmallett if (cvmx_likely(use_locking)) 565210284Sjmallett __cvmx_cmd_queue_unlock(qptr); 566210284Sjmallett return CVMX_CMD_QUEUE_NO_MEMORY; 567210284Sjmallett } 568210284Sjmallett count--; 569210284Sjmallett ptr = (uint64_t *)cvmx_phys_to_ptr((uint64_t)qptr->base_ptr_div128<<7); 570210284Sjmallett ptr += qptr->index; 571210284Sjmallett *ptr++ = cmd1; 572210284Sjmallett if (count) 573210284Sjmallett { 574210284Sjmallett *ptr++ = cmd2; 575210284Sjmallett if (count > 1) 576210284Sjmallett *ptr++ = cmd3; 577210284Sjmallett } 578210284Sjmallett *ptr = cvmx_ptr_to_phys(new_buffer); 579210284Sjmallett /* The current buffer is full and has a link to the next buffer. Time 580210284Sjmallett to write the rest of the commands into the new buffer */ 581210284Sjmallett qptr->base_ptr_div128 = *ptr >> 7; 582210284Sjmallett qptr->index = 0; 583210284Sjmallett ptr = new_buffer; 584210284Sjmallett if (count == 0) 585210284Sjmallett { 586210284Sjmallett *ptr++ = cmd2; 587210284Sjmallett qptr->index++; 588210284Sjmallett } 589210284Sjmallett if (count < 2) 590210284Sjmallett { 591210284Sjmallett *ptr++ = cmd3; 592210284Sjmallett qptr->index++; 593210284Sjmallett } 594210284Sjmallett } 595210284Sjmallett 596210284Sjmallett /* All updates are complete. Release the lock and return */ 597210284Sjmallett if (cvmx_likely(use_locking)) 598210284Sjmallett __cvmx_cmd_queue_unlock(qptr); 599210284Sjmallett return CVMX_CMD_QUEUE_SUCCESS; 600210284Sjmallett} 601210284Sjmallett 602210284Sjmallett#ifdef __cplusplus 603210284Sjmallett} 604210284Sjmallett#endif 605210284Sjmallett 606210284Sjmallett#endif /* __CVMX_CMD_QUEUE_H__ */ 607