1210284Sjmallett/***********************license start*************** 2232812Sjmallett * Copyright (c) 2003-2010 Cavium Inc. (support@cavium.com). All rights 3215990Sjmallett * reserved. 4210284Sjmallett * 5210284Sjmallett * 6215990Sjmallett * Redistribution and use in source and binary forms, with or without 7215990Sjmallett * modification, are permitted provided that the following conditions are 8215990Sjmallett * met: 9210284Sjmallett * 10215990Sjmallett * * Redistributions of source code must retain the above copyright 11215990Sjmallett * notice, this list of conditions and the following disclaimer. 12210284Sjmallett * 13215990Sjmallett * * Redistributions in binary form must reproduce the above 14215990Sjmallett * copyright notice, this list of conditions and the following 15215990Sjmallett * disclaimer in the documentation and/or other materials provided 16215990Sjmallett * with the distribution. 17210284Sjmallett 18232812Sjmallett * * Neither the name of Cavium Inc. nor the names of 19215990Sjmallett * its contributors may be used to endorse or promote products 20215990Sjmallett * derived from this software without specific prior written 21215990Sjmallett * permission. 22210284Sjmallett 23215990Sjmallett * This Software, including technical data, may be subject to U.S. export control 24215990Sjmallett * laws, including the U.S. Export Administration Act and its associated 25215990Sjmallett * regulations, and may be subject to export or import regulations in other 26215990Sjmallett * countries. 27210284Sjmallett 28215990Sjmallett * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" 29232812Sjmallett * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR 30215990Sjmallett * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO 31215990Sjmallett * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR 32215990Sjmallett * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM 33215990Sjmallett * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, 34215990Sjmallett * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF 35215990Sjmallett * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR 36215990Sjmallett * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR 37215990Sjmallett * PERFORMANCE OF THE SOFTWARE LIES WITH YOU. 38215990Sjmallett ***********************license end**************************************/ 39210284Sjmallett 40210284Sjmallett 41210284Sjmallett/** 42210284Sjmallett * @file 43210284Sjmallett * 44210284Sjmallett * Interface to the hardware Packet Output unit. 45210284Sjmallett * 46210284Sjmallett * Starting with SDK 1.7.0, the PKO output functions now support 47210284Sjmallett * two types of locking. CVMX_PKO_LOCK_ATOMIC_TAG continues to 48210284Sjmallett * function similarly to previous SDKs by using POW atomic tags 49210284Sjmallett * to preserve ordering and exclusivity. As a new option, you 50210284Sjmallett * can now pass CVMX_PKO_LOCK_CMD_QUEUE which uses a ll/sc 51210284Sjmallett * memory based locking instead. This locking has the advantage 52210284Sjmallett * of not affecting the tag state but doesn't preserve packet 53210284Sjmallett * ordering. CVMX_PKO_LOCK_CMD_QUEUE is appropriate in most 54210284Sjmallett * generic code while CVMX_PKO_LOCK_CMD_QUEUE should be used 55210284Sjmallett * with hand tuned fast path code. 56210284Sjmallett * 57210284Sjmallett * Some of other SDK differences visible to the command command 58210284Sjmallett * queuing: 59210284Sjmallett * - PKO indexes are no longer stored in the FAU. A large 60210284Sjmallett * percentage of the FAU register block used to be tied up 61210284Sjmallett * maintaining PKO queue pointers. These are now stored in a 62210284Sjmallett * global named block. 63210284Sjmallett * - The PKO <b>use_locking</b> parameter can now have a global 64210284Sjmallett * effect. Since all application use the same named block, 65210284Sjmallett * queue locking correctly applies across all operating 66210284Sjmallett * systems when using CVMX_PKO_LOCK_CMD_QUEUE. 67210284Sjmallett * - PKO 3 word commands are now supported. Use 68210284Sjmallett * cvmx_pko_send_packet_finish3(). 69210284Sjmallett * 70232812Sjmallett * <hr>$Revision: 70030 $<hr> 71210284Sjmallett */ 72210284Sjmallett 73210284Sjmallett 74210284Sjmallett#ifndef __CVMX_PKO_H__ 75210284Sjmallett#define __CVMX_PKO_H__ 76210284Sjmallett 77215990Sjmallett#ifdef CVMX_BUILD_FOR_LINUX_KERNEL 78210284Sjmallett#include "cvmx-config.h" 79215990Sjmallett#include "cvmx-pko-defs.h" 80232812Sjmallett#include <asm/octeon/cvmx-fau.h> 81232812Sjmallett#include <asm/octeon/cvmx-fpa.h> 82232812Sjmallett#include <asm/octeon/cvmx-pow.h> 83232812Sjmallett#include <asm/octeon/cvmx-cmd-queue.h> 84232812Sjmallett#include <asm/octeon/cvmx-helper.h> 85232812Sjmallett#include <asm/octeon/cvmx-helper-cfg.h> 86215990Sjmallett#else 87215990Sjmallett# ifndef CVMX_DONT_INCLUDE_CONFIG 88215990Sjmallett# include "executive-config.h" 89215990Sjmallett# ifdef CVMX_ENABLE_PKO_FUNCTIONS 90215990Sjmallett# include "cvmx-config.h" 91215990Sjmallett# endif 92215990Sjmallett# endif 93210284Sjmallett#include "cvmx-fau.h" 94210284Sjmallett#include "cvmx-fpa.h" 95210284Sjmallett#include "cvmx-pow.h" 96210284Sjmallett#include "cvmx-cmd-queue.h" 97232812Sjmallett#include "cvmx-helper.h" 98232812Sjmallett#include "cvmx-helper-util.h" 99232812Sjmallett#include "cvmx-helper-cfg.h" 100232812Sjmallett#endif 101210284Sjmallett 102210284Sjmallett/* Adjust the command buffer size by 1 word so that in the case of using only 103210284Sjmallett** two word PKO commands no command words stradle buffers. The useful values 104210284Sjmallett** for this are 0 and 1. */ 105210284Sjmallett#define CVMX_PKO_COMMAND_BUFFER_SIZE_ADJUST (1) 106210284Sjmallett 107210284Sjmallett#ifdef __cplusplus 108210284Sjmallettextern "C" { 109210284Sjmallett#endif 110210284Sjmallett 111210284Sjmallett#define CVMX_PKO_MAX_OUTPUT_QUEUES_STATIC 256 112232812Sjmallett#define CVMX_PKO_MAX_OUTPUT_QUEUES ((OCTEON_IS_MODEL(OCTEON_CN31XX) || \ 113232812Sjmallett OCTEON_IS_MODEL(OCTEON_CN3010) || \ 114232812Sjmallett OCTEON_IS_MODEL(OCTEON_CN3005) || \ 115232812Sjmallett OCTEON_IS_MODEL(OCTEON_CN50XX)) ? \ 116232812Sjmallett 32 : \ 117232812Sjmallett (OCTEON_IS_MODEL(OCTEON_CN58XX) || \ 118232812Sjmallett OCTEON_IS_MODEL(OCTEON_CN56XX) || \ 119232812Sjmallett OCTEON_IS_MODEL(OCTEON_CN52XX) || \ 120232812Sjmallett OCTEON_IS_MODEL(OCTEON_CN6XXX) || \ 121232812Sjmallett OCTEON_IS_MODEL(OCTEON_CNF7XXX)) ? \ 122232812Sjmallett 256 : 128) 123232812Sjmallett#define CVMX_PKO_NUM_OUTPUT_PORTS ((OCTEON_IS_MODEL(OCTEON_CN63XX)) ? 44 : (OCTEON_IS_MODEL(OCTEON_CN66XX) ? 46 : 40)) 124215990Sjmallett#define CVMX_PKO_MEM_QUEUE_PTRS_ILLEGAL_PID 63 /* use this for queues that are not used */ 125210284Sjmallett#define CVMX_PKO_QUEUE_STATIC_PRIORITY 9 126210284Sjmallett#define CVMX_PKO_ILLEGAL_QUEUE 0xFFFF 127210284Sjmallett#define CVMX_PKO_MAX_QUEUE_DEPTH 0 128210284Sjmallett 129210284Sjmalletttypedef enum 130210284Sjmallett{ 131210284Sjmallett CVMX_PKO_SUCCESS, 132210284Sjmallett CVMX_PKO_INVALID_PORT, 133210284Sjmallett CVMX_PKO_INVALID_QUEUE, 134210284Sjmallett CVMX_PKO_INVALID_PRIORITY, 135210284Sjmallett CVMX_PKO_NO_MEMORY, 136210284Sjmallett CVMX_PKO_PORT_ALREADY_SETUP, 137210284Sjmallett CVMX_PKO_CMD_QUEUE_INIT_ERROR 138210284Sjmallett} cvmx_pko_status_t; 139210284Sjmallett 140210284Sjmallett/** 141210284Sjmallett * This enumeration represents the differnet locking modes supported by PKO. 142210284Sjmallett */ 143210284Sjmalletttypedef enum 144210284Sjmallett{ 145210284Sjmallett CVMX_PKO_LOCK_NONE = 0, /**< PKO doesn't do any locking. It is the responsibility 146210284Sjmallett of the application to make sure that no other core is 147232812Sjmallett accessing the same queue at the same time */ 148210284Sjmallett CVMX_PKO_LOCK_ATOMIC_TAG = 1, /**< PKO performs an atomic tagswitch to insure exclusive 149210284Sjmallett access to the output queue. This will maintain 150210284Sjmallett packet ordering on output */ 151210284Sjmallett CVMX_PKO_LOCK_CMD_QUEUE = 2, /**< PKO uses the common command queue locks to insure 152210284Sjmallett exclusive access to the output queue. This is a memory 153210284Sjmallett based ll/sc. This is the most portable locking 154210284Sjmallett mechanism */ 155210284Sjmallett} cvmx_pko_lock_t; 156210284Sjmallett 157210284Sjmalletttypedef struct 158210284Sjmallett{ 159232812Sjmallett uint32_t packets; 160232812Sjmallett uint64_t octets; 161232812Sjmallett uint64_t doorbell; 162210284Sjmallett} cvmx_pko_port_status_t; 163210284Sjmallett 164210284Sjmallett/** 165210284Sjmallett * This structure defines the address to use on a packet enqueue 166210284Sjmallett */ 167210284Sjmalletttypedef union 168210284Sjmallett{ 169210284Sjmallett uint64_t u64; 170210284Sjmallett struct 171210284Sjmallett { 172210284Sjmallett cvmx_mips_space_t mem_space : 2; /**< Must CVMX_IO_SEG */ 173210284Sjmallett uint64_t reserved :13; /**< Must be zero */ 174210284Sjmallett uint64_t is_io : 1; /**< Must be one */ 175210284Sjmallett uint64_t did : 8; /**< The ID of the device on the non-coherent bus */ 176210284Sjmallett uint64_t reserved2 : 4; /**< Must be zero */ 177232812Sjmallett uint64_t reserved3 :15; /**< Must be zero */ 178232812Sjmallett uint64_t port : 9; /**< The hardware must have the output port in addition to the output queue */ 179210284Sjmallett uint64_t queue : 9; /**< The output queue to send the packet to (0-127 are legal) */ 180210284Sjmallett uint64_t reserved4 : 3; /**< Must be zero */ 181210284Sjmallett } s; 182210284Sjmallett} cvmx_pko_doorbell_address_t; 183210284Sjmallett 184210284Sjmallett/** 185210284Sjmallett * Structure of the first packet output command word. 186210284Sjmallett */ 187210284Sjmalletttypedef union 188210284Sjmallett{ 189210284Sjmallett uint64_t u64; 190210284Sjmallett struct 191210284Sjmallett { 192210284Sjmallett cvmx_fau_op_size_t size1 : 2; /**< The size of the reg1 operation - could be 8, 16, 32, or 64 bits */ 193210284Sjmallett cvmx_fau_op_size_t size0 : 2; /**< The size of the reg0 operation - could be 8, 16, 32, or 64 bits */ 194210284Sjmallett uint64_t subone1 : 1; /**< If set, subtract 1, if clear, subtract packet size */ 195210284Sjmallett uint64_t reg1 :11; /**< The register, subtract will be done if reg1 is non-zero */ 196210284Sjmallett uint64_t subone0 : 1; /**< If set, subtract 1, if clear, subtract packet size */ 197210284Sjmallett uint64_t reg0 :11; /**< The register, subtract will be done if reg0 is non-zero */ 198210284Sjmallett uint64_t le : 1; /**< When set, interpret segment pointer and segment bytes in little endian order */ 199210284Sjmallett uint64_t n2 : 1; /**< When set, packet data not allocated in L2 cache by PKO */ 200210284Sjmallett uint64_t wqp : 1; /**< If set and rsp is set, word3 contains a pointer to a work queue entry */ 201210284Sjmallett uint64_t rsp : 1; /**< If set, the hardware will send a response when done */ 202210284Sjmallett uint64_t gather : 1; /**< If set, the supplied pkt_ptr is really a pointer to a list of pkt_ptr's */ 203210284Sjmallett uint64_t ipoffp1 : 7; /**< If ipoffp1 is non zero, (ipoffp1-1) is the number of bytes to IP header, 204210284Sjmallett and the hardware will calculate and insert the UDP/TCP checksum */ 205210284Sjmallett uint64_t ignore_i : 1; /**< If set, ignore the I bit (force to zero) from all pointer structures */ 206210284Sjmallett uint64_t dontfree : 1; /**< If clear, the hardware will attempt to free the buffers containing the packet */ 207210284Sjmallett uint64_t segs : 6; /**< The total number of segs in the packet, if gather set, also gather list length */ 208210284Sjmallett uint64_t total_bytes :16; /**< Including L2, but no trailing CRC */ 209210284Sjmallett } s; 210210284Sjmallett} cvmx_pko_command_word0_t; 211210284Sjmallett 212215990Sjmallett/* CSR typedefs have been moved to cvmx-pko-defs.h */ 213210284Sjmallett 214210284Sjmallett/** 215210284Sjmallett * Definition of internal state for Packet output processing 216210284Sjmallett */ 217210284Sjmalletttypedef struct 218210284Sjmallett{ 219210284Sjmallett uint64_t * start_ptr; /**< ptr to start of buffer, offset kept in FAU reg */ 220210284Sjmallett} cvmx_pko_state_elem_t; 221210284Sjmallett 222210284Sjmallett 223210284Sjmallett#ifdef CVMX_ENABLE_PKO_FUNCTIONS 224210284Sjmallett/** 225210284Sjmallett * Call before any other calls to initialize the packet 226210284Sjmallett * output system. 227210284Sjmallett */ 228210284Sjmallettextern void cvmx_pko_initialize_global(void); 229210284Sjmallettextern int cvmx_pko_initialize_local(void); 230210284Sjmallett 231210284Sjmallett#endif 232210284Sjmallett 233210284Sjmallett 234210284Sjmallett/** 235210284Sjmallett * Enables the packet output hardware. It must already be 236210284Sjmallett * configured. 237210284Sjmallett */ 238210284Sjmallettextern void cvmx_pko_enable(void); 239210284Sjmallett 240210284Sjmallett 241210284Sjmallett/** 242210284Sjmallett * Disables the packet output. Does not affect any configuration. 243210284Sjmallett */ 244210284Sjmallettextern void cvmx_pko_disable(void); 245210284Sjmallett 246210284Sjmallett 247210284Sjmallett/** 248210284Sjmallett * Shutdown and free resources required by packet output. 249210284Sjmallett */ 250210284Sjmallett 251210284Sjmallett#ifdef CVMX_ENABLE_PKO_FUNCTIONS 252210284Sjmallettextern void cvmx_pko_shutdown(void); 253210284Sjmallett 254210284Sjmallett/** 255210284Sjmallett * Configure a output port and the associated queues for use. 256210284Sjmallett * 257210284Sjmallett * @param port Port to configure. 258210284Sjmallett * @param base_queue First queue number to associate with this port. 259210284Sjmallett * @param num_queues Number of queues t oassociate with this port 260210284Sjmallett * @param priority Array of priority levels for each queue. Values are 261210284Sjmallett * allowed to be 1-8. A value of 8 get 8 times the traffic 262210284Sjmallett * of a value of 1. There must be num_queues elements in the 263210284Sjmallett * array. 264210284Sjmallett */ 265210284Sjmallettextern cvmx_pko_status_t cvmx_pko_config_port(uint64_t port, uint64_t base_queue, uint64_t num_queues, const uint64_t priority[]); 266210284Sjmallett 267210284Sjmallett 268210284Sjmallett/** 269210284Sjmallett * Ring the packet output doorbell. This tells the packet 270210284Sjmallett * output hardware that "len" command words have been added 271210284Sjmallett * to its pending list. This command includes the required 272210284Sjmallett * CVMX_SYNCWS before the doorbell ring. 273210284Sjmallett * 274232812Sjmallett * WARNING: This function may have to look up the proper PKO port in 275232812Sjmallett * the IPD port to PKO port map, and is thus slower than calling 276232812Sjmallett * cvmx_pko_doorbell_pkoid() directly if the PKO port identifier is 277232812Sjmallett * known. 278232812Sjmallett * 279232812Sjmallett * @param ipd_port The IPD port corresponding the to pko port the packet is for 280210284Sjmallett * @param queue Queue the packet is for 281210284Sjmallett * @param len Length of the command in 64 bit words 282210284Sjmallett */ 283232812Sjmallettstatic inline void cvmx_pko_doorbell(uint64_t ipd_port, uint64_t queue, uint64_t len) 284210284Sjmallett{ 285210284Sjmallett cvmx_pko_doorbell_address_t ptr; 286232812Sjmallett uint64_t pko_port; 287210284Sjmallett 288232812Sjmallett pko_port = ipd_port; 289232812Sjmallett if (octeon_has_feature(OCTEON_FEATURE_PKND)) 290232812Sjmallett pko_port = cvmx_helper_cfg_ipd2pko_port_base(ipd_port); 291232812Sjmallett 292210284Sjmallett ptr.u64 = 0; 293210284Sjmallett ptr.s.mem_space = CVMX_IO_SEG; 294210284Sjmallett ptr.s.did = CVMX_OCT_DID_PKT_SEND; 295210284Sjmallett ptr.s.is_io = 1; 296232812Sjmallett ptr.s.port = pko_port; 297210284Sjmallett ptr.s.queue = queue; 298210284Sjmallett CVMX_SYNCWS; /* Need to make sure output queue data is in DRAM before doorbell write */ 299210284Sjmallett cvmx_write_io(ptr.u64, len); 300210284Sjmallett} 301232812Sjmallett#endif 302210284Sjmallett 303210284Sjmallett 304210284Sjmallett/** 305210284Sjmallett * Prepare to send a packet. This may initiate a tag switch to 306210284Sjmallett * get exclusive access to the output queue structure, and 307210284Sjmallett * performs other prep work for the packet send operation. 308210284Sjmallett * 309210284Sjmallett * cvmx_pko_send_packet_finish() MUST be called after this function is called, 310210284Sjmallett * and must be called with the same port/queue/use_locking arguments. 311210284Sjmallett * 312210284Sjmallett * The use_locking parameter allows the caller to use three 313210284Sjmallett * possible locking modes. 314210284Sjmallett * - CVMX_PKO_LOCK_NONE 315210284Sjmallett * - PKO doesn't do any locking. It is the responsibility 316210284Sjmallett * of the application to make sure that no other core 317232812Sjmallett * is accessing the same queue at the same time. 318210284Sjmallett * - CVMX_PKO_LOCK_ATOMIC_TAG 319210284Sjmallett * - PKO performs an atomic tagswitch to insure exclusive 320210284Sjmallett * access to the output queue. This will maintain 321210284Sjmallett * packet ordering on output. 322210284Sjmallett * - CVMX_PKO_LOCK_CMD_QUEUE 323210284Sjmallett * - PKO uses the common command queue locks to insure 324210284Sjmallett * exclusive access to the output queue. This is a 325210284Sjmallett * memory based ll/sc. This is the most portable 326210284Sjmallett * locking mechanism. 327210284Sjmallett * 328210284Sjmallett * NOTE: If atomic locking is used, the POW entry CANNOT be 329210284Sjmallett * descheduled, as it does not contain a valid WQE pointer. 330210284Sjmallett * 331232812Sjmallett * @param port Port to send it on, this can be either IPD port or PKO 332232812Sjmallett * port. 333210284Sjmallett * @param queue Queue to use 334210284Sjmallett * @param use_locking 335210284Sjmallett * CVMX_PKO_LOCK_NONE, CVMX_PKO_LOCK_ATOMIC_TAG, or CVMX_PKO_LOCK_CMD_QUEUE 336210284Sjmallett */ 337210284Sjmallett#ifdef CVMX_ENABLE_PKO_FUNCTIONS 338210284Sjmallettstatic inline void cvmx_pko_send_packet_prepare(uint64_t port, uint64_t queue, cvmx_pko_lock_t use_locking) 339210284Sjmallett{ 340210284Sjmallett if (use_locking == CVMX_PKO_LOCK_ATOMIC_TAG) 341210284Sjmallett { 342210284Sjmallett /* Must do a full switch here to handle all cases. We use a fake WQE pointer, as the POW does 343210284Sjmallett ** not access this memory. The WQE pointer and group are only used if this work is descheduled, 344210284Sjmallett ** which is not supported by the cvmx_pko_send_packet_prepare/cvmx_pko_send_packet_finish combination. 345210284Sjmallett ** Note that this is a special case in which these fake values can be used - this is not a general technique. 346210284Sjmallett */ 347210284Sjmallett uint32_t tag = CVMX_TAG_SW_BITS_INTERNAL << CVMX_TAG_SW_SHIFT | CVMX_TAG_SUBGROUP_PKO << CVMX_TAG_SUBGROUP_SHIFT | (CVMX_TAG_SUBGROUP_MASK & queue); 348210284Sjmallett cvmx_pow_tag_sw_full((cvmx_wqe_t *)cvmx_phys_to_ptr(0x80), tag, CVMX_POW_TAG_TYPE_ATOMIC, 0); 349210284Sjmallett } 350210284Sjmallett} 351210284Sjmallett 352232812Sjmallett#define cvmx_pko_send_packet_prepare_pkoid cvmx_pko_send_packet_prepare 353210284Sjmallett 354210284Sjmallett/** 355210284Sjmallett * Complete packet output. cvmx_pko_send_packet_prepare() must be called exactly once before this, 356210284Sjmallett * and the same parameters must be passed to both cvmx_pko_send_packet_prepare() and 357210284Sjmallett * cvmx_pko_send_packet_finish(). 358210284Sjmallett * 359232812Sjmallett * WARNING: This function may have to look up the proper PKO port in 360232812Sjmallett * the IPD port to PKO port map, and is thus slower than calling 361232812Sjmallett * cvmx_pko_send_packet_finish_pkoid() directly if the PKO port 362232812Sjmallett * identifier is known. 363232812Sjmallett * 364232812Sjmallett * @param ipd_port The IPD port corresponding the to pko port the packet is for 365210284Sjmallett * @param queue Queue to use 366210284Sjmallett * @param pko_command 367210284Sjmallett * PKO HW command word 368210284Sjmallett * @param packet Packet to send 369210284Sjmallett * @param use_locking 370210284Sjmallett * CVMX_PKO_LOCK_NONE, CVMX_PKO_LOCK_ATOMIC_TAG, or CVMX_PKO_LOCK_CMD_QUEUE 371210284Sjmallett * 372210284Sjmallett * @return returns CVMX_PKO_SUCCESS on success, or error code on failure of output 373210284Sjmallett */ 374232812Sjmallettstatic inline cvmx_pko_status_t cvmx_pko_send_packet_finish(uint64_t ipd_port, uint64_t queue, 375210284Sjmallett cvmx_pko_command_word0_t pko_command, 376210284Sjmallett cvmx_buf_ptr_t packet, cvmx_pko_lock_t use_locking) 377210284Sjmallett{ 378210284Sjmallett cvmx_cmd_queue_result_t result; 379210284Sjmallett if (use_locking == CVMX_PKO_LOCK_ATOMIC_TAG) 380210284Sjmallett cvmx_pow_tag_sw_wait(); 381210284Sjmallett result = cvmx_cmd_queue_write2(CVMX_CMD_QUEUE_PKO(queue), 382210284Sjmallett (use_locking == CVMX_PKO_LOCK_CMD_QUEUE), 383210284Sjmallett pko_command.u64, 384210284Sjmallett packet.u64); 385210284Sjmallett if (cvmx_likely(result == CVMX_CMD_QUEUE_SUCCESS)) 386210284Sjmallett { 387232812Sjmallett cvmx_pko_doorbell(ipd_port, queue, 2); 388210284Sjmallett return CVMX_PKO_SUCCESS; 389210284Sjmallett } 390210284Sjmallett else if ((result == CVMX_CMD_QUEUE_NO_MEMORY) || (result == CVMX_CMD_QUEUE_FULL)) 391210284Sjmallett { 392210284Sjmallett return CVMX_PKO_NO_MEMORY; 393210284Sjmallett } 394210284Sjmallett else 395210284Sjmallett { 396210284Sjmallett return CVMX_PKO_INVALID_QUEUE; 397210284Sjmallett } 398210284Sjmallett} 399210284Sjmallett 400210284Sjmallett 401210284Sjmallett/** 402210284Sjmallett * Complete packet output. cvmx_pko_send_packet_prepare() must be called exactly once before this, 403210284Sjmallett * and the same parameters must be passed to both cvmx_pko_send_packet_prepare() and 404210284Sjmallett * cvmx_pko_send_packet_finish(). 405210284Sjmallett * 406232812Sjmallett * WARNING: This function may have to look up the proper PKO port in 407232812Sjmallett * the IPD port to PKO port map, and is thus slower than calling 408232812Sjmallett * cvmx_pko_send_packet_finish3_pkoid() directly if the PKO port 409232812Sjmallett * identifier is known. 410232812Sjmallett * 411232812Sjmallett * @param ipd_port The IPD port corresponding the to pko port the packet is for 412210284Sjmallett * @param queue Queue to use 413210284Sjmallett * @param pko_command 414210284Sjmallett * PKO HW command word 415210284Sjmallett * @param packet Packet to send 416210284Sjmallett * @param addr Plysical address of a work queue entry or physical address to zero on complete. 417210284Sjmallett * @param use_locking 418210284Sjmallett * CVMX_PKO_LOCK_NONE, CVMX_PKO_LOCK_ATOMIC_TAG, or CVMX_PKO_LOCK_CMD_QUEUE 419210284Sjmallett * 420210284Sjmallett * @return returns CVMX_PKO_SUCCESS on success, or error code on failure of output 421210284Sjmallett */ 422232812Sjmallettstatic inline cvmx_pko_status_t cvmx_pko_send_packet_finish3(uint64_t ipd_port, uint64_t queue, 423210284Sjmallett cvmx_pko_command_word0_t pko_command, 424210284Sjmallett cvmx_buf_ptr_t packet, uint64_t addr, cvmx_pko_lock_t use_locking) 425210284Sjmallett{ 426210284Sjmallett cvmx_cmd_queue_result_t result; 427210284Sjmallett if (use_locking == CVMX_PKO_LOCK_ATOMIC_TAG) 428210284Sjmallett cvmx_pow_tag_sw_wait(); 429210284Sjmallett result = cvmx_cmd_queue_write3(CVMX_CMD_QUEUE_PKO(queue), 430210284Sjmallett (use_locking == CVMX_PKO_LOCK_CMD_QUEUE), 431210284Sjmallett pko_command.u64, 432210284Sjmallett packet.u64, 433210284Sjmallett addr); 434210284Sjmallett if (cvmx_likely(result == CVMX_CMD_QUEUE_SUCCESS)) 435210284Sjmallett { 436232812Sjmallett cvmx_pko_doorbell(ipd_port, queue, 3); 437210284Sjmallett return CVMX_PKO_SUCCESS; 438210284Sjmallett } 439210284Sjmallett else if ((result == CVMX_CMD_QUEUE_NO_MEMORY) || (result == CVMX_CMD_QUEUE_FULL)) 440210284Sjmallett { 441210284Sjmallett return CVMX_PKO_NO_MEMORY; 442210284Sjmallett } 443210284Sjmallett else 444210284Sjmallett { 445210284Sjmallett return CVMX_PKO_INVALID_QUEUE; 446210284Sjmallett } 447210284Sjmallett} 448210284Sjmallett 449210284Sjmallett/** 450232812Sjmallett * Get the first pko_port for the (interface, index) 451232812Sjmallett * 452232812Sjmallett * @param interface 453232812Sjmallett * @param index 454232812Sjmallett */ 455232812Sjmallettextern int cvmx_pko_get_base_pko_port(int interface, int index); 456232812Sjmallett 457232812Sjmallett/** 458232812Sjmallett * Get the number of pko_ports for the (interface, index) 459232812Sjmallett * 460232812Sjmallett * @param interface 461232812Sjmallett * @param index 462232812Sjmallett */ 463232812Sjmallettextern int cvmx_pko_get_num_pko_ports(int interface, int index); 464232812Sjmallett 465232812Sjmallett/** 466210284Sjmallett * Return the pko output queue associated with a port and a specific core. 467210284Sjmallett * In normal mode (PKO lockless operation is disabled), the value returned 468210284Sjmallett * is the base queue. 469210284Sjmallett * 470210284Sjmallett * @param port Port number 471210284Sjmallett * @param core Core to get queue for 472210284Sjmallett * 473232812Sjmallett * @return Core-specific output queue and -1 on error. 474232812Sjmallett * 475232812Sjmallett * Note: This function is invalid for o68. 476210284Sjmallett */ 477210284Sjmallettstatic inline int cvmx_pko_get_base_queue_per_core(int port, int core) 478210284Sjmallett{ 479232812Sjmallett if (OCTEON_IS_MODEL(OCTEON_CN68XX)) 480232812Sjmallett { 481232812Sjmallett cvmx_dprintf("cvmx_pko_get_base_queue_per_core() not" 482232812Sjmallett "supported starting from o68!\n"); 483232812Sjmallett return -1; 484232812Sjmallett } 485232812Sjmallett 486210284Sjmallett#ifndef CVMX_HELPER_PKO_MAX_PORTS_INTERFACE0 487210284Sjmallett #define CVMX_HELPER_PKO_MAX_PORTS_INTERFACE0 16 488210284Sjmallett#endif 489210284Sjmallett#ifndef CVMX_HELPER_PKO_MAX_PORTS_INTERFACE1 490210284Sjmallett #define CVMX_HELPER_PKO_MAX_PORTS_INTERFACE1 16 491210284Sjmallett#endif 492215990Sjmallett#ifndef CVMX_PKO_QUEUES_PER_PORT_SRIO0 493215990Sjmallett /* We use two queues per port for SRIO0. Having two queues per 494215990Sjmallett port with two ports gives us four queues, one for each mailbox */ 495215990Sjmallett #define CVMX_PKO_QUEUES_PER_PORT_SRIO0 2 496215990Sjmallett#endif 497215990Sjmallett#ifndef CVMX_PKO_QUEUES_PER_PORT_SRIO1 498215990Sjmallett /* We use two queues per port for SRIO1. Having two queues per 499215990Sjmallett port with two ports gives us four queues, one for each mailbox */ 500215990Sjmallett #define CVMX_PKO_QUEUES_PER_PORT_SRIO1 2 501215990Sjmallett#endif 502232812Sjmallett#ifndef CVMX_PKO_QUEUES_PER_PORT_SRIO2 503232812Sjmallett /* We use two queues per port for SRIO2. Having two queues per 504232812Sjmallett port with two ports gives us four queues, one for each mailbox */ 505232812Sjmallett #define CVMX_PKO_QUEUES_PER_PORT_SRIO2 2 506232812Sjmallett#endif 507210284Sjmallett if (port < CVMX_PKO_MAX_PORTS_INTERFACE0) 508210284Sjmallett return port * CVMX_PKO_QUEUES_PER_PORT_INTERFACE0 + core; 509210284Sjmallett else if (port >=16 && port < 16 + CVMX_PKO_MAX_PORTS_INTERFACE1) 510210284Sjmallett return CVMX_PKO_MAX_PORTS_INTERFACE0 * CVMX_PKO_QUEUES_PER_PORT_INTERFACE0 + 511210284Sjmallett (port-16) * CVMX_PKO_QUEUES_PER_PORT_INTERFACE1 + core; 512210284Sjmallett else if ((port >= 32) && (port < 36)) 513210284Sjmallett return CVMX_PKO_MAX_PORTS_INTERFACE0 * CVMX_PKO_QUEUES_PER_PORT_INTERFACE0 + 514210284Sjmallett CVMX_PKO_MAX_PORTS_INTERFACE1 * CVMX_PKO_QUEUES_PER_PORT_INTERFACE1 + 515210284Sjmallett (port-32) * CVMX_PKO_QUEUES_PER_PORT_PCI; 516210284Sjmallett else if ((port >= 36) && (port < 40)) 517210284Sjmallett return CVMX_PKO_MAX_PORTS_INTERFACE0 * CVMX_PKO_QUEUES_PER_PORT_INTERFACE0 + 518210284Sjmallett CVMX_PKO_MAX_PORTS_INTERFACE1 * CVMX_PKO_QUEUES_PER_PORT_INTERFACE1 + 519210284Sjmallett 4 * CVMX_PKO_QUEUES_PER_PORT_PCI + 520210284Sjmallett (port-36) * CVMX_PKO_QUEUES_PER_PORT_LOOP; 521215990Sjmallett else if ((port >= 40) && (port < 42)) 522215990Sjmallett return CVMX_PKO_MAX_PORTS_INTERFACE0 * CVMX_PKO_QUEUES_PER_PORT_INTERFACE0 + 523215990Sjmallett CVMX_PKO_MAX_PORTS_INTERFACE1 * CVMX_PKO_QUEUES_PER_PORT_INTERFACE1 + 524215990Sjmallett 4 * CVMX_PKO_QUEUES_PER_PORT_PCI + 525215990Sjmallett 4 * CVMX_PKO_QUEUES_PER_PORT_LOOP + 526215990Sjmallett (port-40) * CVMX_PKO_QUEUES_PER_PORT_SRIO0; 527215990Sjmallett else if ((port >= 42) && (port < 44)) 528215990Sjmallett return CVMX_PKO_MAX_PORTS_INTERFACE0 * CVMX_PKO_QUEUES_PER_PORT_INTERFACE0 + 529215990Sjmallett CVMX_PKO_MAX_PORTS_INTERFACE1 * CVMX_PKO_QUEUES_PER_PORT_INTERFACE1 + 530215990Sjmallett 4 * CVMX_PKO_QUEUES_PER_PORT_PCI + 531215990Sjmallett 4 * CVMX_PKO_QUEUES_PER_PORT_LOOP + 532215990Sjmallett 2 * CVMX_PKO_QUEUES_PER_PORT_SRIO0 + 533215990Sjmallett (port-42) * CVMX_PKO_QUEUES_PER_PORT_SRIO1; 534232812Sjmallett else if ((port >= 44) && (port < 46)) 535232812Sjmallett return CVMX_PKO_MAX_PORTS_INTERFACE0 * CVMX_PKO_QUEUES_PER_PORT_INTERFACE0 + 536232812Sjmallett CVMX_PKO_MAX_PORTS_INTERFACE1 * CVMX_PKO_QUEUES_PER_PORT_INTERFACE1 + 537232812Sjmallett 4 * CVMX_PKO_QUEUES_PER_PORT_PCI + 538232812Sjmallett 4 * CVMX_PKO_QUEUES_PER_PORT_LOOP + 539232812Sjmallett 4 * CVMX_PKO_QUEUES_PER_PORT_SRIO0 + 540232812Sjmallett (port-44) * CVMX_PKO_QUEUES_PER_PORT_SRIO2; 541210284Sjmallett else 542210284Sjmallett /* Given the limit on the number of ports we can map to 543210284Sjmallett * CVMX_MAX_OUTPUT_QUEUES_STATIC queues (currently 256, 544210284Sjmallett * divided among all cores), the remaining unmapped ports 545210284Sjmallett * are assigned an illegal queue number */ 546210284Sjmallett return CVMX_PKO_ILLEGAL_QUEUE; 547210284Sjmallett} 548210284Sjmallett 549210284Sjmallett/** 550210284Sjmallett * For a given port number, return the base pko output queue 551210284Sjmallett * for the port. 552210284Sjmallett * 553232812Sjmallett * @param port IPD port number 554210284Sjmallett * @return Base output queue 555210284Sjmallett */ 556232812Sjmallettextern int cvmx_pko_get_base_queue(int port); 557210284Sjmallett 558210284Sjmallett/** 559210284Sjmallett * For a given port number, return the number of pko output queues. 560210284Sjmallett * 561232812Sjmallett * @param port IPD port number 562210284Sjmallett * @return Number of output queues 563210284Sjmallett */ 564232812Sjmallettextern int cvmx_pko_get_num_queues(int port); 565210284Sjmallett 566232812Sjmallett#ifdef CVMX_ENABLE_PKO_FUNCTIONS 567232812Sjmallett 568210284Sjmallett/** 569210284Sjmallett * Get the status counters for a port. 570210284Sjmallett * 571232812Sjmallett * @param ipd_port Port number (ipd_port) to get statistics for. 572210284Sjmallett * @param clear Set to 1 to clear the counters after they are read 573210284Sjmallett * @param status Where to put the results. 574232812Sjmallett * 575232812Sjmallett * Note: 576232812Sjmallett * - Only the doorbell for the base queue of the ipd_port is 577232812Sjmallett * collected. 578232812Sjmallett * - Retrieving the stats involves writing the index through 579232812Sjmallett * CVMX_PKO_REG_READ_IDX and reading the stat CSRs, in that 580232812Sjmallett * order. It is not MP-safe and caller should guarantee 581232812Sjmallett * atomicity. 582210284Sjmallett */ 583232812Sjmallettstatic inline void cvmx_pko_get_port_status(uint64_t ipd_port, uint64_t clear, 584232812Sjmallett cvmx_pko_port_status_t *status) 585210284Sjmallett{ 586210284Sjmallett cvmx_pko_reg_read_idx_t pko_reg_read_idx; 587210284Sjmallett cvmx_pko_mem_count0_t pko_mem_count0; 588210284Sjmallett cvmx_pko_mem_count1_t pko_mem_count1; 589232812Sjmallett int pko_port, port_base, port_limit; 590210284Sjmallett 591232812Sjmallett if (octeon_has_feature(OCTEON_FEATURE_PKND)) { 592232812Sjmallett int interface = cvmx_helper_get_interface_num(ipd_port); 593232812Sjmallett int index = cvmx_helper_get_interface_index_num(ipd_port); 594232812Sjmallett port_base = cvmx_helper_get_pko_port(interface, index); 595232812Sjmallett if (port_base == -1) 596232812Sjmallett cvmx_dprintf("Warning: Invalid port_base\n"); 597232812Sjmallett port_limit = port_base + cvmx_pko_get_num_pko_ports(interface, index); 598232812Sjmallett } else { 599232812Sjmallett port_base = ipd_port; 600232812Sjmallett port_limit = port_base + 1; 601232812Sjmallett } 602232812Sjmallett 603232812Sjmallett /* 604232812Sjmallett * status->packets and status->octets 605232812Sjmallett */ 606232812Sjmallett status->packets = 0; 607232812Sjmallett status->octets = 0; 608210284Sjmallett pko_reg_read_idx.u64 = 0; 609210284Sjmallett 610232812Sjmallett for (pko_port = port_base; pko_port < port_limit; pko_port++) 611210284Sjmallett { 612210284Sjmallett 613232812Sjmallett /* 614232812Sjmallett * In theory, one doesn't need to write the index csr every 615232812Sjmallett * time as he can set pko_reg_read_idx.s.inc to increment 616232812Sjmallett * the index automatically. Need to find out exactly how XXX. 617232812Sjmallett */ 618232812Sjmallett pko_reg_read_idx.s.index = pko_port; 619232812Sjmallett cvmx_write_csr(CVMX_PKO_REG_READ_IDX, pko_reg_read_idx.u64); 620232812Sjmallett 621232812Sjmallett pko_mem_count0.u64 = cvmx_read_csr(CVMX_PKO_MEM_COUNT0); 622232812Sjmallett status->packets += pko_mem_count0.s.count; 623232812Sjmallett if (clear) 624232812Sjmallett { 625232812Sjmallett pko_mem_count0.s.count = pko_port; 626232812Sjmallett cvmx_write_csr(CVMX_PKO_MEM_COUNT0, pko_mem_count0.u64); 627232812Sjmallett } 628232812Sjmallett 629232812Sjmallett pko_mem_count1.u64 = cvmx_read_csr(CVMX_PKO_MEM_COUNT1); 630232812Sjmallett status->octets += pko_mem_count1.s.count; 631232812Sjmallett if (clear) 632232812Sjmallett { 633232812Sjmallett pko_mem_count1.s.count = pko_port; 634232812Sjmallett cvmx_write_csr(CVMX_PKO_MEM_COUNT1, pko_mem_count1.u64); 635232812Sjmallett } 636210284Sjmallett } 637210284Sjmallett 638232812Sjmallett /* 639232812Sjmallett * status->doorbell 640232812Sjmallett */ 641210284Sjmallett if (OCTEON_IS_MODEL(OCTEON_CN3XXX)) 642210284Sjmallett { 643210284Sjmallett cvmx_pko_mem_debug9_t debug9; 644232812Sjmallett pko_reg_read_idx.s.index = cvmx_pko_get_base_queue(ipd_port); 645210284Sjmallett cvmx_write_csr(CVMX_PKO_REG_READ_IDX, pko_reg_read_idx.u64); 646210284Sjmallett debug9.u64 = cvmx_read_csr(CVMX_PKO_MEM_DEBUG9); 647210284Sjmallett status->doorbell = debug9.cn38xx.doorbell; 648210284Sjmallett } 649210284Sjmallett else 650210284Sjmallett { 651210284Sjmallett cvmx_pko_mem_debug8_t debug8; 652232812Sjmallett pko_reg_read_idx.s.index = cvmx_pko_get_base_queue(ipd_port); 653210284Sjmallett cvmx_write_csr(CVMX_PKO_REG_READ_IDX, pko_reg_read_idx.u64); 654210284Sjmallett debug8.u64 = cvmx_read_csr(CVMX_PKO_MEM_DEBUG8); 655232812Sjmallett if (OCTEON_IS_MODEL(OCTEON_CN68XX)) 656232812Sjmallett status->doorbell = debug8.cn68xx.doorbell; 657232812Sjmallett else 658232812Sjmallett status->doorbell = debug8.cn58xx.doorbell; 659210284Sjmallett } 660210284Sjmallett} 661210284Sjmallett 662232812Sjmallett#endif /* CVMX_ENABLE_PKO_FUNCTION */ 663210284Sjmallett 664232812Sjmallett 665210284Sjmallett/** 666210284Sjmallett * Rate limit a PKO port to a max packets/sec. This function is only 667210284Sjmallett * supported on CN57XX, CN56XX, CN55XX, and CN54XX. 668210284Sjmallett * 669210284Sjmallett * @param port Port to rate limit 670210284Sjmallett * @param packets_s Maximum packet/sec 671210284Sjmallett * @param burst Maximum number of packets to burst in a row before rate 672210284Sjmallett * limiting cuts in. 673210284Sjmallett * 674210284Sjmallett * @return Zero on success, negative on failure 675210284Sjmallett */ 676210284Sjmallettextern int cvmx_pko_rate_limit_packets(int port, int packets_s, int burst); 677210284Sjmallett 678210284Sjmallett/** 679210284Sjmallett * Rate limit a PKO port to a max bits/sec. This function is only 680210284Sjmallett * supported on CN57XX, CN56XX, CN55XX, and CN54XX. 681210284Sjmallett * 682210284Sjmallett * @param port Port to rate limit 683210284Sjmallett * @param bits_s PKO rate limit in bits/sec 684210284Sjmallett * @param burst Maximum number of bits to burst before rate 685210284Sjmallett * limiting cuts in. 686210284Sjmallett * 687210284Sjmallett * @return Zero on success, negative on failure 688210284Sjmallett */ 689210284Sjmallettextern int cvmx_pko_rate_limit_bits(int port, uint64_t bits_s, int burst); 690210284Sjmallett 691232812Sjmallett/** 692232812Sjmallett * @INTERNAL 693232812Sjmallett * 694232812Sjmallett * Retrieve the PKO pipe number for a port 695232812Sjmallett * 696232812Sjmallett * @param interface 697232812Sjmallett * @param index 698232812Sjmallett * 699232812Sjmallett * @return negative on error. 700232812Sjmallett * 701232812Sjmallett * This applies only to the non-loopback interfaces. 702232812Sjmallett * 703232812Sjmallett */ 704232812Sjmallettextern int __cvmx_pko_get_pipe(int interface, int index); 705232812Sjmallett 706232812Sjmallett/** 707232812Sjmallett * For a given PKO port number, return the base output queue 708232812Sjmallett * for the port. 709232812Sjmallett * 710232812Sjmallett * @param pko_port PKO port number 711232812Sjmallett * @return Base output queue 712232812Sjmallett */ 713232812Sjmallettextern int cvmx_pko_get_base_queue_pkoid(int pko_port); 714232812Sjmallett 715232812Sjmallett/** 716232812Sjmallett * For a given PKO port number, return the number of output queues 717232812Sjmallett * for the port. 718232812Sjmallett * 719232812Sjmallett * @param pko_port PKO port number 720232812Sjmallett * @return the number of output queues 721232812Sjmallett */ 722232812Sjmallettextern int cvmx_pko_get_num_queues_pkoid(int pko_port); 723232812Sjmallett 724232812Sjmallett/** 725232812Sjmallett * Ring the packet output doorbell. This tells the packet 726232812Sjmallett * output hardware that "len" command words have been added 727232812Sjmallett * to its pending list. This command includes the required 728232812Sjmallett * CVMX_SYNCWS before the doorbell ring. 729232812Sjmallett * 730232812Sjmallett * @param pko_port Port the packet is for 731232812Sjmallett * @param queue Queue the packet is for 732232812Sjmallett * @param len Length of the command in 64 bit words 733232812Sjmallett */ 734232812Sjmallettstatic inline void cvmx_pko_doorbell_pkoid(uint64_t pko_port, uint64_t queue, uint64_t len) 735232812Sjmallett{ 736232812Sjmallett cvmx_pko_doorbell_address_t ptr; 737232812Sjmallett 738232812Sjmallett ptr.u64 = 0; 739232812Sjmallett ptr.s.mem_space = CVMX_IO_SEG; 740232812Sjmallett ptr.s.did = CVMX_OCT_DID_PKT_SEND; 741232812Sjmallett ptr.s.is_io = 1; 742232812Sjmallett ptr.s.port = pko_port; 743232812Sjmallett ptr.s.queue = queue; 744232812Sjmallett CVMX_SYNCWS; /* Need to make sure output queue data is in DRAM before doorbell write */ 745232812Sjmallett cvmx_write_io(ptr.u64, len); 746232812Sjmallett} 747232812Sjmallett 748232812Sjmallett/** 749232812Sjmallett * Complete packet output. cvmx_pko_send_packet_prepare() must be called exactly once before this, 750232812Sjmallett * and the same parameters must be passed to both cvmx_pko_send_packet_prepare() and 751232812Sjmallett * cvmx_pko_send_packet_finish_pkoid(). 752232812Sjmallett * 753232812Sjmallett * @param pko_port Port to send it on 754232812Sjmallett * @param queue Queue to use 755232812Sjmallett * @param pko_command 756232812Sjmallett * PKO HW command word 757232812Sjmallett * @param packet Packet to send 758232812Sjmallett * @param use_locking 759232812Sjmallett * CVMX_PKO_LOCK_NONE, CVMX_PKO_LOCK_ATOMIC_TAG, or CVMX_PKO_LOCK_CMD_QUEUE 760232812Sjmallett * 761232812Sjmallett * @return returns CVMX_PKO_SUCCESS on success, or error code on failure of output 762232812Sjmallett */ 763232812Sjmallettstatic inline cvmx_pko_status_t cvmx_pko_send_packet_finish_pkoid(int pko_port, uint64_t queue, 764232812Sjmallett cvmx_pko_command_word0_t pko_command, 765232812Sjmallett cvmx_buf_ptr_t packet, cvmx_pko_lock_t use_locking) 766232812Sjmallett{ 767232812Sjmallett cvmx_cmd_queue_result_t result; 768232812Sjmallett if (use_locking == CVMX_PKO_LOCK_ATOMIC_TAG) 769232812Sjmallett cvmx_pow_tag_sw_wait(); 770232812Sjmallett result = cvmx_cmd_queue_write2(CVMX_CMD_QUEUE_PKO(queue), 771232812Sjmallett (use_locking == CVMX_PKO_LOCK_CMD_QUEUE), 772232812Sjmallett pko_command.u64, 773232812Sjmallett packet.u64); 774232812Sjmallett if (cvmx_likely(result == CVMX_CMD_QUEUE_SUCCESS)) 775232812Sjmallett { 776232812Sjmallett cvmx_pko_doorbell_pkoid(pko_port, queue, 2); 777232812Sjmallett return CVMX_PKO_SUCCESS; 778232812Sjmallett } 779232812Sjmallett else if ((result == CVMX_CMD_QUEUE_NO_MEMORY) || (result == CVMX_CMD_QUEUE_FULL)) 780232812Sjmallett { 781232812Sjmallett return CVMX_PKO_NO_MEMORY; 782232812Sjmallett } 783232812Sjmallett else 784232812Sjmallett { 785232812Sjmallett return CVMX_PKO_INVALID_QUEUE; 786232812Sjmallett } 787232812Sjmallett} 788232812Sjmallett 789232812Sjmallett/** 790232812Sjmallett * Complete packet output. cvmx_pko_send_packet_prepare() must be called exactly once before this, 791232812Sjmallett * and the same parameters must be passed to both cvmx_pko_send_packet_prepare() and 792232812Sjmallett * cvmx_pko_send_packet_finish_pkoid(). 793232812Sjmallett * 794232812Sjmallett * @param pko_port The PKO port the packet is for 795232812Sjmallett * @param queue Queue to use 796232812Sjmallett * @param pko_command 797232812Sjmallett * PKO HW command word 798232812Sjmallett * @param packet Packet to send 799232812Sjmallett * @param addr Plysical address of a work queue entry or physical address to zero on complete. 800232812Sjmallett * @param use_locking 801232812Sjmallett * CVMX_PKO_LOCK_NONE, CVMX_PKO_LOCK_ATOMIC_TAG, or CVMX_PKO_LOCK_CMD_QUEUE 802232812Sjmallett * 803232812Sjmallett * @return returns CVMX_PKO_SUCCESS on success, or error code on failure of output 804232812Sjmallett */ 805232812Sjmallettstatic inline cvmx_pko_status_t cvmx_pko_send_packet_finish3_pkoid(uint64_t pko_port, uint64_t queue, 806232812Sjmallett cvmx_pko_command_word0_t pko_command, 807232812Sjmallett cvmx_buf_ptr_t packet, uint64_t addr, cvmx_pko_lock_t use_locking) 808232812Sjmallett{ 809232812Sjmallett cvmx_cmd_queue_result_t result; 810232812Sjmallett if (use_locking == CVMX_PKO_LOCK_ATOMIC_TAG) 811232812Sjmallett cvmx_pow_tag_sw_wait(); 812232812Sjmallett result = cvmx_cmd_queue_write3(CVMX_CMD_QUEUE_PKO(queue), 813232812Sjmallett (use_locking == CVMX_PKO_LOCK_CMD_QUEUE), 814232812Sjmallett pko_command.u64, 815232812Sjmallett packet.u64, 816232812Sjmallett addr); 817232812Sjmallett if (cvmx_likely(result == CVMX_CMD_QUEUE_SUCCESS)) 818232812Sjmallett { 819232812Sjmallett cvmx_pko_doorbell_pkoid(pko_port, queue, 3); 820232812Sjmallett return CVMX_PKO_SUCCESS; 821232812Sjmallett } 822232812Sjmallett else if ((result == CVMX_CMD_QUEUE_NO_MEMORY) || (result == CVMX_CMD_QUEUE_FULL)) 823232812Sjmallett { 824232812Sjmallett return CVMX_PKO_NO_MEMORY; 825232812Sjmallett } 826232812Sjmallett else 827232812Sjmallett { 828232812Sjmallett return CVMX_PKO_INVALID_QUEUE; 829232812Sjmallett } 830232812Sjmallett} 831232812Sjmallett 832210284Sjmallett#endif /* CVMX_ENABLE_PKO_FUNCTIONS */ 833210284Sjmallett 834210284Sjmallett#ifdef __cplusplus 835210284Sjmallett} 836210284Sjmallett#endif 837210284Sjmallett 838210284Sjmallett#endif /* __CVMX_PKO_H__ */ 839