1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Copyright (C) 2020 Marvell International Ltd. 4 * 5 * Interface to the hardware Packet Output unit. 6 * 7 * Starting with SDK 1.7.0, the PKO output functions now support 8 * two types of locking. CVMX_PKO_LOCK_ATOMIC_TAG continues to 9 * function similarly to previous SDKs by using POW atomic tags 10 * to preserve ordering and exclusivity. As a new option, you 11 * can now pass CVMX_PKO_LOCK_CMD_QUEUE which uses a ll/sc 12 * memory based locking instead. This locking has the advantage 13 * of not affecting the tag state but doesn't preserve packet 14 * ordering. CVMX_PKO_LOCK_CMD_QUEUE is appropriate in most 15 * generic code while CVMX_PKO_LOCK_CMD_QUEUE should be used 16 * with hand tuned fast path code. 17 * 18 * Some of other SDK differences visible to the command command 19 * queuing: 20 * - PKO indexes are no longer stored in the FAU. A large 21 * percentage of the FAU register block used to be tied up 22 * maintaining PKO queue pointers. These are now stored in a 23 * global named block. 24 * - The PKO <b>use_locking</b> parameter can now have a global 25 * effect. Since all application use the same named block, 26 * queue locking correctly applies across all operating 27 * systems when using CVMX_PKO_LOCK_CMD_QUEUE. 28 * - PKO 3 word commands are now supported. Use 29 * cvmx_pko_send_packet_finish3(). 30 */ 31 32#ifndef __CVMX_HWPKO_H__ 33#define __CVMX_HWPKO_H__ 34 35#include "cvmx-hwfau.h" 36#include "cvmx-fpa.h" 37#include "cvmx-pow.h" 38#include "cvmx-cmd-queue.h" 39#include "cvmx-helper.h" 40#include "cvmx-helper-util.h" 41#include "cvmx-helper-cfg.h" 42 43/* Adjust the command buffer size by 1 word so that in the case of using only 44** two word PKO commands no command words stradle buffers. The useful values 45** for this are 0 and 1. */ 46#define CVMX_PKO_COMMAND_BUFFER_SIZE_ADJUST (1) 47 48#define CVMX_PKO_MAX_OUTPUT_QUEUES_STATIC 256 49#define CVMX_PKO_MAX_OUTPUT_QUEUES \ 50 ((OCTEON_IS_OCTEON2() || OCTEON_IS_MODEL(OCTEON_CN70XX)) ? 256 : 128) 51#define CVMX_PKO_NUM_OUTPUT_PORTS \ 52 ((OCTEON_IS_MODEL(OCTEON_CN63XX)) ? 44 : (OCTEON_IS_MODEL(OCTEON_CN66XX) ? 48 : 40)) 53#define CVMX_PKO_MEM_QUEUE_PTRS_ILLEGAL_PID 63 54#define CVMX_PKO_QUEUE_STATIC_PRIORITY 9 55#define CVMX_PKO_ILLEGAL_QUEUE 0xFFFF 56#define CVMX_PKO_MAX_QUEUE_DEPTH 0 57 58typedef enum { 59 CVMX_PKO_SUCCESS, 60 CVMX_PKO_INVALID_PORT, 61 CVMX_PKO_INVALID_QUEUE, 62 CVMX_PKO_INVALID_PRIORITY, 63 CVMX_PKO_NO_MEMORY, 64 CVMX_PKO_PORT_ALREADY_SETUP, 65 CVMX_PKO_CMD_QUEUE_INIT_ERROR 66} cvmx_pko_return_value_t; 67 68/** 69 * This enumeration represents the differnet locking modes supported by PKO. 70 */ 71typedef enum { 72 CVMX_PKO_LOCK_NONE = 0, 73 CVMX_PKO_LOCK_ATOMIC_TAG = 1, 74 CVMX_PKO_LOCK_CMD_QUEUE = 2, 75} cvmx_pko_lock_t; 76 77typedef struct cvmx_pko_port_status { 78 u32 packets; 79 u64 octets; 80 u64 doorbell; 81} cvmx_pko_port_status_t; 82 83/** 84 * This structure defines the address to use on a packet enqueue 85 */ 86typedef union { 87 u64 u64; 88 struct { 89 cvmx_mips_space_t mem_space : 2; 90 u64 reserved : 13; 91 u64 is_io : 1; 92 u64 did : 8; 93 u64 reserved2 : 4; 94 u64 reserved3 : 15; 95 u64 port : 9; 96 u64 queue : 9; 97 u64 reserved4 : 3; 98 } s; 99} cvmx_pko_doorbell_address_t; 100 101/** 102 * Structure of the first packet output command word. 103 */ 104typedef union { 105 u64 u64; 106 struct { 107 cvmx_fau_op_size_t size1 : 2; 108 cvmx_fau_op_size_t size0 : 2; 109 u64 subone1 : 1; 110 u64 reg1 : 11; 111 u64 subone0 : 1; 112 u64 reg0 : 11; 113 u64 le : 1; 114 u64 n2 : 1; 115 u64 wqp : 1; 116 u64 rsp : 1; 117 u64 gather : 1; 118 u64 ipoffp1 : 7; 119 u64 ignore_i : 1; 120 u64 dontfree : 1; 121 u64 segs : 6; 122 u64 total_bytes : 16; 123 } s; 124} cvmx_pko_command_word0_t; 125 126/** 127 * Call before any other calls to initialize the packet 128 * output system. 129 */ 130 131void cvmx_pko_hw_init(u8 pool, unsigned int bufsize); 132 133/** 134 * Enables the packet output hardware. It must already be 135 * configured. 136 */ 137void cvmx_pko_enable(void); 138 139/** 140 * Disables the packet output. Does not affect any configuration. 141 */ 142void cvmx_pko_disable(void); 143 144/** 145 * Shutdown and free resources required by packet output. 146 */ 147 148void cvmx_pko_shutdown(void); 149 150/** 151 * Configure a output port and the associated queues for use. 152 * 153 * @param port Port to configure. 154 * @param base_queue First queue number to associate with this port. 155 * @param num_queues Number of queues t oassociate with this port 156 * @param priority Array of priority levels for each queue. Values are 157 * allowed to be 1-8. A value of 8 get 8 times the traffic 158 * of a value of 1. There must be num_queues elements in the 159 * array. 160 */ 161cvmx_pko_return_value_t cvmx_pko_config_port(int port, int base_queue, int num_queues, 162 const u8 priority[]); 163 164/** 165 * Ring the packet output doorbell. This tells the packet 166 * output hardware that "len" command words have been added 167 * to its pending list. This command includes the required 168 * CVMX_SYNCWS before the doorbell ring. 169 * 170 * WARNING: This function may have to look up the proper PKO port in 171 * the IPD port to PKO port map, and is thus slower than calling 172 * cvmx_pko_doorbell_pkoid() directly if the PKO port identifier is 173 * known. 174 * 175 * @param ipd_port The IPD port corresponding the to pko port the packet is for 176 * @param queue Queue the packet is for 177 * @param len Length of the command in 64 bit words 178 */ 179static inline void cvmx_pko_doorbell(u64 ipd_port, u64 queue, u64 len) 180{ 181 cvmx_pko_doorbell_address_t ptr; 182 u64 pko_port; 183 184 pko_port = ipd_port; 185 if (octeon_has_feature(OCTEON_FEATURE_PKND)) 186 pko_port = cvmx_helper_cfg_ipd2pko_port_base(ipd_port); 187 188 ptr.u64 = 0; 189 ptr.s.mem_space = CVMX_IO_SEG; 190 ptr.s.did = CVMX_OCT_DID_PKT_SEND; 191 ptr.s.is_io = 1; 192 ptr.s.port = pko_port; 193 ptr.s.queue = queue; 194 /* Need to make sure output queue data is in DRAM before doorbell write */ 195 CVMX_SYNCWS; 196 cvmx_write_io(ptr.u64, len); 197} 198 199/** 200 * Prepare to send a packet. This may initiate a tag switch to 201 * get exclusive access to the output queue structure, and 202 * performs other prep work for the packet send operation. 203 * 204 * cvmx_pko_send_packet_finish() MUST be called after this function is called, 205 * and must be called with the same port/queue/use_locking arguments. 206 * 207 * The use_locking parameter allows the caller to use three 208 * possible locking modes. 209 * - CVMX_PKO_LOCK_NONE 210 * - PKO doesn't do any locking. It is the responsibility 211 * of the application to make sure that no other core 212 * is accessing the same queue at the same time. 213 * - CVMX_PKO_LOCK_ATOMIC_TAG 214 * - PKO performs an atomic tagswitch to insure exclusive 215 * access to the output queue. This will maintain 216 * packet ordering on output. 217 * - CVMX_PKO_LOCK_CMD_QUEUE 218 * - PKO uses the common command queue locks to insure 219 * exclusive access to the output queue. This is a 220 * memory based ll/sc. This is the most portable 221 * locking mechanism. 222 * 223 * NOTE: If atomic locking is used, the POW entry CANNOT be 224 * descheduled, as it does not contain a valid WQE pointer. 225 * 226 * @param port Port to send it on, this can be either IPD port or PKO 227 * port. 228 * @param queue Queue to use 229 * @param use_locking 230 * CVMX_PKO_LOCK_NONE, CVMX_PKO_LOCK_ATOMIC_TAG, or CVMX_PKO_LOCK_CMD_QUEUE 231 */ 232static inline void cvmx_pko_send_packet_prepare(u64 port __attribute__((unused)), u64 queue, 233 cvmx_pko_lock_t use_locking) 234{ 235 if (use_locking == CVMX_PKO_LOCK_ATOMIC_TAG) { 236 /* 237 * Must do a full switch here to handle all cases. We use a 238 * fake WQE pointer, as the POW does not access this memory. 239 * The WQE pointer and group are only used if this work is 240 * descheduled, which is not supported by the 241 * cvmx_pko_send_packet_prepare/cvmx_pko_send_packet_finish 242 * combination. Note that this is a special case in which these 243 * fake values can be used - this is not a general technique. 244 */ 245 u32 tag = CVMX_TAG_SW_BITS_INTERNAL << CVMX_TAG_SW_SHIFT | 246 CVMX_TAG_SUBGROUP_PKO << CVMX_TAG_SUBGROUP_SHIFT | 247 (CVMX_TAG_SUBGROUP_MASK & queue); 248 cvmx_pow_tag_sw_full((cvmx_wqe_t *)cvmx_phys_to_ptr(0x80), tag, 249 CVMX_POW_TAG_TYPE_ATOMIC, 0); 250 } 251} 252 253#define cvmx_pko_send_packet_prepare_pkoid cvmx_pko_send_packet_prepare 254 255/** 256 * Complete packet output. cvmx_pko_send_packet_prepare() must be called exactly once before this, 257 * and the same parameters must be passed to both cvmx_pko_send_packet_prepare() and 258 * cvmx_pko_send_packet_finish(). 259 * 260 * WARNING: This function may have to look up the proper PKO port in 261 * the IPD port to PKO port map, and is thus slower than calling 262 * cvmx_pko_send_packet_finish_pkoid() directly if the PKO port 263 * identifier is known. 264 * 265 * @param ipd_port The IPD port corresponding the to pko port the packet is for 266 * @param queue Queue to use 267 * @param pko_command 268 * PKO HW command word 269 * @param packet Packet to send 270 * @param use_locking 271 * CVMX_PKO_LOCK_NONE, CVMX_PKO_LOCK_ATOMIC_TAG, or CVMX_PKO_LOCK_CMD_QUEUE 272 * 273 * Return: returns CVMX_PKO_SUCCESS on success, or error code on failure of output 274 */ 275static inline cvmx_pko_return_value_t 276cvmx_hwpko_send_packet_finish(u64 ipd_port, u64 queue, cvmx_pko_command_word0_t pko_command, 277 cvmx_buf_ptr_t packet, cvmx_pko_lock_t use_locking) 278{ 279 cvmx_cmd_queue_result_t result; 280 281 if (use_locking == CVMX_PKO_LOCK_ATOMIC_TAG) 282 cvmx_pow_tag_sw_wait(); 283 284 result = cvmx_cmd_queue_write2(CVMX_CMD_QUEUE_PKO(queue), 285 (use_locking == CVMX_PKO_LOCK_CMD_QUEUE), pko_command.u64, 286 packet.u64); 287 if (cvmx_likely(result == CVMX_CMD_QUEUE_SUCCESS)) { 288 cvmx_pko_doorbell(ipd_port, queue, 2); 289 return CVMX_PKO_SUCCESS; 290 } else if ((result == CVMX_CMD_QUEUE_NO_MEMORY) || (result == CVMX_CMD_QUEUE_FULL)) { 291 return CVMX_PKO_NO_MEMORY; 292 } else { 293 return CVMX_PKO_INVALID_QUEUE; 294 } 295} 296 297/** 298 * Complete packet output. cvmx_pko_send_packet_prepare() must be called exactly once before this, 299 * and the same parameters must be passed to both cvmx_pko_send_packet_prepare() and 300 * cvmx_pko_send_packet_finish(). 301 * 302 * WARNING: This function may have to look up the proper PKO port in 303 * the IPD port to PKO port map, and is thus slower than calling 304 * cvmx_pko_send_packet_finish3_pkoid() directly if the PKO port 305 * identifier is known. 306 * 307 * @param ipd_port The IPD port corresponding the to pko port the packet is for 308 * @param queue Queue to use 309 * @param pko_command 310 * PKO HW command word 311 * @param packet Packet to send 312 * @param addr Plysical address of a work queue entry or physical address to zero on complete. 313 * @param use_locking 314 * CVMX_PKO_LOCK_NONE, CVMX_PKO_LOCK_ATOMIC_TAG, or CVMX_PKO_LOCK_CMD_QUEUE 315 * 316 * Return: returns CVMX_PKO_SUCCESS on success, or error code on failure of output 317 */ 318static inline cvmx_pko_return_value_t 319cvmx_hwpko_send_packet_finish3(u64 ipd_port, u64 queue, cvmx_pko_command_word0_t pko_command, 320 cvmx_buf_ptr_t packet, u64 addr, cvmx_pko_lock_t use_locking) 321{ 322 cvmx_cmd_queue_result_t result; 323 324 if (use_locking == CVMX_PKO_LOCK_ATOMIC_TAG) 325 cvmx_pow_tag_sw_wait(); 326 327 result = cvmx_cmd_queue_write3(CVMX_CMD_QUEUE_PKO(queue), 328 (use_locking == CVMX_PKO_LOCK_CMD_QUEUE), pko_command.u64, 329 packet.u64, addr); 330 if (cvmx_likely(result == CVMX_CMD_QUEUE_SUCCESS)) { 331 cvmx_pko_doorbell(ipd_port, queue, 3); 332 return CVMX_PKO_SUCCESS; 333 } else if ((result == CVMX_CMD_QUEUE_NO_MEMORY) || (result == CVMX_CMD_QUEUE_FULL)) { 334 return CVMX_PKO_NO_MEMORY; 335 } else { 336 return CVMX_PKO_INVALID_QUEUE; 337 } 338} 339 340/** 341 * Get the first pko_port for the (interface, index) 342 * 343 * @param interface 344 * @param index 345 */ 346int cvmx_pko_get_base_pko_port(int interface, int index); 347 348/** 349 * Get the number of pko_ports for the (interface, index) 350 * 351 * @param interface 352 * @param index 353 */ 354int cvmx_pko_get_num_pko_ports(int interface, int index); 355 356/** 357 * For a given port number, return the base pko output queue 358 * for the port. 359 * 360 * @param port IPD port number 361 * Return: Base output queue 362 */ 363int cvmx_pko_get_base_queue(int port); 364 365/** 366 * For a given port number, return the number of pko output queues. 367 * 368 * @param port IPD port number 369 * Return: Number of output queues 370 */ 371int cvmx_pko_get_num_queues(int port); 372 373/** 374 * Sets the internal FPA pool data structure for PKO comamnd queue. 375 * @param pool fpa pool number yo use 376 * @param buffer_size buffer size of pool 377 * @param buffer_count number of buufers to allocate to pool 378 * 379 * @note the caller is responsable for setting up the pool with 380 * an appropriate buffer size and sufficient buffer count. 381 */ 382void cvmx_pko_set_cmd_que_pool_config(s64 pool, u64 buffer_size, u64 buffer_count); 383 384/** 385 * Get the status counters for a port. 386 * 387 * @param ipd_port Port number (ipd_port) to get statistics for. 388 * @param clear Set to 1 to clear the counters after they are read 389 * @param status Where to put the results. 390 * 391 * Note: 392 * - Only the doorbell for the base queue of the ipd_port is 393 * collected. 394 * - Retrieving the stats involves writing the index through 395 * CVMX_PKO_REG_READ_IDX and reading the stat CSRs, in that 396 * order. It is not MP-safe and caller should guarantee 397 * atomicity. 398 */ 399void cvmx_pko_get_port_status(u64 ipd_port, u64 clear, cvmx_pko_port_status_t *status); 400 401/** 402 * Rate limit a PKO port to a max packets/sec. This function is only 403 * supported on CN57XX, CN56XX, CN55XX, and CN54XX. 404 * 405 * @param port Port to rate limit 406 * @param packets_s Maximum packet/sec 407 * @param burst Maximum number of packets to burst in a row before rate 408 * limiting cuts in. 409 * 410 * Return: Zero on success, negative on failure 411 */ 412int cvmx_pko_rate_limit_packets(int port, int packets_s, int burst); 413 414/** 415 * Rate limit a PKO port to a max bits/sec. This function is only 416 * supported on CN57XX, CN56XX, CN55XX, and CN54XX. 417 * 418 * @param port Port to rate limit 419 * @param bits_s PKO rate limit in bits/sec 420 * @param burst Maximum number of bits to burst before rate 421 * limiting cuts in. 422 * 423 * Return: Zero on success, negative on failure 424 */ 425int cvmx_pko_rate_limit_bits(int port, u64 bits_s, int burst); 426 427/** 428 * @INTERNAL 429 * 430 * Retrieve the PKO pipe number for a port 431 * 432 * @param interface 433 * @param index 434 * 435 * Return: negative on error. 436 * 437 * This applies only to the non-loopback interfaces. 438 * 439 */ 440int __cvmx_pko_get_pipe(int interface, int index); 441 442/** 443 * For a given PKO port number, return the base output queue 444 * for the port. 445 * 446 * @param pko_port PKO port number 447 * Return: Base output queue 448 */ 449int cvmx_pko_get_base_queue_pkoid(int pko_port); 450 451/** 452 * For a given PKO port number, return the number of output queues 453 * for the port. 454 * 455 * @param pko_port PKO port number 456 * Return: the number of output queues 457 */ 458int cvmx_pko_get_num_queues_pkoid(int pko_port); 459 460/** 461 * Ring the packet output doorbell. This tells the packet 462 * output hardware that "len" command words have been added 463 * to its pending list. This command includes the required 464 * CVMX_SYNCWS before the doorbell ring. 465 * 466 * @param pko_port Port the packet is for 467 * @param queue Queue the packet is for 468 * @param len Length of the command in 64 bit words 469 */ 470static inline void cvmx_pko_doorbell_pkoid(u64 pko_port, u64 queue, u64 len) 471{ 472 cvmx_pko_doorbell_address_t ptr; 473 474 ptr.u64 = 0; 475 ptr.s.mem_space = CVMX_IO_SEG; 476 ptr.s.did = CVMX_OCT_DID_PKT_SEND; 477 ptr.s.is_io = 1; 478 ptr.s.port = pko_port; 479 ptr.s.queue = queue; 480 /* Need to make sure output queue data is in DRAM before doorbell write */ 481 CVMX_SYNCWS; 482 cvmx_write_io(ptr.u64, len); 483} 484 485/** 486 * Complete packet output. cvmx_pko_send_packet_prepare() must be called exactly once before this, 487 * and the same parameters must be passed to both cvmx_pko_send_packet_prepare() and 488 * cvmx_pko_send_packet_finish_pkoid(). 489 * 490 * @param pko_port Port to send it on 491 * @param queue Queue to use 492 * @param pko_command 493 * PKO HW command word 494 * @param packet Packet to send 495 * @param use_locking 496 * CVMX_PKO_LOCK_NONE, CVMX_PKO_LOCK_ATOMIC_TAG, or CVMX_PKO_LOCK_CMD_QUEUE 497 * 498 * Return: returns CVMX_PKO_SUCCESS on success, or error code on failure of output 499 */ 500static inline cvmx_pko_return_value_t 501cvmx_hwpko_send_packet_finish_pkoid(int pko_port, u64 queue, cvmx_pko_command_word0_t pko_command, 502 cvmx_buf_ptr_t packet, cvmx_pko_lock_t use_locking) 503{ 504 cvmx_cmd_queue_result_t result; 505 506 if (use_locking == CVMX_PKO_LOCK_ATOMIC_TAG) 507 cvmx_pow_tag_sw_wait(); 508 509 result = cvmx_cmd_queue_write2(CVMX_CMD_QUEUE_PKO(queue), 510 (use_locking == CVMX_PKO_LOCK_CMD_QUEUE), pko_command.u64, 511 packet.u64); 512 if (cvmx_likely(result == CVMX_CMD_QUEUE_SUCCESS)) { 513 cvmx_pko_doorbell_pkoid(pko_port, queue, 2); 514 return CVMX_PKO_SUCCESS; 515 } else if ((result == CVMX_CMD_QUEUE_NO_MEMORY) || (result == CVMX_CMD_QUEUE_FULL)) { 516 return CVMX_PKO_NO_MEMORY; 517 } else { 518 return CVMX_PKO_INVALID_QUEUE; 519 } 520} 521 522/** 523 * Complete packet output. cvmx_pko_send_packet_prepare() must be called exactly once before this, 524 * and the same parameters must be passed to both cvmx_pko_send_packet_prepare() and 525 * cvmx_pko_send_packet_finish_pkoid(). 526 * 527 * @param pko_port The PKO port the packet is for 528 * @param queue Queue to use 529 * @param pko_command 530 * PKO HW command word 531 * @param packet Packet to send 532 * @param addr Plysical address of a work queue entry or physical address to zero on complete. 533 * @param use_locking 534 * CVMX_PKO_LOCK_NONE, CVMX_PKO_LOCK_ATOMIC_TAG, or CVMX_PKO_LOCK_CMD_QUEUE 535 * 536 * Return: returns CVMX_PKO_SUCCESS on success, or error code on failure of output 537 */ 538static inline cvmx_pko_return_value_t 539cvmx_hwpko_send_packet_finish3_pkoid(u64 pko_port, u64 queue, cvmx_pko_command_word0_t pko_command, 540 cvmx_buf_ptr_t packet, u64 addr, cvmx_pko_lock_t use_locking) 541{ 542 cvmx_cmd_queue_result_t result; 543 544 if (use_locking == CVMX_PKO_LOCK_ATOMIC_TAG) 545 cvmx_pow_tag_sw_wait(); 546 547 result = cvmx_cmd_queue_write3(CVMX_CMD_QUEUE_PKO(queue), 548 (use_locking == CVMX_PKO_LOCK_CMD_QUEUE), pko_command.u64, 549 packet.u64, addr); 550 if (cvmx_likely(result == CVMX_CMD_QUEUE_SUCCESS)) { 551 cvmx_pko_doorbell_pkoid(pko_port, queue, 3); 552 return CVMX_PKO_SUCCESS; 553 } else if ((result == CVMX_CMD_QUEUE_NO_MEMORY) || (result == CVMX_CMD_QUEUE_FULL)) { 554 return CVMX_PKO_NO_MEMORY; 555 } else { 556 return CVMX_PKO_INVALID_QUEUE; 557 } 558} 559 560/* 561 * Obtain the number of PKO commands pending in a queue 562 * 563 * @param queue is the queue identifier to be queried 564 * Return: the number of commands pending transmission or -1 on error 565 */ 566int cvmx_pko_queue_pend_count(cvmx_cmd_queue_id_t queue); 567 568void cvmx_pko_set_cmd_queue_pool_buffer_count(u64 buffer_count); 569 570#endif /* __CVMX_HWPKO_H__ */ 571