1325618Ssbruno/* 2325618Ssbruno * BSD LICENSE 3325618Ssbruno * 4325618Ssbruno * Copyright(c) 2017 Cavium, Inc.. All rights reserved. 5325618Ssbruno * All rights reserved. 6325618Ssbruno * 7325618Ssbruno * Redistribution and use in source and binary forms, with or without 8325618Ssbruno * modification, are permitted provided that the following conditions 9325618Ssbruno * are met: 10325618Ssbruno * 11325618Ssbruno * * Redistributions of source code must retain the above copyright 12325618Ssbruno * notice, this list of conditions and the following disclaimer. 13325618Ssbruno * * Redistributions in binary form must reproduce the above copyright 14325618Ssbruno * notice, this list of conditions and the following disclaimer in 15325618Ssbruno * the documentation and/or other materials provided with the 16325618Ssbruno * distribution. 17325618Ssbruno * * Neither the name of Cavium, Inc. nor the names of its 18325618Ssbruno * contributors may be used to endorse or promote products derived 19325618Ssbruno * from this software without specific prior written permission. 20325618Ssbruno * 21325618Ssbruno * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22325618Ssbruno * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23325618Ssbruno * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24325618Ssbruno * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25325618Ssbruno * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26325618Ssbruno * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27325618Ssbruno * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28325618Ssbruno * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29325618Ssbruno * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30325618Ssbruno * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31325618Ssbruno * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32325618Ssbruno */ 33325618Ssbruno/*$FreeBSD: stable/11/sys/dev/liquidio/base/lio_droq.h 325618 2017-11-09 19:52:56Z sbruno $*/ 34325618Ssbruno 35325618Ssbruno/* \file lio_droq.h 36325618Ssbruno * \brief Implementation of Octeon Output queues. "Output" is with 37325618Ssbruno * respect to the Octeon device on the NIC. From this driver's point of 38325618Ssbruno * view they are ingress queues. 39325618Ssbruno */ 40325618Ssbruno 41325618Ssbruno#ifndef __LIO_DROQ_H__ 42325618Ssbruno#define __LIO_DROQ_H__ 43325618Ssbruno 44325618Ssbruno/* 45325618Ssbruno * Octeon descriptor format. 46325618Ssbruno * The descriptor ring is made of descriptors which have 2 64-bit values: 47325618Ssbruno * -# Physical (bus) address of the data buffer. 48325618Ssbruno * -# Physical (bus) address of a lio_droq_info structure. 49325618Ssbruno * The Octeon device DMA's incoming packets and its information at the address 50325618Ssbruno * given by these descriptor fields. 51325618Ssbruno */ 52325618Ssbrunostruct lio_droq_desc { 53325618Ssbruno /* The buffer pointer */ 54325618Ssbruno uint64_t buffer_ptr; 55325618Ssbruno 56325618Ssbruno /* The Info pointer */ 57325618Ssbruno uint64_t info_ptr; 58325618Ssbruno}; 59325618Ssbruno 60325618Ssbruno#define LIO_DROQ_DESC_SIZE (sizeof(struct lio_droq_desc)) 61325618Ssbruno 62325618Ssbruno/* 63325618Ssbruno * Information about packet DMA'ed by Octeon. 64325618Ssbruno * The format of the information available at Info Pointer after Octeon 65325618Ssbruno * has posted a packet. Not all descriptors have valid information. Only 66325618Ssbruno * the Info field of the first descriptor for a packet has information 67325618Ssbruno * about the packet. 68325618Ssbruno */ 69325618Ssbrunostruct lio_droq_info { 70325618Ssbruno /* The Length of the packet. */ 71325618Ssbruno uint64_t length; 72325618Ssbruno 73325618Ssbruno /* The Output Receive Header. */ 74325618Ssbruno union octeon_rh rh; 75325618Ssbruno 76325618Ssbruno}; 77325618Ssbruno 78325618Ssbruno#define LIO_DROQ_INFO_SIZE (sizeof(struct lio_droq_info)) 79325618Ssbruno 80325618Ssbruno/* 81325618Ssbruno * Pointer to data buffer. 82325618Ssbruno * Driver keeps a pointer to the data buffer that it made available to 83325618Ssbruno * the Octeon device. Since the descriptor ring keeps physical (bus) 84325618Ssbruno * addresses, this field is required for the driver to keep track of 85325618Ssbruno * the virtual address pointers. 86325618Ssbruno */ 87325618Ssbrunostruct lio_recv_buffer { 88325618Ssbruno /* Packet buffer, including metadata. */ 89325618Ssbruno void *buffer; 90325618Ssbruno 91325618Ssbruno /* Data in the packet buffer. */ 92325618Ssbruno uint8_t *data; 93325618Ssbruno}; 94325618Ssbruno 95325618Ssbruno#define LIO_DROQ_RECVBUF_SIZE (sizeof(struct lio_recv_buffer)) 96325618Ssbruno 97325618Ssbruno/* Output Queue statistics. Each output queue has four stats fields. */ 98325618Ssbrunostruct lio_droq_stats { 99325618Ssbruno /* Number of packets received in this queue. */ 100325618Ssbruno uint64_t pkts_received; 101325618Ssbruno 102325618Ssbruno /* Bytes received by this queue. */ 103325618Ssbruno uint64_t bytes_received; 104325618Ssbruno 105325618Ssbruno /* Packets dropped due to no dispatch function. */ 106325618Ssbruno uint64_t dropped_nodispatch; 107325618Ssbruno 108325618Ssbruno /* Packets dropped due to no memory available. */ 109325618Ssbruno uint64_t dropped_nomem; 110325618Ssbruno 111325618Ssbruno /* Packets dropped due to large number of pkts to process. */ 112325618Ssbruno uint64_t dropped_toomany; 113325618Ssbruno 114325618Ssbruno /* Number of packets sent to stack from this queue. */ 115325618Ssbruno uint64_t rx_pkts_received; 116325618Ssbruno 117325618Ssbruno /* Number of Bytes sent to stack from this queue. */ 118325618Ssbruno uint64_t rx_bytes_received; 119325618Ssbruno 120325618Ssbruno /* Num of Packets dropped due to receive path failures. */ 121325618Ssbruno uint64_t rx_dropped; 122325618Ssbruno 123325618Ssbruno uint64_t rx_vxlan; 124325618Ssbruno 125325618Ssbruno /* Num of failures of lio_recv_buffer_alloc() */ 126325618Ssbruno uint64_t rx_alloc_failure; 127325618Ssbruno 128325618Ssbruno}; 129325618Ssbruno 130325618Ssbruno/* 131325618Ssbruno * The maximum number of buffers that can be dispatched from the 132325618Ssbruno * output/dma queue. Set to 64 assuming 1K buffers in DROQ and the fact that 133325618Ssbruno * max packet size from DROQ is 64K. 134325618Ssbruno */ 135325618Ssbruno#define LIO_MAX_RECV_BUFS 64 136325618Ssbruno 137325618Ssbruno/* 138325618Ssbruno * Receive Packet format used when dispatching output queue packets 139325618Ssbruno * with non-raw opcodes. 140325618Ssbruno * The received packet will be sent to the upper layers using this 141325618Ssbruno * structure which is passed as a parameter to the dispatch function 142325618Ssbruno */ 143325618Ssbrunostruct lio_recv_pkt { 144325618Ssbruno /* Number of buffers in this received packet */ 145325618Ssbruno uint16_t buffer_count; 146325618Ssbruno 147325618Ssbruno /* Id of the device that is sending the packet up */ 148325618Ssbruno uint16_t octeon_id; 149325618Ssbruno 150325618Ssbruno /* Length of data in the packet buffer */ 151325618Ssbruno uint32_t length; 152325618Ssbruno 153325618Ssbruno /* The receive header */ 154325618Ssbruno union octeon_rh rh; 155325618Ssbruno 156325618Ssbruno /* Pointer to the OS-specific packet buffer */ 157325618Ssbruno struct mbuf *buffer_ptr[LIO_MAX_RECV_BUFS]; 158325618Ssbruno 159325618Ssbruno /* Size of the buffers pointed to by ptr's in buffer_ptr */ 160325618Ssbruno uint32_t buffer_size[LIO_MAX_RECV_BUFS]; 161325618Ssbruno}; 162325618Ssbruno 163325618Ssbruno#define LIO_RECV_PKT_SIZE (sizeof(struct lio_recv_pkt)) 164325618Ssbruno 165325618Ssbruno/* 166325618Ssbruno * The first parameter of a dispatch function. 167325618Ssbruno * For a raw mode opcode, the driver dispatches with the device 168325618Ssbruno * pointer in this structure. 169325618Ssbruno * For non-raw mode opcode, the driver dispatches the recv_pkt 170325618Ssbruno * created to contain the buffers with data received from Octeon. 171325618Ssbruno * --------------------- 172325618Ssbruno * | *recv_pkt ----|--- 173325618Ssbruno * |-------------------| | 174325618Ssbruno * | 0 or more bytes | | 175325618Ssbruno * | reserved by driver| | 176325618Ssbruno * |-------------------|<-/ 177325618Ssbruno * | lio_recv_pkt | 178325618Ssbruno * | | 179325618Ssbruno * |___________________| 180325618Ssbruno */ 181325618Ssbrunostruct lio_recv_info { 182325618Ssbruno void *rsvd; 183325618Ssbruno struct lio_recv_pkt *recv_pkt; 184325618Ssbruno}; 185325618Ssbruno 186325618Ssbruno#define LIO_RECV_INFO_SIZE (sizeof(struct lio_recv_info)) 187325618Ssbruno 188325618Ssbruno/* 189325618Ssbruno * Allocate a recv_info structure. The recv_pkt pointer in the recv_info 190325618Ssbruno * structure is filled in before this call returns. 191325618Ssbruno * @param extra_bytes - extra bytes to be allocated at the end of the recv info 192325618Ssbruno * structure. 193325618Ssbruno * @return - pointer to a newly allocated recv_info structure. 194325618Ssbruno */ 195325618Ssbrunostatic inline struct lio_recv_info * 196325618Ssbrunolio_alloc_recv_info(int extra_bytes) 197325618Ssbruno{ 198325618Ssbruno struct lio_recv_info *recv_info; 199325618Ssbruno uint8_t *buf; 200325618Ssbruno 201325618Ssbruno buf = malloc(LIO_RECV_PKT_SIZE + LIO_RECV_INFO_SIZE + 202325618Ssbruno extra_bytes, M_DEVBUF, M_NOWAIT | M_ZERO); 203325618Ssbruno if (buf == NULL) 204325618Ssbruno return (NULL); 205325618Ssbruno 206325618Ssbruno recv_info = (struct lio_recv_info *)buf; 207325618Ssbruno recv_info->recv_pkt = (struct lio_recv_pkt *)(buf + LIO_RECV_INFO_SIZE); 208325618Ssbruno recv_info->rsvd = NULL; 209325618Ssbruno if (extra_bytes) 210325618Ssbruno recv_info->rsvd = buf + LIO_RECV_INFO_SIZE + LIO_RECV_PKT_SIZE; 211325618Ssbruno 212325618Ssbruno return (recv_info); 213325618Ssbruno} 214325618Ssbruno 215325618Ssbruno/* 216325618Ssbruno * Free a recv_info structure. 217325618Ssbruno * @param recv_info - Pointer to receive_info to be freed 218325618Ssbruno */ 219325618Ssbrunostatic inline void 220325618Ssbrunolio_free_recv_info(struct lio_recv_info *recv_info) 221325618Ssbruno{ 222325618Ssbruno 223325618Ssbruno free(recv_info, M_DEVBUF); 224325618Ssbruno} 225325618Ssbruno 226325618Ssbrunotypedef int (*lio_dispatch_fn_t)(struct lio_recv_info *, void *); 227325618Ssbruno 228325618Ssbruno/* 229325618Ssbruno * Used by NIC module to register packet handler and to get device 230325618Ssbruno * information for each octeon device. 231325618Ssbruno */ 232325618Ssbrunostruct lio_droq_ops { 233325618Ssbruno /* 234325618Ssbruno * This registered function will be called by the driver with 235325618Ssbruno * the pointer to buffer from droq and length of 236325618Ssbruno * data in the buffer. The receive header gives the port 237325618Ssbruno * number to the caller. Function pointer is set by caller. 238325618Ssbruno */ 239325618Ssbruno void (*fptr) (void *, uint32_t, union octeon_rh *, void *, 240325618Ssbruno void *); 241325618Ssbruno void *farg; 242325618Ssbruno 243325618Ssbruno /* 244325618Ssbruno * Flag indicating if the DROQ handler should drop packets that 245325618Ssbruno * it cannot handle in one iteration. Set by caller. 246325618Ssbruno */ 247325618Ssbruno uint32_t drop_on_max; 248325618Ssbruno}; 249325618Ssbruno 250325618Ssbruno/* 251325618Ssbruno * The Descriptor Ring Output Queue structure. 252325618Ssbruno * This structure has all the information required to implement a 253325618Ssbruno * Octeon DROQ. 254325618Ssbruno */ 255325618Ssbrunostruct lio_droq { 256325618Ssbruno /* A lock to protect access to this ring. */ 257325618Ssbruno struct mtx lock; 258325618Ssbruno 259325618Ssbruno uint32_t q_no; 260325618Ssbruno 261325618Ssbruno uint32_t pkt_count; 262325618Ssbruno 263325618Ssbruno struct lio_droq_ops ops; 264325618Ssbruno 265325618Ssbruno struct octeon_device *oct_dev; 266325618Ssbruno 267325618Ssbruno /* The 8B aligned descriptor ring starts at this address. */ 268325618Ssbruno struct lio_droq_desc *desc_ring; 269325618Ssbruno 270325618Ssbruno /* Index in the ring where the driver should read the next packet */ 271325618Ssbruno uint32_t read_idx; 272325618Ssbruno 273325618Ssbruno /* 274325618Ssbruno * Index in the ring where the driver will refill the descriptor's 275325618Ssbruno * buffer 276325618Ssbruno */ 277325618Ssbruno uint32_t refill_idx; 278325618Ssbruno 279325618Ssbruno /* Packets pending to be processed */ 280325618Ssbruno volatile int pkts_pending; 281325618Ssbruno 282325618Ssbruno /* Number of descriptors in this ring. */ 283325618Ssbruno uint32_t max_count; 284325618Ssbruno 285325618Ssbruno /* The number of descriptors pending refill. */ 286325618Ssbruno uint32_t refill_count; 287325618Ssbruno 288325618Ssbruno uint32_t pkts_per_intr; 289325618Ssbruno uint32_t refill_threshold; 290325618Ssbruno 291325618Ssbruno /* 292325618Ssbruno * The max number of descriptors in DROQ without a buffer. 293325618Ssbruno * This field is used to keep track of empty space threshold. If the 294325618Ssbruno * refill_count reaches this value, the DROQ cannot accept a max-sized 295325618Ssbruno * (64K) packet. 296325618Ssbruno */ 297325618Ssbruno uint32_t max_empty_descs; 298325618Ssbruno 299325618Ssbruno /* 300325618Ssbruno * The receive buffer list. This list has the virtual addresses of 301325618Ssbruno * the buffers. 302325618Ssbruno */ 303325618Ssbruno struct lio_recv_buffer *recv_buf_list; 304325618Ssbruno 305325618Ssbruno /* The size of each buffer pointed by the buffer pointer. */ 306325618Ssbruno uint32_t buffer_size; 307325618Ssbruno 308325618Ssbruno /* 309325618Ssbruno * Offset to packet credit register. 310325618Ssbruno * Host writes number of info/buffer ptrs available to this register 311325618Ssbruno */ 312325618Ssbruno uint32_t pkts_credit_reg; 313325618Ssbruno 314325618Ssbruno /* 315325618Ssbruno * Offset packet sent register. 316325618Ssbruno * Octeon writes the number of packets DMA'ed to host memory 317325618Ssbruno * in this register. 318325618Ssbruno */ 319325618Ssbruno uint32_t pkts_sent_reg; 320325618Ssbruno 321325618Ssbruno struct lio_stailq_head dispatch_stq_head; 322325618Ssbruno 323325618Ssbruno /* Statistics for this DROQ. */ 324325618Ssbruno struct lio_droq_stats stats; 325325618Ssbruno 326325618Ssbruno /* DMA mapped address of the DROQ descriptor ring. */ 327325618Ssbruno vm_paddr_t desc_ring_dma; 328325618Ssbruno 329325618Ssbruno /* application context */ 330325618Ssbruno void *app_ctx; 331325618Ssbruno 332325618Ssbruno uint32_t cpu_id; 333325618Ssbruno 334325618Ssbruno struct task droq_task; 335325618Ssbruno struct taskqueue *droq_taskqueue; 336325618Ssbruno 337325618Ssbruno struct lro_ctrl lro; 338325618Ssbruno}; 339325618Ssbruno 340325618Ssbruno#define LIO_DROQ_SIZE (sizeof(struct lio_droq)) 341325618Ssbruno 342325618Ssbruno/* 343325618Ssbruno * Allocates space for the descriptor ring for the droq and sets the 344325618Ssbruno * base addr, num desc etc in Octeon registers. 345325618Ssbruno * 346325618Ssbruno * @param oct_dev - pointer to the octeon device structure 347325618Ssbruno * @param q_no - droq no. 348325618Ssbruno * @param app_ctx - pointer to application context 349325618Ssbruno * @return Success: 0 Failure: 1 350325618Ssbruno */ 351325618Ssbrunoint lio_init_droq(struct octeon_device *oct_dev, 352325618Ssbruno uint32_t q_no, uint32_t num_descs, uint32_t desc_size, 353325618Ssbruno void *app_ctx); 354325618Ssbruno 355325618Ssbruno/* 356325618Ssbruno * Frees the space for descriptor ring for the droq. 357325618Ssbruno * 358325618Ssbruno * @param oct_dev - pointer to the octeon device structure 359325618Ssbruno * @param q_no - droq no. 360325618Ssbruno * @return: Success: 0 Failure: 1 361325618Ssbruno */ 362325618Ssbrunoint lio_delete_droq(struct octeon_device *oct_dev, uint32_t q_no); 363325618Ssbruno 364325618Ssbruno/* 365325618Ssbruno * Register a change in droq operations. The ops field has a pointer to a 366325618Ssbruno * function which will called by the DROQ handler for all packets arriving 367325618Ssbruno * on output queues given by q_no irrespective of the type of packet. 368325618Ssbruno * The ops field also has a flag which if set tells the DROQ handler to 369325618Ssbruno * drop packets if it receives more than what it can process in one 370325618Ssbruno * invocation of the handler. 371325618Ssbruno * @param oct - octeon device 372325618Ssbruno * @param q_no - octeon output queue number (0 <= q_no <= MAX_OCTEON_DROQ-1 373325618Ssbruno * @param ops - the droq_ops settings for this queue 374325618Ssbruno * @return - 0 on success, -ENODEV or -EINVAL on error. 375325618Ssbruno */ 376325618Ssbrunoint lio_register_droq_ops(struct octeon_device *oct, uint32_t q_no, 377325618Ssbruno struct lio_droq_ops *ops); 378325618Ssbruno 379325618Ssbruno/* 380325618Ssbruno * Resets the function pointer and flag settings made by 381325618Ssbruno * lio_register_droq_ops(). After this routine is called, the DROQ handler 382325618Ssbruno * will lookup dispatch function for each arriving packet on the output queue 383325618Ssbruno * given by q_no. 384325618Ssbruno * @param oct - octeon device 385325618Ssbruno * @param q_no - octeon output queue number (0 <= q_no <= MAX_OCTEON_DROQ-1 386325618Ssbruno * @return - 0 on success, -ENODEV or -EINVAL on error. 387325618Ssbruno */ 388325618Ssbrunoint lio_unregister_droq_ops(struct octeon_device *oct, uint32_t q_no); 389325618Ssbruno 390325618Ssbruno/* 391325618Ssbruno * Register a dispatch function for a opcode/subcode. The driver will call 392325618Ssbruno * this dispatch function when it receives a packet with the given 393325618Ssbruno * opcode/subcode in its output queues along with the user specified 394325618Ssbruno * argument. 395325618Ssbruno * @param oct - the octeon device to register with. 396325618Ssbruno * @param opcode - the opcode for which the dispatch will be registered. 397325618Ssbruno * @param subcode - the subcode for which the dispatch will be registered 398325618Ssbruno * @param fn - the dispatch function. 399325618Ssbruno * @param fn_arg - user specified that will be passed along with the 400325618Ssbruno * dispatch function by the driver. 401325618Ssbruno * @return Success: 0; Failure: 1 402325618Ssbruno */ 403325618Ssbrunoint lio_register_dispatch_fn(struct octeon_device *oct, uint16_t opcode, 404325618Ssbruno uint16_t subcode, lio_dispatch_fn_t fn, 405325618Ssbruno void *fn_arg); 406325618Ssbruno 407325618Ssbruno/* 408325618Ssbruno * Remove registration for an opcode/subcode. This will delete the mapping for 409325618Ssbruno * an opcode/subcode. The dispatch function will be unregistered and will no 410325618Ssbruno * longer be called if a packet with the opcode/subcode arrives in the driver 411325618Ssbruno * output queues. 412325618Ssbruno * @param oct - the octeon device to unregister from. 413325618Ssbruno * @param opcode - the opcode to be unregistered. 414325618Ssbruno * @param subcode - the subcode to be unregistered. 415325618Ssbruno * 416325618Ssbruno * @return Success: 0; Failure: 1 417325618Ssbruno */ 418325618Ssbrunoint lio_unregister_dispatch_fn(struct octeon_device *oct, uint16_t opcode, 419325618Ssbruno uint16_t subcode); 420325618Ssbruno 421325618Ssbrunouint32_t lio_droq_check_hw_for_pkts(struct lio_droq *droq); 422325618Ssbruno 423325618Ssbrunoint lio_create_droq(struct octeon_device *oct, uint32_t q_no, 424325618Ssbruno uint32_t num_descs, uint32_t desc_size, void *app_ctx); 425325618Ssbruno 426325618Ssbrunoint lio_droq_process_packets(struct octeon_device *oct, 427325618Ssbruno struct lio_droq *droq, uint32_t budget); 428325618Ssbruno 429325618Ssbrunouint32_t lio_droq_refill(struct octeon_device *octeon_dev, 430325618Ssbruno struct lio_droq *droq); 431325618Ssbrunovoid lio_droq_bh(void *ptr, int pending __unused); 432325618Ssbruno#endif /* __LIO_DROQ_H__ */ 433