1325618Ssbruno/* 2325618Ssbruno * BSD LICENSE 3325618Ssbruno * 4325618Ssbruno * Copyright(c) 2017 Cavium, Inc.. All rights reserved. 5325618Ssbruno * All rights reserved. 6325618Ssbruno * 7325618Ssbruno * Redistribution and use in source and binary forms, with or without 8325618Ssbruno * modification, are permitted provided that the following conditions 9325618Ssbruno * are met: 10325618Ssbruno * 11325618Ssbruno * * Redistributions of source code must retain the above copyright 12325618Ssbruno * notice, this list of conditions and the following disclaimer. 13325618Ssbruno * * Redistributions in binary form must reproduce the above copyright 14325618Ssbruno * notice, this list of conditions and the following disclaimer in 15325618Ssbruno * the documentation and/or other materials provided with the 16325618Ssbruno * distribution. 17325618Ssbruno * * Neither the name of Cavium, Inc. nor the names of its 18325618Ssbruno * contributors may be used to endorse or promote products derived 19325618Ssbruno * from this software without specific prior written permission. 20325618Ssbruno * 21325618Ssbruno * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22325618Ssbruno * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23325618Ssbruno * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24325618Ssbruno * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25325618Ssbruno * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26325618Ssbruno * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27325618Ssbruno * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28325618Ssbruno * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29325618Ssbruno * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30325618Ssbruno * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31325618Ssbruno * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32325618Ssbruno */ 33325618Ssbruno/*$FreeBSD: stable/11/sys/dev/liquidio/base/lio_droq.c 325618 2017-11-09 19:52:56Z sbruno $*/ 34325618Ssbruno 35325618Ssbruno#include "lio_bsd.h" 36325618Ssbruno#include "lio_common.h" 37325618Ssbruno#include "lio_droq.h" 38325618Ssbruno#include "lio_iq.h" 39325618Ssbruno#include "lio_response_manager.h" 40325618Ssbruno#include "lio_device.h" 41325618Ssbruno#include "lio_main.h" 42325618Ssbruno#include "cn23xx_pf_device.h" 43325618Ssbruno#include "lio_network.h" 44325618Ssbruno 45325618Ssbrunostruct __dispatch { 46325618Ssbruno struct lio_stailq_node node; 47325618Ssbruno struct lio_recv_info *rinfo; 48325618Ssbruno lio_dispatch_fn_t disp_fn; 49325618Ssbruno}; 50325618Ssbruno 51325618Ssbrunovoid *lio_get_dispatch_arg(struct octeon_device *oct, 52325618Ssbruno uint16_t opcode, uint16_t subcode); 53325618Ssbruno 54325618Ssbruno/* 55325618Ssbruno * Get the argument that the user set when registering dispatch 56325618Ssbruno * function for a given opcode/subcode. 57325618Ssbruno * @param octeon_dev - the octeon device pointer. 58325618Ssbruno * @param opcode - the opcode for which the dispatch argument 59325618Ssbruno * is to be checked. 60325618Ssbruno * @param subcode - the subcode for which the dispatch argument 61325618Ssbruno * is to be checked. 62325618Ssbruno * @return Success: void * (argument to the dispatch function) 63325618Ssbruno * @return Failure: NULL 64325618Ssbruno * 65325618Ssbruno */ 66325618Ssbrunovoid * 67325618Ssbrunolio_get_dispatch_arg(struct octeon_device *octeon_dev, 68325618Ssbruno uint16_t opcode, uint16_t subcode) 69325618Ssbruno{ 70325618Ssbruno struct lio_stailq_node *dispatch; 71325618Ssbruno void *fn_arg = NULL; 72325618Ssbruno int idx; 73325618Ssbruno uint16_t combined_opcode; 74325618Ssbruno 75325618Ssbruno combined_opcode = LIO_OPCODE_SUBCODE(opcode, subcode); 76325618Ssbruno 77325618Ssbruno idx = combined_opcode & LIO_OPCODE_MASK; 78325618Ssbruno 79325618Ssbruno mtx_lock(&octeon_dev->dispatch.lock); 80325618Ssbruno 81325618Ssbruno if (octeon_dev->dispatch.count == 0) { 82325618Ssbruno mtx_unlock(&octeon_dev->dispatch.lock); 83325618Ssbruno return (NULL); 84325618Ssbruno } 85325618Ssbruno 86325618Ssbruno if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) { 87325618Ssbruno fn_arg = octeon_dev->dispatch.dlist[idx].arg; 88325618Ssbruno } else { 89325618Ssbruno STAILQ_FOREACH(dispatch, 90325618Ssbruno &octeon_dev->dispatch.dlist[idx].head, entries) { 91325618Ssbruno if (((struct lio_dispatch *)dispatch)->opcode == 92325618Ssbruno combined_opcode) { 93325618Ssbruno fn_arg = ((struct lio_dispatch *)dispatch)->arg; 94325618Ssbruno break; 95325618Ssbruno } 96325618Ssbruno } 97325618Ssbruno } 98325618Ssbruno 99325618Ssbruno mtx_unlock(&octeon_dev->dispatch.lock); 100325618Ssbruno return (fn_arg); 101325618Ssbruno} 102325618Ssbruno 103325618Ssbruno/* 104325618Ssbruno * Check for packets on Droq. This function should be called with lock held. 105325618Ssbruno * @param droq - Droq on which count is checked. 106325618Ssbruno * @return Returns packet count. 107325618Ssbruno */ 108325618Ssbrunouint32_t 109325618Ssbrunolio_droq_check_hw_for_pkts(struct lio_droq *droq) 110325618Ssbruno{ 111325618Ssbruno struct octeon_device *oct = droq->oct_dev; 112325618Ssbruno uint32_t last_count; 113325618Ssbruno uint32_t pkt_count = 0; 114325618Ssbruno 115325618Ssbruno pkt_count = lio_read_csr32(oct, droq->pkts_sent_reg); 116325618Ssbruno 117325618Ssbruno last_count = pkt_count - droq->pkt_count; 118325618Ssbruno droq->pkt_count = pkt_count; 119325618Ssbruno 120325618Ssbruno /* we shall write to cnts at the end of processing */ 121325618Ssbruno if (last_count) 122325618Ssbruno atomic_add_int(&droq->pkts_pending, last_count); 123325618Ssbruno 124325618Ssbruno return (last_count); 125325618Ssbruno} 126325618Ssbruno 127325618Ssbrunostatic void 128325618Ssbrunolio_droq_compute_max_packet_bufs(struct lio_droq *droq) 129325618Ssbruno{ 130325618Ssbruno uint32_t count = 0; 131325618Ssbruno 132325618Ssbruno /* 133325618Ssbruno * max_empty_descs is the max. no. of descs that can have no buffers. 134325618Ssbruno * If the empty desc count goes beyond this value, we cannot safely 135325618Ssbruno * read in a 64K packet sent by Octeon 136325618Ssbruno * (64K is max pkt size from Octeon) 137325618Ssbruno */ 138325618Ssbruno droq->max_empty_descs = 0; 139325618Ssbruno 140325618Ssbruno do { 141325618Ssbruno droq->max_empty_descs++; 142325618Ssbruno count += droq->buffer_size; 143325618Ssbruno } while (count < (64 * 1024)); 144325618Ssbruno 145325618Ssbruno droq->max_empty_descs = droq->max_count - droq->max_empty_descs; 146325618Ssbruno} 147325618Ssbruno 148325618Ssbrunostatic void 149325618Ssbrunolio_droq_reset_indices(struct lio_droq *droq) 150325618Ssbruno{ 151325618Ssbruno 152325618Ssbruno droq->read_idx = 0; 153325618Ssbruno droq->refill_idx = 0; 154325618Ssbruno droq->refill_count = 0; 155325618Ssbruno atomic_store_rel_int(&droq->pkts_pending, 0); 156325618Ssbruno} 157325618Ssbruno 158325618Ssbrunostatic void 159325618Ssbrunolio_droq_destroy_ring_buffers(struct octeon_device *oct, 160325618Ssbruno struct lio_droq *droq) 161325618Ssbruno{ 162325618Ssbruno uint32_t i; 163325618Ssbruno 164325618Ssbruno for (i = 0; i < droq->max_count; i++) { 165325618Ssbruno if (droq->recv_buf_list[i].buffer != NULL) { 166325618Ssbruno lio_recv_buffer_free(droq->recv_buf_list[i].buffer); 167325618Ssbruno droq->recv_buf_list[i].buffer = NULL; 168325618Ssbruno } 169325618Ssbruno } 170325618Ssbruno 171325618Ssbruno lio_droq_reset_indices(droq); 172325618Ssbruno} 173325618Ssbruno 174325618Ssbrunostatic int 175325618Ssbrunolio_droq_setup_ring_buffers(struct octeon_device *oct, 176325618Ssbruno struct lio_droq *droq) 177325618Ssbruno{ 178325618Ssbruno struct lio_droq_desc *desc_ring = droq->desc_ring; 179325618Ssbruno void *buf; 180325618Ssbruno uint32_t i; 181325618Ssbruno 182325618Ssbruno for (i = 0; i < droq->max_count; i++) { 183325618Ssbruno buf = lio_recv_buffer_alloc(droq->buffer_size); 184325618Ssbruno 185325618Ssbruno if (buf == NULL) { 186325618Ssbruno lio_dev_err(oct, "%s buffer alloc failed\n", 187325618Ssbruno __func__); 188325618Ssbruno droq->stats.rx_alloc_failure++; 189325618Ssbruno return (-ENOMEM); 190325618Ssbruno } 191325618Ssbruno 192325618Ssbruno droq->recv_buf_list[i].buffer = buf; 193325618Ssbruno droq->recv_buf_list[i].data = ((struct mbuf *)buf)->m_data; 194325618Ssbruno desc_ring[i].info_ptr = 0; 195325618Ssbruno desc_ring[i].buffer_ptr = 196325618Ssbruno lio_map_ring(oct->device, droq->recv_buf_list[i].buffer, 197325618Ssbruno droq->buffer_size); 198325618Ssbruno } 199325618Ssbruno 200325618Ssbruno lio_droq_reset_indices(droq); 201325618Ssbruno 202325618Ssbruno lio_droq_compute_max_packet_bufs(droq); 203325618Ssbruno 204325618Ssbruno return (0); 205325618Ssbruno} 206325618Ssbruno 207325618Ssbrunoint 208325618Ssbrunolio_delete_droq(struct octeon_device *oct, uint32_t q_no) 209325618Ssbruno{ 210325618Ssbruno struct lio_droq *droq = oct->droq[q_no]; 211325618Ssbruno 212325618Ssbruno lio_dev_dbg(oct, "%s[%d]\n", __func__, q_no); 213325618Ssbruno 214325618Ssbruno while (taskqueue_cancel(droq->droq_taskqueue, &droq->droq_task, NULL)) 215325618Ssbruno taskqueue_drain(droq->droq_taskqueue, &droq->droq_task); 216325618Ssbruno 217325618Ssbruno taskqueue_free(droq->droq_taskqueue); 218325618Ssbruno droq->droq_taskqueue = NULL; 219325618Ssbruno 220325618Ssbruno lio_droq_destroy_ring_buffers(oct, droq); 221325618Ssbruno free(droq->recv_buf_list, M_DEVBUF); 222325618Ssbruno 223325618Ssbruno if (droq->desc_ring != NULL) 224325618Ssbruno lio_dma_free((droq->max_count * LIO_DROQ_DESC_SIZE), 225325618Ssbruno droq->desc_ring); 226325618Ssbruno 227325618Ssbruno oct->io_qmask.oq &= ~(1ULL << q_no); 228325618Ssbruno bzero(oct->droq[q_no], sizeof(struct lio_droq)); 229325618Ssbruno oct->num_oqs--; 230325618Ssbruno 231325618Ssbruno return (0); 232325618Ssbruno} 233325618Ssbruno 234325618Ssbrunovoid 235325618Ssbrunolio_droq_bh(void *ptr, int pending __unused) 236325618Ssbruno{ 237325618Ssbruno struct lio_droq *droq = ptr; 238325618Ssbruno struct octeon_device *oct = droq->oct_dev; 239325618Ssbruno struct lio_instr_queue *iq = oct->instr_queue[droq->q_no]; 240325618Ssbruno int reschedule, tx_done = 1; 241325618Ssbruno 242325618Ssbruno reschedule = lio_droq_process_packets(oct, droq, oct->rx_budget); 243325618Ssbruno 244325618Ssbruno if (atomic_load_acq_int(&iq->instr_pending)) 245325618Ssbruno tx_done = lio_flush_iq(oct, iq, oct->tx_budget); 246325618Ssbruno 247325618Ssbruno if (reschedule || !tx_done) 248325618Ssbruno taskqueue_enqueue(droq->droq_taskqueue, &droq->droq_task); 249325618Ssbruno else 250325618Ssbruno lio_enable_irq(droq, iq); 251325618Ssbruno} 252325618Ssbruno 253325618Ssbrunoint 254325618Ssbrunolio_init_droq(struct octeon_device *oct, uint32_t q_no, 255325618Ssbruno uint32_t num_descs, uint32_t desc_size, void *app_ctx) 256325618Ssbruno{ 257325618Ssbruno struct lio_droq *droq; 258325618Ssbruno unsigned long size; 259325618Ssbruno uint32_t c_buf_size = 0, c_num_descs = 0, c_pkts_per_intr = 0; 260325618Ssbruno uint32_t c_refill_threshold = 0, desc_ring_size = 0; 261325618Ssbruno 262325618Ssbruno lio_dev_dbg(oct, "%s[%d]\n", __func__, q_no); 263325618Ssbruno 264325618Ssbruno droq = oct->droq[q_no]; 265325618Ssbruno bzero(droq, LIO_DROQ_SIZE); 266325618Ssbruno 267325618Ssbruno droq->oct_dev = oct; 268325618Ssbruno droq->q_no = q_no; 269325618Ssbruno if (app_ctx != NULL) 270325618Ssbruno droq->app_ctx = app_ctx; 271325618Ssbruno else 272325618Ssbruno droq->app_ctx = (void *)(size_t)q_no; 273325618Ssbruno 274325618Ssbruno c_num_descs = num_descs; 275325618Ssbruno c_buf_size = desc_size; 276325618Ssbruno if (LIO_CN23XX_PF(oct)) { 277325618Ssbruno struct lio_config *conf23 = LIO_CHIP_CONF(oct, cn23xx_pf); 278325618Ssbruno 279325618Ssbruno c_pkts_per_intr = 280325618Ssbruno (uint32_t)LIO_GET_OQ_PKTS_PER_INTR_CFG(conf23); 281325618Ssbruno c_refill_threshold = 282325618Ssbruno (uint32_t)LIO_GET_OQ_REFILL_THRESHOLD_CFG(conf23); 283325618Ssbruno } else { 284325618Ssbruno return (1); 285325618Ssbruno } 286325618Ssbruno 287325618Ssbruno droq->max_count = c_num_descs; 288325618Ssbruno droq->buffer_size = c_buf_size; 289325618Ssbruno 290325618Ssbruno desc_ring_size = droq->max_count * LIO_DROQ_DESC_SIZE; 291325618Ssbruno droq->desc_ring = lio_dma_alloc(desc_ring_size, &droq->desc_ring_dma); 292325618Ssbruno if (droq->desc_ring == NULL) { 293325618Ssbruno lio_dev_err(oct, "Output queue %d ring alloc failed\n", q_no); 294325618Ssbruno return (1); 295325618Ssbruno } 296325618Ssbruno 297325618Ssbruno lio_dev_dbg(oct, "droq[%d]: desc_ring: virt: 0x%p, dma: %llx\n", q_no, 298325618Ssbruno droq->desc_ring, LIO_CAST64(droq->desc_ring_dma)); 299325618Ssbruno lio_dev_dbg(oct, "droq[%d]: num_desc: %d\n", q_no, droq->max_count); 300325618Ssbruno 301325618Ssbruno size = droq->max_count * LIO_DROQ_RECVBUF_SIZE; 302325618Ssbruno droq->recv_buf_list = 303325618Ssbruno (struct lio_recv_buffer *)malloc(size, M_DEVBUF, 304325618Ssbruno M_NOWAIT | M_ZERO); 305325618Ssbruno if (droq->recv_buf_list == NULL) { 306325618Ssbruno lio_dev_err(oct, "Output queue recv buf list alloc failed\n"); 307325618Ssbruno goto init_droq_fail; 308325618Ssbruno } 309325618Ssbruno 310325618Ssbruno if (lio_droq_setup_ring_buffers(oct, droq)) 311325618Ssbruno goto init_droq_fail; 312325618Ssbruno 313325618Ssbruno droq->pkts_per_intr = c_pkts_per_intr; 314325618Ssbruno droq->refill_threshold = c_refill_threshold; 315325618Ssbruno 316325618Ssbruno lio_dev_dbg(oct, "DROQ INIT: max_empty_descs: %d\n", 317325618Ssbruno droq->max_empty_descs); 318325618Ssbruno 319325618Ssbruno mtx_init(&droq->lock, "droq_lock", NULL, MTX_DEF); 320325618Ssbruno 321325618Ssbruno STAILQ_INIT(&droq->dispatch_stq_head); 322325618Ssbruno 323325618Ssbruno oct->fn_list.setup_oq_regs(oct, q_no); 324325618Ssbruno 325325618Ssbruno oct->io_qmask.oq |= BIT_ULL(q_no); 326325618Ssbruno 327325618Ssbruno /* 328325618Ssbruno * Initialize the taskqueue that handles 329325618Ssbruno * output queue packet processing. 330325618Ssbruno */ 331325618Ssbruno lio_dev_dbg(oct, "Initializing droq%d taskqueue\n", q_no); 332325618Ssbruno TASK_INIT(&droq->droq_task, 0, lio_droq_bh, (void *)droq); 333325618Ssbruno 334325618Ssbruno droq->droq_taskqueue = taskqueue_create_fast("lio_droq_task", M_NOWAIT, 335325618Ssbruno taskqueue_thread_enqueue, 336325618Ssbruno &droq->droq_taskqueue); 337325618Ssbruno taskqueue_start_threads_cpuset(&droq->droq_taskqueue, 1, PI_NET, 338325618Ssbruno &oct->ioq_vector[q_no].affinity_mask, 339325618Ssbruno "lio%d_droq%d_task", oct->octeon_id, 340325618Ssbruno q_no); 341325618Ssbruno 342325618Ssbruno return (0); 343325618Ssbruno 344325618Ssbrunoinit_droq_fail: 345325618Ssbruno lio_delete_droq(oct, q_no); 346325618Ssbruno return (1); 347325618Ssbruno} 348325618Ssbruno 349325618Ssbruno/* 350325618Ssbruno * lio_create_recv_info 351325618Ssbruno * Parameters: 352325618Ssbruno * octeon_dev - pointer to the octeon device structure 353325618Ssbruno * droq - droq in which the packet arrived. 354325618Ssbruno * buf_cnt - no. of buffers used by the packet. 355325618Ssbruno * idx - index in the descriptor for the first buffer in the packet. 356325618Ssbruno * Description: 357325618Ssbruno * Allocates a recv_info_t and copies the buffer addresses for packet data 358325618Ssbruno * into the recv_pkt space which starts at an 8B offset from recv_info_t. 359325618Ssbruno * Flags the descriptors for refill later. If available descriptors go 360325618Ssbruno * below the threshold to receive a 64K pkt, new buffers are first allocated 361325618Ssbruno * before the recv_pkt_t is created. 362325618Ssbruno * This routine will be called in interrupt context. 363325618Ssbruno * Returns: 364325618Ssbruno * Success: Pointer to recv_info_t 365325618Ssbruno * Failure: NULL. 366325618Ssbruno * Locks: 367325618Ssbruno * The droq->lock is held when this routine is called. 368325618Ssbruno */ 369325618Ssbrunostatic inline struct lio_recv_info * 370325618Ssbrunolio_create_recv_info(struct octeon_device *octeon_dev, struct lio_droq *droq, 371325618Ssbruno uint32_t buf_cnt, uint32_t idx) 372325618Ssbruno{ 373325618Ssbruno struct lio_droq_info *info; 374325618Ssbruno struct lio_recv_pkt *recv_pkt; 375325618Ssbruno struct lio_recv_info *recv_info; 376325618Ssbruno uint32_t bytes_left, i; 377325618Ssbruno 378325618Ssbruno info = (struct lio_droq_info *)droq->recv_buf_list[idx].data; 379325618Ssbruno 380325618Ssbruno recv_info = lio_alloc_recv_info(sizeof(struct __dispatch)); 381325618Ssbruno if (recv_info == NULL) 382325618Ssbruno return (NULL); 383325618Ssbruno 384325618Ssbruno recv_pkt = recv_info->recv_pkt; 385325618Ssbruno recv_pkt->rh = info->rh; 386325618Ssbruno recv_pkt->length = (uint32_t)info->length; 387325618Ssbruno recv_pkt->buffer_count = (uint16_t)buf_cnt; 388325618Ssbruno recv_pkt->octeon_id = (uint16_t)octeon_dev->octeon_id; 389325618Ssbruno 390325618Ssbruno i = 0; 391325618Ssbruno bytes_left = (uint32_t)info->length; 392325618Ssbruno 393325618Ssbruno while (buf_cnt) { 394325618Ssbruno recv_pkt->buffer_size[i] = (bytes_left >= droq->buffer_size) ? 395325618Ssbruno droq->buffer_size : bytes_left; 396325618Ssbruno 397325618Ssbruno recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer; 398325618Ssbruno droq->recv_buf_list[idx].buffer = NULL; 399325618Ssbruno 400325618Ssbruno idx = lio_incr_index(idx, 1, droq->max_count); 401325618Ssbruno bytes_left -= droq->buffer_size; 402325618Ssbruno i++; 403325618Ssbruno buf_cnt--; 404325618Ssbruno } 405325618Ssbruno 406325618Ssbruno return (recv_info); 407325618Ssbruno} 408325618Ssbruno 409325618Ssbruno/* 410325618Ssbruno * If we were not able to refill all buffers, try to move around 411325618Ssbruno * the buffers that were not dispatched. 412325618Ssbruno */ 413325618Ssbrunostatic inline uint32_t 414325618Ssbrunolio_droq_refill_pullup_descs(struct lio_droq *droq, 415325618Ssbruno struct lio_droq_desc *desc_ring) 416325618Ssbruno{ 417325618Ssbruno uint32_t desc_refilled = 0; 418325618Ssbruno uint32_t refill_index = droq->refill_idx; 419325618Ssbruno 420325618Ssbruno while (refill_index != droq->read_idx) { 421325618Ssbruno if (droq->recv_buf_list[refill_index].buffer != NULL) { 422325618Ssbruno droq->recv_buf_list[droq->refill_idx].buffer = 423325618Ssbruno droq->recv_buf_list[refill_index].buffer; 424325618Ssbruno droq->recv_buf_list[droq->refill_idx].data = 425325618Ssbruno droq->recv_buf_list[refill_index].data; 426325618Ssbruno desc_ring[droq->refill_idx].buffer_ptr = 427325618Ssbruno desc_ring[refill_index].buffer_ptr; 428325618Ssbruno droq->recv_buf_list[refill_index].buffer = NULL; 429325618Ssbruno desc_ring[refill_index].buffer_ptr = 0; 430325618Ssbruno do { 431325618Ssbruno droq->refill_idx = 432325618Ssbruno lio_incr_index(droq->refill_idx, 1, 433325618Ssbruno droq->max_count); 434325618Ssbruno desc_refilled++; 435325618Ssbruno droq->refill_count--; 436325618Ssbruno } while (droq->recv_buf_list[droq->refill_idx].buffer != 437325618Ssbruno NULL); 438325618Ssbruno } 439325618Ssbruno refill_index = lio_incr_index(refill_index, 1, droq->max_count); 440325618Ssbruno } /* while */ 441325618Ssbruno return (desc_refilled); 442325618Ssbruno} 443325618Ssbruno 444325618Ssbruno/* 445325618Ssbruno * lio_droq_refill 446325618Ssbruno * Parameters: 447325618Ssbruno * droq - droq in which descriptors require new buffers. 448325618Ssbruno * Description: 449325618Ssbruno * Called during normal DROQ processing in interrupt mode or by the poll 450325618Ssbruno * thread to refill the descriptors from which buffers were dispatched 451325618Ssbruno * to upper layers. Attempts to allocate new buffers. If that fails, moves 452325618Ssbruno * up buffers (that were not dispatched) to form a contiguous ring. 453325618Ssbruno * Returns: 454325618Ssbruno * No of descriptors refilled. 455325618Ssbruno * Locks: 456325618Ssbruno * This routine is called with droq->lock held. 457325618Ssbruno */ 458325618Ssbrunouint32_t 459325618Ssbrunolio_droq_refill(struct octeon_device *octeon_dev, struct lio_droq *droq) 460325618Ssbruno{ 461325618Ssbruno struct lio_droq_desc *desc_ring; 462325618Ssbruno void *buf = NULL; 463325618Ssbruno uint32_t desc_refilled = 0; 464325618Ssbruno uint8_t *data; 465325618Ssbruno 466325618Ssbruno desc_ring = droq->desc_ring; 467325618Ssbruno 468325618Ssbruno while (droq->refill_count && (desc_refilled < droq->max_count)) { 469325618Ssbruno /* 470325618Ssbruno * If a valid buffer exists (happens if there is no dispatch), 471325618Ssbruno * reuse 472325618Ssbruno * the buffer, else allocate. 473325618Ssbruno */ 474325618Ssbruno if (droq->recv_buf_list[droq->refill_idx].buffer == NULL) { 475325618Ssbruno buf = lio_recv_buffer_alloc(droq->buffer_size); 476325618Ssbruno /* 477325618Ssbruno * If a buffer could not be allocated, no point in 478325618Ssbruno * continuing 479325618Ssbruno */ 480325618Ssbruno if (buf == NULL) { 481325618Ssbruno droq->stats.rx_alloc_failure++; 482325618Ssbruno break; 483325618Ssbruno } 484325618Ssbruno 485325618Ssbruno droq->recv_buf_list[droq->refill_idx].buffer = buf; 486325618Ssbruno data = ((struct mbuf *)buf)->m_data; 487325618Ssbruno } else { 488325618Ssbruno data = ((struct mbuf *)droq->recv_buf_list 489325618Ssbruno [droq->refill_idx].buffer)->m_data; 490325618Ssbruno } 491325618Ssbruno 492325618Ssbruno droq->recv_buf_list[droq->refill_idx].data = data; 493325618Ssbruno 494325618Ssbruno desc_ring[droq->refill_idx].buffer_ptr = 495325618Ssbruno lio_map_ring(octeon_dev->device, 496325618Ssbruno droq->recv_buf_list[droq->refill_idx].buffer, 497325618Ssbruno droq->buffer_size); 498325618Ssbruno 499325618Ssbruno droq->refill_idx = lio_incr_index(droq->refill_idx, 1, 500325618Ssbruno droq->max_count); 501325618Ssbruno desc_refilled++; 502325618Ssbruno droq->refill_count--; 503325618Ssbruno } 504325618Ssbruno 505325618Ssbruno if (droq->refill_count) 506325618Ssbruno desc_refilled += lio_droq_refill_pullup_descs(droq, desc_ring); 507325618Ssbruno 508325618Ssbruno /* 509325618Ssbruno * if droq->refill_count 510325618Ssbruno * The refill count would not change in pass two. We only moved buffers 511325618Ssbruno * to close the gap in the ring, but we would still have the same no. of 512325618Ssbruno * buffers to refill. 513325618Ssbruno */ 514325618Ssbruno return (desc_refilled); 515325618Ssbruno} 516325618Ssbruno 517325618Ssbrunostatic inline uint32_t 518325618Ssbrunolio_droq_get_bufcount(uint32_t buf_size, uint32_t total_len) 519325618Ssbruno{ 520325618Ssbruno 521325618Ssbruno return ((total_len + buf_size - 1) / buf_size); 522325618Ssbruno} 523325618Ssbruno 524325618Ssbrunostatic int 525325618Ssbrunolio_droq_dispatch_pkt(struct octeon_device *oct, struct lio_droq *droq, 526325618Ssbruno union octeon_rh *rh, struct lio_droq_info *info) 527325618Ssbruno{ 528325618Ssbruno struct lio_recv_info *rinfo; 529325618Ssbruno lio_dispatch_fn_t disp_fn; 530325618Ssbruno uint32_t cnt; 531325618Ssbruno 532325618Ssbruno cnt = lio_droq_get_bufcount(droq->buffer_size, (uint32_t)info->length); 533325618Ssbruno 534325618Ssbruno disp_fn = lio_get_dispatch(oct, (uint16_t)rh->r.opcode, 535325618Ssbruno (uint16_t)rh->r.subcode); 536325618Ssbruno if (disp_fn) { 537325618Ssbruno rinfo = lio_create_recv_info(oct, droq, cnt, droq->read_idx); 538325618Ssbruno if (rinfo != NULL) { 539325618Ssbruno struct __dispatch *rdisp = rinfo->rsvd; 540325618Ssbruno 541325618Ssbruno rdisp->rinfo = rinfo; 542325618Ssbruno rdisp->disp_fn = disp_fn; 543325618Ssbruno rinfo->recv_pkt->rh = *rh; 544325618Ssbruno STAILQ_INSERT_TAIL(&droq->dispatch_stq_head, 545325618Ssbruno &rdisp->node, entries); 546325618Ssbruno } else { 547325618Ssbruno droq->stats.dropped_nomem++; 548325618Ssbruno } 549325618Ssbruno } else { 550325618Ssbruno lio_dev_err(oct, "DROQ: No dispatch function (opcode %u/%u)\n", 551325618Ssbruno (unsigned int)rh->r.opcode, 552325618Ssbruno (unsigned int)rh->r.subcode); 553325618Ssbruno droq->stats.dropped_nodispatch++; 554325618Ssbruno } 555325618Ssbruno 556325618Ssbruno return (cnt); 557325618Ssbruno} 558325618Ssbruno 559325618Ssbrunostatic inline void 560325618Ssbrunolio_droq_drop_packets(struct octeon_device *oct, struct lio_droq *droq, 561325618Ssbruno uint32_t cnt) 562325618Ssbruno{ 563325618Ssbruno struct lio_droq_info *info; 564325618Ssbruno uint32_t i = 0, buf_cnt; 565325618Ssbruno 566325618Ssbruno for (i = 0; i < cnt; i++) { 567325618Ssbruno info = (struct lio_droq_info *) 568325618Ssbruno droq->recv_buf_list[droq->read_idx].data; 569325618Ssbruno 570325618Ssbruno lio_swap_8B_data((uint64_t *)info, 2); 571325618Ssbruno 572325618Ssbruno if (info->length) { 573325618Ssbruno info->length += 8; 574325618Ssbruno droq->stats.bytes_received += info->length; 575325618Ssbruno buf_cnt = lio_droq_get_bufcount(droq->buffer_size, 576325618Ssbruno (uint32_t)info->length); 577325618Ssbruno } else { 578325618Ssbruno lio_dev_err(oct, "DROQ: In drop: pkt with len 0\n"); 579325618Ssbruno buf_cnt = 1; 580325618Ssbruno } 581325618Ssbruno 582325618Ssbruno droq->read_idx = lio_incr_index(droq->read_idx, buf_cnt, 583325618Ssbruno droq->max_count); 584325618Ssbruno droq->refill_count += buf_cnt; 585325618Ssbruno } 586325618Ssbruno} 587325618Ssbruno 588325618Ssbrunostatic uint32_t 589325618Ssbrunolio_droq_fast_process_packets(struct octeon_device *oct, struct lio_droq *droq, 590325618Ssbruno uint32_t pkts_to_process) 591325618Ssbruno{ 592325618Ssbruno struct lio_droq_info *info; 593325618Ssbruno union octeon_rh *rh; 594325618Ssbruno uint32_t pkt, pkt_count, total_len = 0; 595325618Ssbruno 596325618Ssbruno pkt_count = pkts_to_process; 597325618Ssbruno 598325618Ssbruno for (pkt = 0; pkt < pkt_count; pkt++) { 599325618Ssbruno struct mbuf *nicbuf = NULL; 600325618Ssbruno uint32_t pkt_len = 0; 601325618Ssbruno 602325618Ssbruno info = (struct lio_droq_info *) 603325618Ssbruno droq->recv_buf_list[droq->read_idx].data; 604325618Ssbruno 605325618Ssbruno lio_swap_8B_data((uint64_t *)info, 2); 606325618Ssbruno 607325618Ssbruno if (!info->length) { 608325618Ssbruno lio_dev_err(oct, 609325618Ssbruno "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n", 610325618Ssbruno droq->q_no, droq->read_idx, pkt_count); 611325618Ssbruno hexdump((uint8_t *)info, LIO_DROQ_INFO_SIZE, NULL, 612325618Ssbruno HD_OMIT_CHARS); 613325618Ssbruno pkt++; 614325618Ssbruno lio_incr_index(droq->read_idx, 1, droq->max_count); 615325618Ssbruno droq->refill_count++; 616325618Ssbruno break; 617325618Ssbruno } 618325618Ssbruno 619325618Ssbruno rh = &info->rh; 620325618Ssbruno 621325618Ssbruno info->length += 8; 622325618Ssbruno rh->r_dh.len += (LIO_DROQ_INFO_SIZE + 7) / 8; 623325618Ssbruno 624325618Ssbruno total_len += (uint32_t)info->length; 625325618Ssbruno if (lio_opcode_slow_path(rh)) { 626325618Ssbruno uint32_t buf_cnt; 627325618Ssbruno 628325618Ssbruno buf_cnt = lio_droq_dispatch_pkt(oct, droq, rh, info); 629325618Ssbruno droq->read_idx = lio_incr_index(droq->read_idx, buf_cnt, 630325618Ssbruno droq->max_count); 631325618Ssbruno droq->refill_count += buf_cnt; 632325618Ssbruno } else { 633325618Ssbruno if (info->length <= droq->buffer_size) { 634325618Ssbruno pkt_len = (uint32_t)info->length; 635325618Ssbruno nicbuf = droq->recv_buf_list[ 636325618Ssbruno droq->read_idx].buffer; 637325618Ssbruno nicbuf->m_len = pkt_len; 638325618Ssbruno droq->recv_buf_list[droq->read_idx].buffer = 639325618Ssbruno NULL; 640325618Ssbruno 641325618Ssbruno droq->read_idx = 642325618Ssbruno lio_incr_index(droq->read_idx, 643325618Ssbruno 1, droq->max_count); 644325618Ssbruno droq->refill_count++; 645325618Ssbruno } else { 646325618Ssbruno bool secondary_frag = false; 647325618Ssbruno 648325618Ssbruno pkt_len = 0; 649325618Ssbruno 650325618Ssbruno while (pkt_len < info->length) { 651325618Ssbruno int frag_len, idx = droq->read_idx; 652325618Ssbruno struct mbuf *buffer; 653325618Ssbruno 654325618Ssbruno frag_len = 655325618Ssbruno ((pkt_len + droq->buffer_size) > 656325618Ssbruno info->length) ? 657325618Ssbruno ((uint32_t)info->length - 658325618Ssbruno pkt_len) : droq->buffer_size; 659325618Ssbruno 660325618Ssbruno buffer = ((struct mbuf *) 661325618Ssbruno droq->recv_buf_list[idx]. 662325618Ssbruno buffer); 663325618Ssbruno buffer->m_len = frag_len; 664325618Ssbruno if (__predict_true(secondary_frag)) { 665325618Ssbruno m_cat(nicbuf, buffer); 666325618Ssbruno } else { 667325618Ssbruno nicbuf = buffer; 668325618Ssbruno secondary_frag = true; 669325618Ssbruno } 670325618Ssbruno 671325618Ssbruno droq->recv_buf_list[droq->read_idx]. 672325618Ssbruno buffer = NULL; 673325618Ssbruno 674325618Ssbruno pkt_len += frag_len; 675325618Ssbruno droq->read_idx = 676325618Ssbruno lio_incr_index(droq->read_idx, 677325618Ssbruno 1, 678325618Ssbruno droq->max_count); 679325618Ssbruno droq->refill_count++; 680325618Ssbruno } 681325618Ssbruno } 682325618Ssbruno 683325618Ssbruno if (nicbuf != NULL) { 684325618Ssbruno if (droq->ops.fptr != NULL) { 685325618Ssbruno droq->ops.fptr(nicbuf, pkt_len, rh, 686325618Ssbruno droq, droq->ops.farg); 687325618Ssbruno } else { 688325618Ssbruno lio_recv_buffer_free(nicbuf); 689325618Ssbruno } 690325618Ssbruno } 691325618Ssbruno } 692325618Ssbruno 693325618Ssbruno if (droq->refill_count >= droq->refill_threshold) { 694325618Ssbruno int desc_refilled = lio_droq_refill(oct, droq); 695325618Ssbruno 696325618Ssbruno /* 697325618Ssbruno * Flush the droq descriptor data to memory to be sure 698325618Ssbruno * that when we update the credits the data in memory 699325618Ssbruno * is accurate. 700325618Ssbruno */ 701325618Ssbruno wmb(); 702325618Ssbruno lio_write_csr32(oct, droq->pkts_credit_reg, 703325618Ssbruno desc_refilled); 704325618Ssbruno /* make sure mmio write completes */ 705325618Ssbruno __compiler_membar(); 706325618Ssbruno } 707325618Ssbruno } /* for (each packet)... */ 708325618Ssbruno 709325618Ssbruno /* Increment refill_count by the number of buffers processed. */ 710325618Ssbruno droq->stats.pkts_received += pkt; 711325618Ssbruno droq->stats.bytes_received += total_len; 712325618Ssbruno 713325618Ssbruno tcp_lro_flush_all(&droq->lro); 714325618Ssbruno 715325618Ssbruno if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) { 716325618Ssbruno lio_droq_drop_packets(oct, droq, (pkts_to_process - pkt)); 717325618Ssbruno 718325618Ssbruno droq->stats.dropped_toomany += (pkts_to_process - pkt); 719325618Ssbruno return (pkts_to_process); 720325618Ssbruno } 721325618Ssbruno 722325618Ssbruno return (pkt); 723325618Ssbruno} 724325618Ssbruno 725325618Ssbrunoint 726325618Ssbrunolio_droq_process_packets(struct octeon_device *oct, struct lio_droq *droq, 727325618Ssbruno uint32_t budget) 728325618Ssbruno{ 729325618Ssbruno struct lio_stailq_node *tmp, *tmp2; 730325618Ssbruno uint32_t pkt_count = 0, pkts_processed = 0; 731325618Ssbruno 732325618Ssbruno /* Grab the droq lock */ 733325618Ssbruno mtx_lock(&droq->lock); 734325618Ssbruno 735325618Ssbruno lio_droq_check_hw_for_pkts(droq); 736325618Ssbruno pkt_count = atomic_load_acq_int(&droq->pkts_pending); 737325618Ssbruno 738325618Ssbruno if (!pkt_count) { 739325618Ssbruno mtx_unlock(&droq->lock); 740325618Ssbruno return (0); 741325618Ssbruno } 742325618Ssbruno if (pkt_count > budget) 743325618Ssbruno pkt_count = budget; 744325618Ssbruno 745325618Ssbruno pkts_processed = lio_droq_fast_process_packets(oct, droq, pkt_count); 746325618Ssbruno 747325618Ssbruno atomic_subtract_int(&droq->pkts_pending, pkts_processed); 748325618Ssbruno 749325618Ssbruno /* Release the lock */ 750325618Ssbruno mtx_unlock(&droq->lock); 751325618Ssbruno 752325618Ssbruno STAILQ_FOREACH_SAFE(tmp, &droq->dispatch_stq_head, entries, tmp2) { 753325618Ssbruno struct __dispatch *rdisp = (struct __dispatch *)tmp; 754325618Ssbruno 755325618Ssbruno STAILQ_REMOVE_HEAD(&droq->dispatch_stq_head, entries); 756325618Ssbruno rdisp->disp_fn(rdisp->rinfo, lio_get_dispatch_arg(oct, 757325618Ssbruno (uint16_t)rdisp->rinfo->recv_pkt->rh.r.opcode, 758325618Ssbruno (uint16_t)rdisp->rinfo->recv_pkt->rh.r.subcode)); 759325618Ssbruno } 760325618Ssbruno 761325618Ssbruno /* If there are packets pending. schedule tasklet again */ 762325618Ssbruno if (atomic_load_acq_int(&droq->pkts_pending)) 763325618Ssbruno return (1); 764325618Ssbruno 765325618Ssbruno return (0); 766325618Ssbruno} 767325618Ssbruno 768325618Ssbrunoint 769325618Ssbrunolio_register_droq_ops(struct octeon_device *oct, uint32_t q_no, 770325618Ssbruno struct lio_droq_ops *ops) 771325618Ssbruno{ 772325618Ssbruno struct lio_droq *droq; 773325618Ssbruno struct lio_config *lio_cfg = NULL; 774325618Ssbruno 775325618Ssbruno lio_cfg = lio_get_conf(oct); 776325618Ssbruno 777325618Ssbruno if (lio_cfg == NULL) 778325618Ssbruno return (-EINVAL); 779325618Ssbruno 780325618Ssbruno if (ops == NULL) { 781325618Ssbruno lio_dev_err(oct, "%s: droq_ops pointer is NULL\n", __func__); 782325618Ssbruno return (-EINVAL); 783325618Ssbruno } 784325618Ssbruno 785325618Ssbruno if (q_no >= LIO_GET_OQ_MAX_Q_CFG(lio_cfg)) { 786325618Ssbruno lio_dev_err(oct, "%s: droq id (%d) exceeds MAX (%d)\n", 787325618Ssbruno __func__, q_no, (oct->num_oqs - 1)); 788325618Ssbruno return (-EINVAL); 789325618Ssbruno } 790325618Ssbruno droq = oct->droq[q_no]; 791325618Ssbruno 792325618Ssbruno mtx_lock(&droq->lock); 793325618Ssbruno 794325618Ssbruno memcpy(&droq->ops, ops, sizeof(struct lio_droq_ops)); 795325618Ssbruno 796325618Ssbruno mtx_unlock(&droq->lock); 797325618Ssbruno 798325618Ssbruno return (0); 799325618Ssbruno} 800325618Ssbruno 801325618Ssbrunoint 802325618Ssbrunolio_unregister_droq_ops(struct octeon_device *oct, uint32_t q_no) 803325618Ssbruno{ 804325618Ssbruno struct lio_droq *droq; 805325618Ssbruno struct lio_config *lio_cfg = NULL; 806325618Ssbruno 807325618Ssbruno lio_cfg = lio_get_conf(oct); 808325618Ssbruno 809325618Ssbruno if (lio_cfg == NULL) 810325618Ssbruno return (-EINVAL); 811325618Ssbruno 812325618Ssbruno if (q_no >= LIO_GET_OQ_MAX_Q_CFG(lio_cfg)) { 813325618Ssbruno lio_dev_err(oct, "%s: droq id (%d) exceeds MAX (%d)\n", 814325618Ssbruno __func__, q_no, oct->num_oqs - 1); 815325618Ssbruno return (-EINVAL); 816325618Ssbruno } 817325618Ssbruno 818325618Ssbruno droq = oct->droq[q_no]; 819325618Ssbruno 820325618Ssbruno if (droq == NULL) { 821325618Ssbruno lio_dev_info(oct, "Droq id (%d) not available.\n", q_no); 822325618Ssbruno return (0); 823325618Ssbruno } 824325618Ssbruno 825325618Ssbruno mtx_lock(&droq->lock); 826325618Ssbruno 827325618Ssbruno droq->ops.fptr = NULL; 828325618Ssbruno droq->ops.farg = NULL; 829325618Ssbruno droq->ops.drop_on_max = 0; 830325618Ssbruno 831325618Ssbruno mtx_unlock(&droq->lock); 832325618Ssbruno 833325618Ssbruno return (0); 834325618Ssbruno} 835325618Ssbruno 836325618Ssbrunoint 837325618Ssbrunolio_create_droq(struct octeon_device *oct, uint32_t q_no, uint32_t num_descs, 838325618Ssbruno uint32_t desc_size, void *app_ctx) 839325618Ssbruno{ 840325618Ssbruno 841325618Ssbruno if (oct->droq[q_no]->oct_dev != NULL) { 842325618Ssbruno lio_dev_dbg(oct, "Droq already in use. Cannot create droq %d again\n", 843325618Ssbruno q_no); 844325618Ssbruno return (1); 845325618Ssbruno } 846325618Ssbruno 847325618Ssbruno /* Initialize the Droq */ 848325618Ssbruno if (lio_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) { 849325618Ssbruno bzero(oct->droq[q_no], sizeof(struct lio_droq)); 850325618Ssbruno goto create_droq_fail; 851325618Ssbruno } 852325618Ssbruno 853325618Ssbruno oct->num_oqs++; 854325618Ssbruno 855325618Ssbruno lio_dev_dbg(oct, "%s: Total number of OQ: %d\n", __func__, 856325618Ssbruno oct->num_oqs); 857325618Ssbruno 858325618Ssbruno /* Global Droq register settings */ 859325618Ssbruno 860325618Ssbruno /* 861325618Ssbruno * As of now not required, as setting are done for all 32 Droqs at 862325618Ssbruno * the same time. 863325618Ssbruno */ 864325618Ssbruno return (0); 865325618Ssbruno 866325618Ssbrunocreate_droq_fail: 867325618Ssbruno return (-ENOMEM); 868325618Ssbruno} 869