1// SPDX-License-Identifier: GPL-2.0 2/* Marvell Octeon EP (EndPoint) VF Ethernet Driver 3 * 4 * Copyright (C) 2020 Marvell. 5 * 6 */ 7 8#include <linux/pci.h> 9#include <linux/etherdevice.h> 10#include <linux/vmalloc.h> 11#include <net/netdev_queues.h> 12 13#include "octep_vf_config.h" 14#include "octep_vf_main.h" 15 16/* Reset various index of Tx queue data structure. */ 17static void octep_vf_iq_reset_indices(struct octep_vf_iq *iq) 18{ 19 iq->fill_cnt = 0; 20 iq->host_write_index = 0; 21 iq->octep_vf_read_index = 0; 22 iq->flush_index = 0; 23 iq->pkts_processed = 0; 24 iq->pkt_in_done = 0; 25} 26 27/** 28 * octep_vf_iq_process_completions() - Process Tx queue completions. 29 * 30 * @iq: Octeon Tx queue data structure. 31 * @budget: max number of completions to be processed in one invocation. 32 */ 33int octep_vf_iq_process_completions(struct octep_vf_iq *iq, u16 budget) 34{ 35 u32 compl_pkts, compl_bytes, compl_sg; 36 struct octep_vf_device *oct = iq->octep_vf_dev; 37 struct octep_vf_tx_buffer *tx_buffer; 38 struct skb_shared_info *shinfo; 39 u32 fi = iq->flush_index; 40 struct sk_buff *skb; 41 u8 frags, i; 42 43 compl_pkts = 0; 44 compl_sg = 0; 45 compl_bytes = 0; 46 iq->octep_vf_read_index = oct->hw_ops.update_iq_read_idx(iq); 47 48 while (likely(budget && (fi != iq->octep_vf_read_index))) { 49 tx_buffer = iq->buff_info + fi; 50 skb = tx_buffer->skb; 51 52 fi++; 53 if (unlikely(fi == iq->max_count)) 54 fi = 0; 55 compl_bytes += skb->len; 56 compl_pkts++; 57 budget--; 58 59 if (!tx_buffer->gather) { 60 dma_unmap_single(iq->dev, tx_buffer->dma, 61 tx_buffer->skb->len, DMA_TO_DEVICE); 62 dev_kfree_skb_any(skb); 63 continue; 64 } 65 66 /* Scatter/Gather */ 67 shinfo = skb_shinfo(skb); 68 frags = shinfo->nr_frags; 69 compl_sg++; 70 71 dma_unmap_single(iq->dev, tx_buffer->sglist[0].dma_ptr[0], 72 tx_buffer->sglist[0].len[3], DMA_TO_DEVICE); 73 74 i = 1; /* entry 0 is main skb, unmapped above */ 75 while (frags--) { 76 dma_unmap_page(iq->dev, tx_buffer->sglist[i >> 2].dma_ptr[i & 3], 77 tx_buffer->sglist[i >> 2].len[3 - (i & 3)], DMA_TO_DEVICE); 78 i++; 79 } 80 81 dev_kfree_skb_any(skb); 82 } 83 84 iq->pkts_processed += compl_pkts; 85 iq->stats.instr_completed += compl_pkts; 86 iq->stats.bytes_sent += compl_bytes; 87 iq->stats.sgentry_sent += compl_sg; 88 iq->flush_index = fi; 89 90 netif_subqueue_completed_wake(iq->netdev, iq->q_no, compl_pkts, 91 compl_bytes, IQ_INSTR_SPACE(iq), 92 OCTEP_VF_WAKE_QUEUE_THRESHOLD); 93 94 return !budget; 95} 96 97/** 98 * octep_vf_iq_free_pending() - Free Tx buffers for pending completions. 99 * 100 * @iq: Octeon Tx queue data structure. 101 */ 102static void octep_vf_iq_free_pending(struct octep_vf_iq *iq) 103{ 104 struct octep_vf_tx_buffer *tx_buffer; 105 struct skb_shared_info *shinfo; 106 u32 fi = iq->flush_index; 107 struct sk_buff *skb; 108 u8 frags, i; 109 110 while (fi != iq->host_write_index) { 111 tx_buffer = iq->buff_info + fi; 112 skb = tx_buffer->skb; 113 114 fi++; 115 if (unlikely(fi == iq->max_count)) 116 fi = 0; 117 118 if (!tx_buffer->gather) { 119 dma_unmap_single(iq->dev, tx_buffer->dma, 120 tx_buffer->skb->len, DMA_TO_DEVICE); 121 dev_kfree_skb_any(skb); 122 continue; 123 } 124 125 /* Scatter/Gather */ 126 shinfo = skb_shinfo(skb); 127 frags = shinfo->nr_frags; 128 129 dma_unmap_single(iq->dev, 130 tx_buffer->sglist[0].dma_ptr[0], 131 tx_buffer->sglist[0].len[0], 132 DMA_TO_DEVICE); 133 134 i = 1; /* entry 0 is main skb, unmapped above */ 135 while (frags--) { 136 dma_unmap_page(iq->dev, tx_buffer->sglist[i >> 2].dma_ptr[i & 3], 137 tx_buffer->sglist[i >> 2].len[i & 3], DMA_TO_DEVICE); 138 i++; 139 } 140 141 dev_kfree_skb_any(skb); 142 } 143 144 iq->flush_index = fi; 145 netdev_tx_reset_queue(netdev_get_tx_queue(iq->netdev, iq->q_no)); 146} 147 148/** 149 * octep_vf_clean_iqs() - Clean Tx queues to shutdown the device. 150 * 151 * @oct: Octeon device private data structure. 152 * 153 * Free the buffers in Tx queue descriptors pending completion and 154 * reset queue indices 155 */ 156void octep_vf_clean_iqs(struct octep_vf_device *oct) 157{ 158 int i; 159 160 for (i = 0; i < oct->num_iqs; i++) { 161 octep_vf_iq_free_pending(oct->iq[i]); 162 octep_vf_iq_reset_indices(oct->iq[i]); 163 } 164} 165 166/** 167 * octep_vf_setup_iq() - Setup a Tx queue. 168 * 169 * @oct: Octeon device private data structure. 170 * @q_no: Tx queue number to be setup. 171 * 172 * Allocate resources for a Tx queue. 173 */ 174static int octep_vf_setup_iq(struct octep_vf_device *oct, int q_no) 175{ 176 u32 desc_ring_size, buff_info_size, sglist_size; 177 struct octep_vf_iq *iq; 178 int i; 179 180 iq = vzalloc(sizeof(*iq)); 181 if (!iq) 182 goto iq_alloc_err; 183 oct->iq[q_no] = iq; 184 185 iq->octep_vf_dev = oct; 186 iq->netdev = oct->netdev; 187 iq->dev = &oct->pdev->dev; 188 iq->q_no = q_no; 189 iq->max_count = CFG_GET_IQ_NUM_DESC(oct->conf); 190 iq->ring_size_mask = iq->max_count - 1; 191 iq->fill_threshold = CFG_GET_IQ_DB_MIN(oct->conf); 192 iq->netdev_q = netdev_get_tx_queue(iq->netdev, q_no); 193 194 /* Allocate memory for hardware queue descriptors */ 195 desc_ring_size = OCTEP_VF_IQ_DESC_SIZE * CFG_GET_IQ_NUM_DESC(oct->conf); 196 iq->desc_ring = dma_alloc_coherent(iq->dev, desc_ring_size, 197 &iq->desc_ring_dma, GFP_KERNEL); 198 if (unlikely(!iq->desc_ring)) { 199 dev_err(iq->dev, 200 "Failed to allocate DMA memory for IQ-%d\n", q_no); 201 goto desc_dma_alloc_err; 202 } 203 204 /* Allocate memory for hardware SGLIST descriptors */ 205 sglist_size = OCTEP_VF_SGLIST_SIZE_PER_PKT * 206 CFG_GET_IQ_NUM_DESC(oct->conf); 207 iq->sglist = dma_alloc_coherent(iq->dev, sglist_size, 208 &iq->sglist_dma, GFP_KERNEL); 209 if (unlikely(!iq->sglist)) { 210 dev_err(iq->dev, 211 "Failed to allocate DMA memory for IQ-%d SGLIST\n", 212 q_no); 213 goto sglist_alloc_err; 214 } 215 216 /* allocate memory to manage Tx packets pending completion */ 217 buff_info_size = OCTEP_VF_IQ_TXBUFF_INFO_SIZE * iq->max_count; 218 iq->buff_info = vzalloc(buff_info_size); 219 if (!iq->buff_info) { 220 dev_err(iq->dev, 221 "Failed to allocate buff info for IQ-%d\n", q_no); 222 goto buff_info_err; 223 } 224 225 /* Setup sglist addresses in tx_buffer entries */ 226 for (i = 0; i < CFG_GET_IQ_NUM_DESC(oct->conf); i++) { 227 struct octep_vf_tx_buffer *tx_buffer; 228 229 tx_buffer = &iq->buff_info[i]; 230 tx_buffer->sglist = 231 &iq->sglist[i * OCTEP_VF_SGLIST_ENTRIES_PER_PKT]; 232 tx_buffer->sglist_dma = 233 iq->sglist_dma + (i * OCTEP_VF_SGLIST_SIZE_PER_PKT); 234 } 235 236 octep_vf_iq_reset_indices(iq); 237 oct->hw_ops.setup_iq_regs(oct, q_no); 238 239 oct->num_iqs++; 240 return 0; 241 242buff_info_err: 243 dma_free_coherent(iq->dev, sglist_size, iq->sglist, iq->sglist_dma); 244sglist_alloc_err: 245 dma_free_coherent(iq->dev, desc_ring_size, 246 iq->desc_ring, iq->desc_ring_dma); 247desc_dma_alloc_err: 248 vfree(iq); 249 oct->iq[q_no] = NULL; 250iq_alloc_err: 251 return -1; 252} 253 254/** 255 * octep_vf_free_iq() - Free Tx queue resources. 256 * 257 * @iq: Octeon Tx queue data structure. 258 * 259 * Free all the resources allocated for a Tx queue. 260 */ 261static void octep_vf_free_iq(struct octep_vf_iq *iq) 262{ 263 struct octep_vf_device *oct = iq->octep_vf_dev; 264 u64 desc_ring_size, sglist_size; 265 int q_no = iq->q_no; 266 267 desc_ring_size = OCTEP_VF_IQ_DESC_SIZE * CFG_GET_IQ_NUM_DESC(oct->conf); 268 269 vfree(iq->buff_info); 270 271 if (iq->desc_ring) 272 dma_free_coherent(iq->dev, desc_ring_size, 273 iq->desc_ring, iq->desc_ring_dma); 274 275 sglist_size = OCTEP_VF_SGLIST_SIZE_PER_PKT * 276 CFG_GET_IQ_NUM_DESC(oct->conf); 277 if (iq->sglist) 278 dma_free_coherent(iq->dev, sglist_size, 279 iq->sglist, iq->sglist_dma); 280 281 vfree(iq); 282 oct->iq[q_no] = NULL; 283 oct->num_iqs--; 284} 285 286/** 287 * octep_vf_setup_iqs() - setup resources for all Tx queues. 288 * 289 * @oct: Octeon device private data structure. 290 */ 291int octep_vf_setup_iqs(struct octep_vf_device *oct) 292{ 293 int i; 294 295 oct->num_iqs = 0; 296 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { 297 if (octep_vf_setup_iq(oct, i)) { 298 dev_err(&oct->pdev->dev, 299 "Failed to setup IQ(TxQ)-%d.\n", i); 300 goto iq_setup_err; 301 } 302 dev_dbg(&oct->pdev->dev, "Successfully setup IQ(TxQ)-%d.\n", i); 303 } 304 305 return 0; 306 307iq_setup_err: 308 while (i) { 309 i--; 310 octep_vf_free_iq(oct->iq[i]); 311 } 312 return -1; 313} 314 315/** 316 * octep_vf_free_iqs() - Free resources of all Tx queues. 317 * 318 * @oct: Octeon device private data structure. 319 */ 320void octep_vf_free_iqs(struct octep_vf_device *oct) 321{ 322 int i; 323 324 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { 325 octep_vf_free_iq(oct->iq[i]); 326 dev_dbg(&oct->pdev->dev, 327 "Successfully destroyed IQ(TxQ)-%d.\n", i); 328 } 329 oct->num_iqs = 0; 330} 331