1295016Sjkim// SPDX-License-Identifier: GPL-2.0 2110010Smarkm/* Marvell Octeon EP (EndPoint) Ethernet Driver 3110010Smarkm * 4142429Snectar * Copyright (C) 2020 Marvell. 5110010Smarkm * 6110010Smarkm */ 7110010Smarkm 8110010Smarkm#include <linux/pci.h> 9110010Smarkm#include <linux/etherdevice.h> 10110010Smarkm#include <linux/vmalloc.h> 11110010Smarkm 12110010Smarkm#include "octep_config.h" 13110010Smarkm#include "octep_main.h" 14110010Smarkm 15110010Smarkm/* Reset various index of Tx queue data structure. */ 16110010Smarkmstatic void octep_iq_reset_indices(struct octep_iq *iq) 17110010Smarkm{ 18110010Smarkm iq->fill_cnt = 0; 19110010Smarkm iq->host_write_index = 0; 20215698Ssimon iq->octep_read_index = 0; 21215698Ssimon iq->flush_index = 0; 22215698Ssimon iq->pkts_processed = 0; 23215698Ssimon iq->pkt_in_done = 0; 24215698Ssimon} 25110010Smarkm 26110010Smarkm/** 27110010Smarkm * octep_iq_process_completions() - Process Tx queue completions. 28110010Smarkm * 29110010Smarkm * @iq: Octeon Tx queue data structure. 30110010Smarkm * @budget: max number of completions to be processed in one invocation. 31110010Smarkm */ 32110010Smarkmint octep_iq_process_completions(struct octep_iq *iq, u16 budget) 33110010Smarkm{ 34110010Smarkm u32 compl_pkts, compl_bytes, compl_sg; 35110010Smarkm struct octep_device *oct = iq->octep_dev; 36110010Smarkm struct octep_tx_buffer *tx_buffer; 37110010Smarkm struct skb_shared_info *shinfo; 38110010Smarkm u32 fi = iq->flush_index; 39110010Smarkm struct sk_buff *skb; 40110010Smarkm u8 frags, i; 41276864Sjkim 42276864Sjkim compl_pkts = 0; 43110010Smarkm compl_sg = 0; 44110010Smarkm compl_bytes = 0; 45215698Ssimon iq->octep_read_index = oct->hw_ops.update_iq_read_idx(iq); 46215698Ssimon 47215698Ssimon while (likely(budget && (fi != iq->octep_read_index))) { 48215698Ssimon tx_buffer = iq->buff_info + fi; 49142429Snectar skb = tx_buffer->skb; 50215698Ssimon 51142429Snectar fi++; 52142429Snectar if (unlikely(fi == iq->max_count)) 53276864Sjkim fi = 0; 54276864Sjkim compl_bytes += skb->len; 55276864Sjkim compl_pkts++; 56110010Smarkm budget--; 57276864Sjkim 58276864Sjkim if (!tx_buffer->gather) { 59276864Sjkim dma_unmap_single(iq->dev, tx_buffer->dma, 60276864Sjkim tx_buffer->skb->len, DMA_TO_DEVICE); 61276864Sjkim dev_kfree_skb_any(skb); 62276864Sjkim continue; 63215698Ssimon } 64276864Sjkim 65276864Sjkim /* Scatter/Gather */ 66276864Sjkim shinfo = skb_shinfo(skb); 67276864Sjkim frags = shinfo->nr_frags; 68276864Sjkim compl_sg++; 69215698Ssimon 70276864Sjkim dma_unmap_single(iq->dev, tx_buffer->sglist[0].dma_ptr[0], 71110010Smarkm tx_buffer->sglist[0].len[3], DMA_TO_DEVICE); 72110010Smarkm 73110010Smarkm i = 1; /* entry 0 is main skb, unmapped above */ 74110010Smarkm while (frags--) { 75110010Smarkm dma_unmap_page(iq->dev, tx_buffer->sglist[i >> 2].dma_ptr[i & 3], 76110010Smarkm tx_buffer->sglist[i >> 2].len[3 - (i & 3)], DMA_TO_DEVICE); 77110010Smarkm i++; 78110010Smarkm } 79110010Smarkm 80110010Smarkm dev_kfree_skb_any(skb); 81110010Smarkm } 82110010Smarkm 83110010Smarkm iq->pkts_processed += compl_pkts; 84110010Smarkm iq->stats.instr_completed += compl_pkts; 85110010Smarkm iq->stats.bytes_sent += compl_bytes; 86110010Smarkm iq->stats.sgentry_sent += compl_sg; 87110010Smarkm iq->flush_index = fi; 88110010Smarkm 89110010Smarkm netdev_tx_completed_queue(iq->netdev_q, compl_pkts, compl_bytes); 90110010Smarkm 91110010Smarkm if (unlikely(__netif_subqueue_stopped(iq->netdev, iq->q_no)) && 92110010Smarkm (IQ_INSTR_SPACE(iq) > 93110010Smarkm OCTEP_WAKE_QUEUE_THRESHOLD)) 94110010Smarkm netif_wake_subqueue(iq->netdev, iq->q_no); 95110010Smarkm return !budget; 96110010Smarkm} 97110010Smarkm 98110010Smarkm/** 99110010Smarkm * octep_iq_free_pending() - Free Tx buffers for pending completions. 100110010Smarkm * 101110010Smarkm * @iq: Octeon Tx queue data structure. 102110010Smarkm */ 103110010Smarkmstatic void octep_iq_free_pending(struct octep_iq *iq) 104110010Smarkm{ 105110010Smarkm struct octep_tx_buffer *tx_buffer; 106110010Smarkm struct skb_shared_info *shinfo; 107110010Smarkm u32 fi = iq->flush_index; 108110010Smarkm struct sk_buff *skb; 109110010Smarkm u8 frags, i; 110110010Smarkm 111110010Smarkm while (fi != iq->host_write_index) { 112110010Smarkm tx_buffer = iq->buff_info + fi; 113110010Smarkm skb = tx_buffer->skb; 114110010Smarkm 115110010Smarkm fi++; 116110010Smarkm if (unlikely(fi == iq->max_count)) 117110010Smarkm fi = 0; 118110010Smarkm 119110010Smarkm if (!tx_buffer->gather) { 120110010Smarkm dma_unmap_single(iq->dev, tx_buffer->dma, 121110010Smarkm tx_buffer->skb->len, DMA_TO_DEVICE); 122110010Smarkm dev_kfree_skb_any(skb); 123110010Smarkm continue; 124110010Smarkm } 125110010Smarkm 126110010Smarkm /* Scatter/Gather */ 127110010Smarkm shinfo = skb_shinfo(skb); 128110010Smarkm frags = shinfo->nr_frags; 129110010Smarkm 130110010Smarkm dma_unmap_single(iq->dev, 131110010Smarkm tx_buffer->sglist[0].dma_ptr[0], 132110010Smarkm tx_buffer->sglist[0].len[3], 133142429Snectar DMA_TO_DEVICE); 134110010Smarkm 135110655Snectar i = 1; /* entry 0 is main skb, unmapped above */ 136296317Sdelphij while (frags--) { 137215698Ssimon dma_unmap_page(iq->dev, tx_buffer->sglist[i >> 2].dma_ptr[i & 3], 138215698Ssimon tx_buffer->sglist[i >> 2].len[3 - (i & 3)], DMA_TO_DEVICE); 139215698Ssimon i++; 140215698Ssimon } 141110010Smarkm 142110010Smarkm dev_kfree_skb_any(skb); 143110010Smarkm } 144110010Smarkm 145110010Smarkm iq->flush_index = fi; 146110010Smarkm netdev_tx_reset_queue(netdev_get_tx_queue(iq->netdev, iq->q_no)); 147110010Smarkm} 148110010Smarkm 149110010Smarkm/** 150110010Smarkm * octep_clean_iqs() - Clean Tx queues to shutdown the device. 151110010Smarkm * 152110010Smarkm * @oct: Octeon device private data structure. 153110010Smarkm * 154110010Smarkm * Free the buffers in Tx queue descriptors pending completion and 155110010Smarkm * reset queue indices 156110010Smarkm */ 157142429Snectarvoid octep_clean_iqs(struct octep_device *oct) 158110010Smarkm{ 159110010Smarkm int i; 160110010Smarkm 161142429Snectar for (i = 0; i < oct->num_iqs; i++) { 162110010Smarkm octep_iq_free_pending(oct->iq[i]); 163110010Smarkm octep_iq_reset_indices(oct->iq[i]); 164142429Snectar } 165110010Smarkm} 166110010Smarkm 167110010Smarkm/** 168110010Smarkm * octep_setup_iq() - Setup a Tx queue. 169110010Smarkm * 170110010Smarkm * @oct: Octeon device private data structure. 171110010Smarkm * @q_no: Tx queue number to be setup. 172110010Smarkm * 173110010Smarkm * Allocate resources for a Tx queue. 174110010Smarkm */ 175215698Ssimonstatic int octep_setup_iq(struct octep_device *oct, int q_no) 176110010Smarkm{ 177142429Snectar u32 desc_ring_size, buff_info_size, sglist_size; 178110010Smarkm struct octep_iq *iq; 179110010Smarkm int i; 180110010Smarkm 181215698Ssimon iq = vzalloc(sizeof(*iq)); 182110010Smarkm if (!iq) 183110010Smarkm goto iq_alloc_err; 184110010Smarkm oct->iq[q_no] = iq; 185110010Smarkm 186110010Smarkm iq->octep_dev = oct; 187110010Smarkm iq->netdev = oct->netdev; 188215698Ssimon iq->dev = &oct->pdev->dev; 189215698Ssimon iq->q_no = q_no; 190110010Smarkm iq->max_count = CFG_GET_IQ_NUM_DESC(oct->conf); 191142429Snectar iq->ring_size_mask = iq->max_count - 1; 192110010Smarkm iq->fill_threshold = CFG_GET_IQ_DB_MIN(oct->conf); 193110010Smarkm iq->netdev_q = netdev_get_tx_queue(iq->netdev, q_no); 194110010Smarkm 195110010Smarkm /* Allocate memory for hardware queue descriptors */ 196110010Smarkm desc_ring_size = OCTEP_IQ_DESC_SIZE * CFG_GET_IQ_NUM_DESC(oct->conf); 197110010Smarkm iq->desc_ring = dma_alloc_coherent(iq->dev, desc_ring_size, 198110010Smarkm &iq->desc_ring_dma, GFP_KERNEL); 199110010Smarkm if (unlikely(!iq->desc_ring)) { 200 dev_err(iq->dev, 201 "Failed to allocate DMA memory for IQ-%d\n", q_no); 202 goto desc_dma_alloc_err; 203 } 204 205 /* Allocate memory for hardware SGLIST descriptors */ 206 sglist_size = OCTEP_SGLIST_SIZE_PER_PKT * 207 CFG_GET_IQ_NUM_DESC(oct->conf); 208 iq->sglist = dma_alloc_coherent(iq->dev, sglist_size, 209 &iq->sglist_dma, GFP_KERNEL); 210 if (unlikely(!iq->sglist)) { 211 dev_err(iq->dev, 212 "Failed to allocate DMA memory for IQ-%d SGLIST\n", 213 q_no); 214 goto sglist_alloc_err; 215 } 216 217 /* allocate memory to manage Tx packets pending completion */ 218 buff_info_size = OCTEP_IQ_TXBUFF_INFO_SIZE * iq->max_count; 219 iq->buff_info = vzalloc(buff_info_size); 220 if (!iq->buff_info) { 221 dev_err(iq->dev, 222 "Failed to allocate buff info for IQ-%d\n", q_no); 223 goto buff_info_err; 224 } 225 226 /* Setup sglist addresses in tx_buffer entries */ 227 for (i = 0; i < CFG_GET_IQ_NUM_DESC(oct->conf); i++) { 228 struct octep_tx_buffer *tx_buffer; 229 230 tx_buffer = &iq->buff_info[i]; 231 tx_buffer->sglist = 232 &iq->sglist[i * OCTEP_SGLIST_ENTRIES_PER_PKT]; 233 tx_buffer->sglist_dma = 234 iq->sglist_dma + (i * OCTEP_SGLIST_SIZE_PER_PKT); 235 } 236 237 octep_iq_reset_indices(iq); 238 oct->hw_ops.setup_iq_regs(oct, q_no); 239 240 oct->num_iqs++; 241 return 0; 242 243buff_info_err: 244 dma_free_coherent(iq->dev, sglist_size, iq->sglist, iq->sglist_dma); 245sglist_alloc_err: 246 dma_free_coherent(iq->dev, desc_ring_size, 247 iq->desc_ring, iq->desc_ring_dma); 248desc_dma_alloc_err: 249 vfree(iq); 250 oct->iq[q_no] = NULL; 251iq_alloc_err: 252 return -1; 253} 254 255/** 256 * octep_free_iq() - Free Tx queue resources. 257 * 258 * @iq: Octeon Tx queue data structure. 259 * 260 * Free all the resources allocated for a Tx queue. 261 */ 262static void octep_free_iq(struct octep_iq *iq) 263{ 264 struct octep_device *oct = iq->octep_dev; 265 u64 desc_ring_size, sglist_size; 266 int q_no = iq->q_no; 267 268 desc_ring_size = OCTEP_IQ_DESC_SIZE * CFG_GET_IQ_NUM_DESC(oct->conf); 269 270 vfree(iq->buff_info); 271 272 if (iq->desc_ring) 273 dma_free_coherent(iq->dev, desc_ring_size, 274 iq->desc_ring, iq->desc_ring_dma); 275 276 sglist_size = OCTEP_SGLIST_SIZE_PER_PKT * 277 CFG_GET_IQ_NUM_DESC(oct->conf); 278 if (iq->sglist) 279 dma_free_coherent(iq->dev, sglist_size, 280 iq->sglist, iq->sglist_dma); 281 282 vfree(iq); 283 oct->iq[q_no] = NULL; 284 oct->num_iqs--; 285} 286 287/** 288 * octep_setup_iqs() - setup resources for all Tx queues. 289 * 290 * @oct: Octeon device private data structure. 291 */ 292int octep_setup_iqs(struct octep_device *oct) 293{ 294 int i; 295 296 oct->num_iqs = 0; 297 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { 298 if (octep_setup_iq(oct, i)) { 299 dev_err(&oct->pdev->dev, 300 "Failed to setup IQ(TxQ)-%d.\n", i); 301 goto iq_setup_err; 302 } 303 dev_dbg(&oct->pdev->dev, "Successfully setup IQ(TxQ)-%d.\n", i); 304 } 305 306 return 0; 307 308iq_setup_err: 309 while (i) { 310 i--; 311 octep_free_iq(oct->iq[i]); 312 } 313 return -1; 314} 315 316/** 317 * octep_free_iqs() - Free resources of all Tx queues. 318 * 319 * @oct: Octeon device private data structure. 320 */ 321void octep_free_iqs(struct octep_device *oct) 322{ 323 int i; 324 325 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { 326 octep_free_iq(oct->iq[i]); 327 dev_dbg(&oct->pdev->dev, 328 "Successfully destroyed IQ(TxQ)-%d.\n", i); 329 } 330 oct->num_iqs = 0; 331} 332