1// SPDX-License-Identifier: GPL-2.0 2/* Marvell Octeon EP (EndPoint) VF Ethernet Driver 3 * 4 * Copyright (C) 2020 Marvell. 5 * 6 */ 7 8#include <linux/pci.h> 9#include <linux/etherdevice.h> 10#include <linux/vmalloc.h> 11 12#include "octep_vf_config.h" 13#include "octep_vf_main.h" 14 15static void octep_vf_oq_reset_indices(struct octep_vf_oq *oq) 16{ 17 oq->host_read_idx = 0; 18 oq->host_refill_idx = 0; 19 oq->refill_count = 0; 20 oq->last_pkt_count = 0; 21 oq->pkts_pending = 0; 22} 23 24/** 25 * octep_vf_oq_fill_ring_buffers() - fill initial receive buffers for Rx ring. 26 * 27 * @oq: Octeon Rx queue data structure. 28 * 29 * Return: 0, if successfully filled receive buffers for all descriptors. 30 * -ENOMEM, if failed to allocate a buffer or failed to map for DMA. 31 */ 32static int octep_vf_oq_fill_ring_buffers(struct octep_vf_oq *oq) 33{ 34 struct octep_vf_oq_desc_hw *desc_ring = oq->desc_ring; 35 struct page *page; 36 u32 i; 37 38 for (i = 0; i < oq->max_count; i++) { 39 page = dev_alloc_page(); 40 if (unlikely(!page)) { 41 dev_err(oq->dev, "Rx buffer alloc failed\n"); 42 goto rx_buf_alloc_err; 43 } 44 desc_ring[i].buffer_ptr = dma_map_page(oq->dev, page, 0, 45 PAGE_SIZE, 46 DMA_FROM_DEVICE); 47 if (dma_mapping_error(oq->dev, desc_ring[i].buffer_ptr)) { 48 dev_err(oq->dev, 49 "OQ-%d buffer alloc: DMA mapping error!\n", 50 oq->q_no); 51 goto dma_map_err; 52 } 53 oq->buff_info[i].page = page; 54 } 55 56 return 0; 57 58dma_map_err: 59 put_page(page); 60rx_buf_alloc_err: 61 while (i) { 62 i--; 63 dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, PAGE_SIZE, DMA_FROM_DEVICE); 64 put_page(oq->buff_info[i].page); 65 oq->buff_info[i].page = NULL; 66 } 67 68 return -ENOMEM; 69} 70 71/** 72 * octep_vf_oq_refill() - refill buffers for used Rx ring descriptors. 73 * 74 * @oct: Octeon device private data structure. 75 * @oq: Octeon Rx queue data structure. 76 * 77 * Return: number of descriptors successfully refilled with receive buffers. 78 */ 79static int octep_vf_oq_refill(struct octep_vf_device *oct, struct octep_vf_oq *oq) 80{ 81 struct octep_vf_oq_desc_hw *desc_ring = oq->desc_ring; 82 struct page *page; 83 u32 refill_idx, i; 84 85 refill_idx = oq->host_refill_idx; 86 for (i = 0; i < oq->refill_count; i++) { 87 page = dev_alloc_page(); 88 if (unlikely(!page)) { 89 dev_err(oq->dev, "refill: rx buffer alloc failed\n"); 90 oq->stats.alloc_failures++; 91 break; 92 } 93 94 desc_ring[refill_idx].buffer_ptr = dma_map_page(oq->dev, page, 0, 95 PAGE_SIZE, DMA_FROM_DEVICE); 96 if (dma_mapping_error(oq->dev, desc_ring[refill_idx].buffer_ptr)) { 97 dev_err(oq->dev, 98 "OQ-%d buffer refill: DMA mapping error!\n", 99 oq->q_no); 100 put_page(page); 101 oq->stats.alloc_failures++; 102 break; 103 } 104 oq->buff_info[refill_idx].page = page; 105 refill_idx++; 106 if (refill_idx == oq->max_count) 107 refill_idx = 0; 108 } 109 oq->host_refill_idx = refill_idx; 110 oq->refill_count -= i; 111 112 return i; 113} 114 115/** 116 * octep_vf_setup_oq() - Setup a Rx queue. 117 * 118 * @oct: Octeon device private data structure. 119 * @q_no: Rx queue number to be setup. 120 * 121 * Allocate resources for a Rx queue. 122 */ 123static int octep_vf_setup_oq(struct octep_vf_device *oct, int q_no) 124{ 125 struct octep_vf_oq *oq; 126 u32 desc_ring_size; 127 128 oq = vzalloc(sizeof(*oq)); 129 if (!oq) 130 goto create_oq_fail; 131 oct->oq[q_no] = oq; 132 133 oq->octep_vf_dev = oct; 134 oq->netdev = oct->netdev; 135 oq->dev = &oct->pdev->dev; 136 oq->q_no = q_no; 137 oq->max_count = CFG_GET_OQ_NUM_DESC(oct->conf); 138 oq->ring_size_mask = oq->max_count - 1; 139 oq->buffer_size = CFG_GET_OQ_BUF_SIZE(oct->conf); 140 oq->max_single_buffer_size = oq->buffer_size - OCTEP_VF_OQ_RESP_HW_SIZE; 141 142 /* When the hardware/firmware supports additional capabilities, 143 * additional header is filled-in by Octeon after length field in 144 * Rx packets. this header contains additional packet information. 145 */ 146 if (oct->fw_info.rx_ol_flags) 147 oq->max_single_buffer_size -= OCTEP_VF_OQ_RESP_HW_EXT_SIZE; 148 149 oq->refill_threshold = CFG_GET_OQ_REFILL_THRESHOLD(oct->conf); 150 151 desc_ring_size = oq->max_count * OCTEP_VF_OQ_DESC_SIZE; 152 oq->desc_ring = dma_alloc_coherent(oq->dev, desc_ring_size, 153 &oq->desc_ring_dma, GFP_KERNEL); 154 155 if (unlikely(!oq->desc_ring)) { 156 dev_err(oq->dev, 157 "Failed to allocate DMA memory for OQ-%d !!\n", q_no); 158 goto desc_dma_alloc_err; 159 } 160 161 oq->buff_info = vzalloc(oq->max_count * OCTEP_VF_OQ_RECVBUF_SIZE); 162 163 if (unlikely(!oq->buff_info)) { 164 dev_err(&oct->pdev->dev, 165 "Failed to allocate buffer info for OQ-%d\n", q_no); 166 goto buf_list_err; 167 } 168 169 if (octep_vf_oq_fill_ring_buffers(oq)) 170 goto oq_fill_buff_err; 171 172 octep_vf_oq_reset_indices(oq); 173 oct->hw_ops.setup_oq_regs(oct, q_no); 174 oct->num_oqs++; 175 176 return 0; 177 178oq_fill_buff_err: 179 vfree(oq->buff_info); 180 oq->buff_info = NULL; 181buf_list_err: 182 dma_free_coherent(oq->dev, desc_ring_size, 183 oq->desc_ring, oq->desc_ring_dma); 184 oq->desc_ring = NULL; 185desc_dma_alloc_err: 186 vfree(oq); 187 oct->oq[q_no] = NULL; 188create_oq_fail: 189 return -ENOMEM; 190} 191 192/** 193 * octep_vf_oq_free_ring_buffers() - Free ring buffers. 194 * 195 * @oq: Octeon Rx queue data structure. 196 * 197 * Free receive buffers in unused Rx queue descriptors. 198 */ 199static void octep_vf_oq_free_ring_buffers(struct octep_vf_oq *oq) 200{ 201 struct octep_vf_oq_desc_hw *desc_ring = oq->desc_ring; 202 int i; 203 204 if (!oq->desc_ring || !oq->buff_info) 205 return; 206 207 for (i = 0; i < oq->max_count; i++) { 208 if (oq->buff_info[i].page) { 209 dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, 210 PAGE_SIZE, DMA_FROM_DEVICE); 211 put_page(oq->buff_info[i].page); 212 oq->buff_info[i].page = NULL; 213 desc_ring[i].buffer_ptr = 0; 214 } 215 } 216 octep_vf_oq_reset_indices(oq); 217} 218 219/** 220 * octep_vf_free_oq() - Free Rx queue resources. 221 * 222 * @oq: Octeon Rx queue data structure. 223 * 224 * Free all resources of a Rx queue. 225 */ 226static int octep_vf_free_oq(struct octep_vf_oq *oq) 227{ 228 struct octep_vf_device *oct = oq->octep_vf_dev; 229 int q_no = oq->q_no; 230 231 octep_vf_oq_free_ring_buffers(oq); 232 233 vfree(oq->buff_info); 234 235 if (oq->desc_ring) 236 dma_free_coherent(oq->dev, 237 oq->max_count * OCTEP_VF_OQ_DESC_SIZE, 238 oq->desc_ring, oq->desc_ring_dma); 239 240 vfree(oq); 241 oct->oq[q_no] = NULL; 242 oct->num_oqs--; 243 return 0; 244} 245 246/** 247 * octep_vf_setup_oqs() - setup resources for all Rx queues. 248 * 249 * @oct: Octeon device private data structure. 250 */ 251int octep_vf_setup_oqs(struct octep_vf_device *oct) 252{ 253 int i, retval = 0; 254 255 oct->num_oqs = 0; 256 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { 257 retval = octep_vf_setup_oq(oct, i); 258 if (retval) { 259 dev_err(&oct->pdev->dev, 260 "Failed to setup OQ(RxQ)-%d.\n", i); 261 goto oq_setup_err; 262 } 263 dev_dbg(&oct->pdev->dev, "Successfully setup OQ(RxQ)-%d.\n", i); 264 } 265 266 return 0; 267 268oq_setup_err: 269 while (i) { 270 i--; 271 octep_vf_free_oq(oct->oq[i]); 272 } 273 return retval; 274} 275 276/** 277 * octep_vf_oq_dbell_init() - Initialize Rx queue doorbell. 278 * 279 * @oct: Octeon device private data structure. 280 * 281 * Write number of descriptors to Rx queue doorbell register. 282 */ 283void octep_vf_oq_dbell_init(struct octep_vf_device *oct) 284{ 285 int i; 286 287 for (i = 0; i < oct->num_oqs; i++) 288 writel(oct->oq[i]->max_count, oct->oq[i]->pkts_credit_reg); 289} 290 291/** 292 * octep_vf_free_oqs() - Free resources of all Rx queues. 293 * 294 * @oct: Octeon device private data structure. 295 */ 296void octep_vf_free_oqs(struct octep_vf_device *oct) 297{ 298 int i; 299 300 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { 301 if (!oct->oq[i]) 302 continue; 303 octep_vf_free_oq(oct->oq[i]); 304 dev_dbg(&oct->pdev->dev, 305 "Successfully freed OQ(RxQ)-%d.\n", i); 306 } 307} 308 309/** 310 * octep_vf_oq_check_hw_for_pkts() - Check for new Rx packets. 311 * 312 * @oct: Octeon device private data structure. 313 * @oq: Octeon Rx queue data structure. 314 * 315 * Return: packets received after previous check. 316 */ 317static int octep_vf_oq_check_hw_for_pkts(struct octep_vf_device *oct, 318 struct octep_vf_oq *oq) 319{ 320 u32 pkt_count, new_pkts; 321 322 pkt_count = readl(oq->pkts_sent_reg); 323 new_pkts = pkt_count - oq->last_pkt_count; 324 325 /* Clear the hardware packets counter register if the rx queue is 326 * being processed continuously with-in a single interrupt and 327 * reached half its max value. 328 * this counter is not cleared every time read, to save write cycles. 329 */ 330 if (unlikely(pkt_count > 0xF0000000U)) { 331 writel(pkt_count, oq->pkts_sent_reg); 332 pkt_count = readl(oq->pkts_sent_reg); 333 new_pkts += pkt_count; 334 } 335 oq->last_pkt_count = pkt_count; 336 oq->pkts_pending += new_pkts; 337 return new_pkts; 338} 339 340/** 341 * __octep_vf_oq_process_rx() - Process hardware Rx queue and push to stack. 342 * 343 * @oct: Octeon device private data structure. 344 * @oq: Octeon Rx queue data structure. 345 * @pkts_to_process: number of packets to be processed. 346 * 347 * Process the new packets in Rx queue. 348 * Packets larger than single Rx buffer arrive in consecutive descriptors. 349 * But, count returned by the API only accounts full packets, not fragments. 350 * 351 * Return: number of packets processed and pushed to stack. 352 */ 353static int __octep_vf_oq_process_rx(struct octep_vf_device *oct, 354 struct octep_vf_oq *oq, u16 pkts_to_process) 355{ 356 struct octep_vf_oq_resp_hw_ext *resp_hw_ext = NULL; 357 netdev_features_t feat = oq->netdev->features; 358 struct octep_vf_rx_buffer *buff_info; 359 struct octep_vf_oq_resp_hw *resp_hw; 360 u32 pkt, rx_bytes, desc_used; 361 u16 data_offset, rx_ol_flags; 362 struct sk_buff *skb; 363 u32 read_idx; 364 365 read_idx = oq->host_read_idx; 366 rx_bytes = 0; 367 desc_used = 0; 368 for (pkt = 0; pkt < pkts_to_process; pkt++) { 369 buff_info = (struct octep_vf_rx_buffer *)&oq->buff_info[read_idx]; 370 dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr, 371 PAGE_SIZE, DMA_FROM_DEVICE); 372 resp_hw = page_address(buff_info->page); 373 buff_info->page = NULL; 374 375 /* Swap the length field that is in Big-Endian to CPU */ 376 buff_info->len = be64_to_cpu(resp_hw->length); 377 if (oct->fw_info.rx_ol_flags) { 378 /* Extended response header is immediately after 379 * response header (resp_hw) 380 */ 381 resp_hw_ext = (struct octep_vf_oq_resp_hw_ext *) 382 (resp_hw + 1); 383 buff_info->len -= OCTEP_VF_OQ_RESP_HW_EXT_SIZE; 384 /* Packet Data is immediately after 385 * extended response header. 386 */ 387 data_offset = OCTEP_VF_OQ_RESP_HW_SIZE + 388 OCTEP_VF_OQ_RESP_HW_EXT_SIZE; 389 rx_ol_flags = resp_hw_ext->rx_ol_flags; 390 } else { 391 /* Data is immediately after 392 * Hardware Rx response header. 393 */ 394 data_offset = OCTEP_VF_OQ_RESP_HW_SIZE; 395 rx_ol_flags = 0; 396 } 397 rx_bytes += buff_info->len; 398 399 if (buff_info->len <= oq->max_single_buffer_size) { 400 skb = napi_build_skb((void *)resp_hw, PAGE_SIZE); 401 skb_reserve(skb, data_offset); 402 skb_put(skb, buff_info->len); 403 read_idx++; 404 desc_used++; 405 if (read_idx == oq->max_count) 406 read_idx = 0; 407 } else { 408 struct skb_shared_info *shinfo; 409 u16 data_len; 410 411 skb = napi_build_skb((void *)resp_hw, PAGE_SIZE); 412 skb_reserve(skb, data_offset); 413 /* Head fragment includes response header(s); 414 * subsequent fragments contains only data. 415 */ 416 skb_put(skb, oq->max_single_buffer_size); 417 read_idx++; 418 desc_used++; 419 if (read_idx == oq->max_count) 420 read_idx = 0; 421 422 shinfo = skb_shinfo(skb); 423 data_len = buff_info->len - oq->max_single_buffer_size; 424 while (data_len) { 425 dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr, 426 PAGE_SIZE, DMA_FROM_DEVICE); 427 buff_info = (struct octep_vf_rx_buffer *) 428 &oq->buff_info[read_idx]; 429 if (data_len < oq->buffer_size) { 430 buff_info->len = data_len; 431 data_len = 0; 432 } else { 433 buff_info->len = oq->buffer_size; 434 data_len -= oq->buffer_size; 435 } 436 437 skb_add_rx_frag(skb, shinfo->nr_frags, 438 buff_info->page, 0, 439 buff_info->len, 440 buff_info->len); 441 buff_info->page = NULL; 442 read_idx++; 443 desc_used++; 444 if (read_idx == oq->max_count) 445 read_idx = 0; 446 } 447 } 448 449 skb->dev = oq->netdev; 450 skb->protocol = eth_type_trans(skb, skb->dev); 451 if (feat & NETIF_F_RXCSUM && 452 OCTEP_VF_RX_CSUM_VERIFIED(rx_ol_flags)) 453 skb->ip_summed = CHECKSUM_UNNECESSARY; 454 else 455 skb->ip_summed = CHECKSUM_NONE; 456 napi_gro_receive(oq->napi, skb); 457 } 458 459 oq->host_read_idx = read_idx; 460 oq->refill_count += desc_used; 461 oq->stats.packets += pkt; 462 oq->stats.bytes += rx_bytes; 463 464 return pkt; 465} 466 467/** 468 * octep_vf_oq_process_rx() - Process Rx queue. 469 * 470 * @oq: Octeon Rx queue data structure. 471 * @budget: max number of packets can be processed in one invocation. 472 * 473 * Check for newly received packets and process them. 474 * Keeps checking for new packets until budget is used or no new packets seen. 475 * 476 * Return: number of packets processed. 477 */ 478int octep_vf_oq_process_rx(struct octep_vf_oq *oq, int budget) 479{ 480 u32 pkts_available, pkts_processed, total_pkts_processed; 481 struct octep_vf_device *oct = oq->octep_vf_dev; 482 483 pkts_available = 0; 484 pkts_processed = 0; 485 total_pkts_processed = 0; 486 while (total_pkts_processed < budget) { 487 /* update pending count only when current one exhausted */ 488 if (oq->pkts_pending == 0) 489 octep_vf_oq_check_hw_for_pkts(oct, oq); 490 pkts_available = min(budget - total_pkts_processed, 491 oq->pkts_pending); 492 if (!pkts_available) 493 break; 494 495 pkts_processed = __octep_vf_oq_process_rx(oct, oq, 496 pkts_available); 497 oq->pkts_pending -= pkts_processed; 498 total_pkts_processed += pkts_processed; 499 } 500 501 if (oq->refill_count >= oq->refill_threshold) { 502 u32 desc_refilled = octep_vf_oq_refill(oct, oq); 503 504 /* flush pending writes before updating credits */ 505 smp_wmb(); 506 writel(desc_refilled, oq->pkts_credit_reg); 507 } 508 509 return total_pkts_processed; 510} 511