1// SPDX-License-Identifier: GPL-2.0 2/* Marvell Octeon EP (EndPoint) VF Ethernet Driver 3 * 4 * Copyright (C) 2020 Marvell. 5 * 6 */ 7 8#include <linux/types.h> 9#include <linux/module.h> 10#include <linux/pci.h> 11#include <linux/aer.h> 12#include <linux/netdevice.h> 13#include <linux/etherdevice.h> 14#include <linux/rtnetlink.h> 15#include <linux/vmalloc.h> 16#include <net/netdev_queues.h> 17 18#include "octep_vf_config.h" 19#include "octep_vf_main.h" 20 21struct workqueue_struct *octep_vf_wq; 22 23/* Supported Devices */ 24static const struct pci_device_id octep_vf_pci_id_tbl[] = { 25 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN93_VF)}, 26 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF95N_VF)}, 27 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN98_VF)}, 28 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN10KA_VF)}, 29 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF10KA_VF)}, 30 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF10KB_VF)}, 31 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN10KB_VF)}, 32 {0, }, 33}; 34MODULE_DEVICE_TABLE(pci, octep_vf_pci_id_tbl); 35 36MODULE_AUTHOR("Veerasenareddy Burru <vburru@marvell.com>"); 37MODULE_DESCRIPTION(OCTEP_VF_DRV_STRING); 38MODULE_LICENSE("GPL"); 39 40/** 41 * octep_vf_alloc_ioq_vectors() - Allocate Tx/Rx Queue interrupt info. 42 * 43 * @oct: Octeon device private data structure. 44 * 45 * Allocate resources to hold per Tx/Rx queue interrupt info. 46 * This is the information passed to interrupt handler, from which napi poll 47 * is scheduled and includes quick access to private data of Tx/Rx queue 48 * corresponding to the interrupt being handled. 49 * 50 * Return: 0, on successful allocation of resources for all queue interrupts. 51 * -1, if failed to allocate any resource. 52 */ 53static int octep_vf_alloc_ioq_vectors(struct octep_vf_device *oct) 54{ 55 struct octep_vf_ioq_vector *ioq_vector; 56 int i; 57 58 for (i = 0; i < oct->num_oqs; i++) { 59 oct->ioq_vector[i] = vzalloc(sizeof(*oct->ioq_vector[i])); 60 if (!oct->ioq_vector[i]) 61 goto free_ioq_vector; 62 63 ioq_vector = oct->ioq_vector[i]; 64 ioq_vector->iq = oct->iq[i]; 65 ioq_vector->oq = oct->oq[i]; 66 ioq_vector->octep_vf_dev = oct; 67 } 68 69 dev_info(&oct->pdev->dev, "Allocated %d IOQ vectors\n", oct->num_oqs); 70 return 0; 71 72free_ioq_vector: 73 while (i) { 74 i--; 75 vfree(oct->ioq_vector[i]); 76 oct->ioq_vector[i] = NULL; 77 } 78 return -1; 79} 80 81/** 82 * octep_vf_free_ioq_vectors() - Free Tx/Rx Queue interrupt vector info. 83 * 84 * @oct: Octeon device private data structure. 85 */ 86static void octep_vf_free_ioq_vectors(struct octep_vf_device *oct) 87{ 88 int i; 89 90 for (i = 0; i < oct->num_oqs; i++) { 91 if (oct->ioq_vector[i]) { 92 vfree(oct->ioq_vector[i]); 93 oct->ioq_vector[i] = NULL; 94 } 95 } 96 netdev_info(oct->netdev, "Freed IOQ Vectors\n"); 97} 98 99/** 100 * octep_vf_enable_msix_range() - enable MSI-x interrupts. 101 * 102 * @oct: Octeon device private data structure. 103 * 104 * Allocate and enable all MSI-x interrupts (queue and non-queue interrupts) 105 * for the Octeon device. 106 * 107 * Return: 0, on successfully enabling all MSI-x interrupts. 108 * -1, if failed to enable any MSI-x interrupt. 109 */ 110static int octep_vf_enable_msix_range(struct octep_vf_device *oct) 111{ 112 int num_msix, msix_allocated; 113 int i; 114 115 /* Generic interrupts apart from input/output queues */ 116 //num_msix = oct->num_oqs + CFG_GET_NON_IOQ_MSIX(oct->conf); 117 num_msix = oct->num_oqs; 118 oct->msix_entries = kcalloc(num_msix, sizeof(struct msix_entry), GFP_KERNEL); 119 if (!oct->msix_entries) 120 goto msix_alloc_err; 121 122 for (i = 0; i < num_msix; i++) 123 oct->msix_entries[i].entry = i; 124 125 msix_allocated = pci_enable_msix_range(oct->pdev, oct->msix_entries, 126 num_msix, num_msix); 127 if (msix_allocated != num_msix) { 128 dev_err(&oct->pdev->dev, 129 "Failed to enable %d msix irqs; got only %d\n", 130 num_msix, msix_allocated); 131 goto enable_msix_err; 132 } 133 oct->num_irqs = msix_allocated; 134 dev_info(&oct->pdev->dev, "MSI-X enabled successfully\n"); 135 136 return 0; 137 138enable_msix_err: 139 if (msix_allocated > 0) 140 pci_disable_msix(oct->pdev); 141 kfree(oct->msix_entries); 142 oct->msix_entries = NULL; 143msix_alloc_err: 144 return -1; 145} 146 147/** 148 * octep_vf_disable_msix() - disable MSI-x interrupts. 149 * 150 * @oct: Octeon device private data structure. 151 * 152 * Disable MSI-x on the Octeon device. 153 */ 154static void octep_vf_disable_msix(struct octep_vf_device *oct) 155{ 156 pci_disable_msix(oct->pdev); 157 kfree(oct->msix_entries); 158 oct->msix_entries = NULL; 159 dev_info(&oct->pdev->dev, "Disabled MSI-X\n"); 160} 161 162/** 163 * octep_vf_ioq_intr_handler() - handler for all Tx/Rx queue interrupts. 164 * 165 * @irq: Interrupt number. 166 * @data: interrupt data contains pointers to Tx/Rx queue private data 167 * and correspong NAPI context. 168 * 169 * this is common handler for all non-queue (generic) interrupts. 170 */ 171static irqreturn_t octep_vf_ioq_intr_handler(int irq, void *data) 172{ 173 struct octep_vf_ioq_vector *ioq_vector = data; 174 struct octep_vf_device *oct = ioq_vector->octep_vf_dev; 175 176 return oct->hw_ops.ioq_intr_handler(ioq_vector); 177} 178 179/** 180 * octep_vf_request_irqs() - Register interrupt handlers. 181 * 182 * @oct: Octeon device private data structure. 183 * 184 * Register handlers for all queue and non-queue interrupts. 185 * 186 * Return: 0, on successful registration of all interrupt handlers. 187 * -1, on any error. 188 */ 189static int octep_vf_request_irqs(struct octep_vf_device *oct) 190{ 191 struct net_device *netdev = oct->netdev; 192 struct octep_vf_ioq_vector *ioq_vector; 193 struct msix_entry *msix_entry; 194 int ret, i; 195 196 /* Request IRQs for Tx/Rx queues */ 197 for (i = 0; i < oct->num_oqs; i++) { 198 ioq_vector = oct->ioq_vector[i]; 199 msix_entry = &oct->msix_entries[i]; 200 201 snprintf(ioq_vector->name, sizeof(ioq_vector->name), 202 "%s-q%d", netdev->name, i); 203 ret = request_irq(msix_entry->vector, 204 octep_vf_ioq_intr_handler, 0, 205 ioq_vector->name, ioq_vector); 206 if (ret) { 207 netdev_err(netdev, 208 "request_irq failed for Q-%d; err=%d", 209 i, ret); 210 goto ioq_irq_err; 211 } 212 213 cpumask_set_cpu(i % num_online_cpus(), 214 &ioq_vector->affinity_mask); 215 irq_set_affinity_hint(msix_entry->vector, 216 &ioq_vector->affinity_mask); 217 } 218 219 return 0; 220ioq_irq_err: 221 while (i) { 222 --i; 223 free_irq(oct->msix_entries[i].vector, oct); 224 } 225 return -1; 226} 227 228/** 229 * octep_vf_free_irqs() - free all registered interrupts. 230 * 231 * @oct: Octeon device private data structure. 232 * 233 * Free all queue and non-queue interrupts of the Octeon device. 234 */ 235static void octep_vf_free_irqs(struct octep_vf_device *oct) 236{ 237 int i; 238 239 for (i = 0; i < oct->num_irqs; i++) { 240 irq_set_affinity_hint(oct->msix_entries[i].vector, NULL); 241 free_irq(oct->msix_entries[i].vector, oct->ioq_vector[i]); 242 } 243 netdev_info(oct->netdev, "IRQs freed\n"); 244} 245 246/** 247 * octep_vf_setup_irqs() - setup interrupts for the Octeon device. 248 * 249 * @oct: Octeon device private data structure. 250 * 251 * Allocate data structures to hold per interrupt information, allocate/enable 252 * MSI-x interrupt and register interrupt handlers. 253 * 254 * Return: 0, on successful allocation and registration of all interrupts. 255 * -1, on any error. 256 */ 257static int octep_vf_setup_irqs(struct octep_vf_device *oct) 258{ 259 if (octep_vf_alloc_ioq_vectors(oct)) 260 goto ioq_vector_err; 261 262 if (octep_vf_enable_msix_range(oct)) 263 goto enable_msix_err; 264 265 if (octep_vf_request_irqs(oct)) 266 goto request_irq_err; 267 268 return 0; 269 270request_irq_err: 271 octep_vf_disable_msix(oct); 272enable_msix_err: 273 octep_vf_free_ioq_vectors(oct); 274ioq_vector_err: 275 return -1; 276} 277 278/** 279 * octep_vf_clean_irqs() - free all interrupts and its resources. 280 * 281 * @oct: Octeon device private data structure. 282 */ 283static void octep_vf_clean_irqs(struct octep_vf_device *oct) 284{ 285 octep_vf_free_irqs(oct); 286 octep_vf_disable_msix(oct); 287 octep_vf_free_ioq_vectors(oct); 288} 289 290/** 291 * octep_vf_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue. 292 * 293 * @iq: Octeon Tx queue data structure. 294 * @oq: Octeon Rx queue data structure. 295 */ 296static void octep_vf_enable_ioq_irq(struct octep_vf_iq *iq, struct octep_vf_oq *oq) 297{ 298 u32 pkts_pend = oq->pkts_pending; 299 300 netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no); 301 if (iq->pkts_processed) { 302 writel(iq->pkts_processed, iq->inst_cnt_reg); 303 iq->pkt_in_done -= iq->pkts_processed; 304 iq->pkts_processed = 0; 305 } 306 if (oq->last_pkt_count - pkts_pend) { 307 writel(oq->last_pkt_count - pkts_pend, oq->pkts_sent_reg); 308 oq->last_pkt_count = pkts_pend; 309 } 310 311 /* Flush the previous wrties before writing to RESEND bit */ 312 smp_wmb(); 313 writeq(1UL << OCTEP_VF_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg); 314 writeq(1UL << OCTEP_VF_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg); 315} 316 317/** 318 * octep_vf_napi_poll() - NAPI poll function for Tx/Rx. 319 * 320 * @napi: pointer to napi context. 321 * @budget: max number of packets to be processed in single invocation. 322 */ 323static int octep_vf_napi_poll(struct napi_struct *napi, int budget) 324{ 325 struct octep_vf_ioq_vector *ioq_vector = 326 container_of(napi, struct octep_vf_ioq_vector, napi); 327 u32 tx_pending, rx_done; 328 329 tx_pending = octep_vf_iq_process_completions(ioq_vector->iq, 64); 330 rx_done = octep_vf_oq_process_rx(ioq_vector->oq, budget); 331 332 /* need more polling if tx completion processing is still pending or 333 * processed at least 'budget' number of rx packets. 334 */ 335 if (tx_pending || rx_done >= budget) 336 return budget; 337 338 if (likely(napi_complete_done(napi, rx_done))) 339 octep_vf_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq); 340 341 return rx_done; 342} 343 344/** 345 * octep_vf_napi_add() - Add NAPI poll for all Tx/Rx queues. 346 * 347 * @oct: Octeon device private data structure. 348 */ 349static void octep_vf_napi_add(struct octep_vf_device *oct) 350{ 351 int i; 352 353 for (i = 0; i < oct->num_oqs; i++) { 354 netdev_dbg(oct->netdev, "Adding NAPI on Q-%d\n", i); 355 netif_napi_add(oct->netdev, &oct->ioq_vector[i]->napi, octep_vf_napi_poll); 356 oct->oq[i]->napi = &oct->ioq_vector[i]->napi; 357 } 358} 359 360/** 361 * octep_vf_napi_delete() - delete NAPI poll callback for all Tx/Rx queues. 362 * 363 * @oct: Octeon device private data structure. 364 */ 365static void octep_vf_napi_delete(struct octep_vf_device *oct) 366{ 367 int i; 368 369 for (i = 0; i < oct->num_oqs; i++) { 370 netdev_dbg(oct->netdev, "Deleting NAPI on Q-%d\n", i); 371 netif_napi_del(&oct->ioq_vector[i]->napi); 372 oct->oq[i]->napi = NULL; 373 } 374} 375 376/** 377 * octep_vf_napi_enable() - enable NAPI for all Tx/Rx queues. 378 * 379 * @oct: Octeon device private data structure. 380 */ 381static void octep_vf_napi_enable(struct octep_vf_device *oct) 382{ 383 int i; 384 385 for (i = 0; i < oct->num_oqs; i++) { 386 netdev_dbg(oct->netdev, "Enabling NAPI on Q-%d\n", i); 387 napi_enable(&oct->ioq_vector[i]->napi); 388 } 389} 390 391/** 392 * octep_vf_napi_disable() - disable NAPI for all Tx/Rx queues. 393 * 394 * @oct: Octeon device private data structure. 395 */ 396static void octep_vf_napi_disable(struct octep_vf_device *oct) 397{ 398 int i; 399 400 for (i = 0; i < oct->num_oqs; i++) { 401 netdev_dbg(oct->netdev, "Disabling NAPI on Q-%d\n", i); 402 napi_disable(&oct->ioq_vector[i]->napi); 403 } 404} 405 406static void octep_vf_link_up(struct net_device *netdev) 407{ 408 netif_carrier_on(netdev); 409 netif_tx_start_all_queues(netdev); 410} 411 412static void octep_vf_set_rx_state(struct octep_vf_device *oct, bool up) 413{ 414 int err; 415 416 err = octep_vf_mbox_set_rx_state(oct, up); 417 if (err) 418 netdev_err(oct->netdev, "Set Rx state to %d failed with err:%d\n", up, err); 419} 420 421static int octep_vf_get_link_status(struct octep_vf_device *oct) 422{ 423 int err; 424 425 err = octep_vf_mbox_get_link_status(oct, &oct->link_info.oper_up); 426 if (err) 427 netdev_err(oct->netdev, "Get link status failed with err:%d\n", err); 428 return oct->link_info.oper_up; 429} 430 431static void octep_vf_set_link_status(struct octep_vf_device *oct, bool up) 432{ 433 int err; 434 435 err = octep_vf_mbox_set_link_status(oct, up); 436 if (err) { 437 netdev_err(oct->netdev, "Set link status to %d failed with err:%d\n", up, err); 438 return; 439 } 440 oct->link_info.oper_up = up; 441} 442 443/** 444 * octep_vf_open() - start the octeon network device. 445 * 446 * @netdev: pointer to kernel network device. 447 * 448 * setup Tx/Rx queues, interrupts and enable hardware operation of Tx/Rx queues 449 * and interrupts.. 450 * 451 * Return: 0, on successfully setting up device and bring it up. 452 * -1, on any error. 453 */ 454static int octep_vf_open(struct net_device *netdev) 455{ 456 struct octep_vf_device *oct = netdev_priv(netdev); 457 int err, ret; 458 459 netdev_info(netdev, "Starting netdev ...\n"); 460 netif_carrier_off(netdev); 461 462 oct->hw_ops.reset_io_queues(oct); 463 464 if (octep_vf_setup_iqs(oct)) 465 goto setup_iq_err; 466 if (octep_vf_setup_oqs(oct)) 467 goto setup_oq_err; 468 if (octep_vf_setup_irqs(oct)) 469 goto setup_irq_err; 470 471 err = netif_set_real_num_tx_queues(netdev, oct->num_oqs); 472 if (err) 473 goto set_queues_err; 474 err = netif_set_real_num_rx_queues(netdev, oct->num_iqs); 475 if (err) 476 goto set_queues_err; 477 478 octep_vf_napi_add(oct); 479 octep_vf_napi_enable(oct); 480 481 oct->link_info.admin_up = 1; 482 octep_vf_set_rx_state(oct, true); 483 484 ret = octep_vf_get_link_status(oct); 485 if (!ret) 486 octep_vf_set_link_status(oct, true); 487 488 /* Enable the input and output queues for this Octeon device */ 489 oct->hw_ops.enable_io_queues(oct); 490 491 /* Enable Octeon device interrupts */ 492 oct->hw_ops.enable_interrupts(oct); 493 494 octep_vf_oq_dbell_init(oct); 495 496 ret = octep_vf_get_link_status(oct); 497 if (ret) 498 octep_vf_link_up(netdev); 499 500 return 0; 501 502set_queues_err: 503 octep_vf_napi_disable(oct); 504 octep_vf_napi_delete(oct); 505 octep_vf_clean_irqs(oct); 506setup_irq_err: 507 octep_vf_free_oqs(oct); 508setup_oq_err: 509 octep_vf_free_iqs(oct); 510setup_iq_err: 511 return -1; 512} 513 514/** 515 * octep_vf_stop() - stop the octeon network device. 516 * 517 * @netdev: pointer to kernel network device. 518 * 519 * stop the device Tx/Rx operations, bring down the link and 520 * free up all resources allocated for Tx/Rx queues and interrupts. 521 */ 522static int octep_vf_stop(struct net_device *netdev) 523{ 524 struct octep_vf_device *oct = netdev_priv(netdev); 525 526 netdev_info(netdev, "Stopping the device ...\n"); 527 528 /* Stop Tx from stack */ 529 netif_carrier_off(netdev); 530 netif_tx_disable(netdev); 531 532 octep_vf_set_link_status(oct, false); 533 octep_vf_set_rx_state(oct, false); 534 535 oct->link_info.admin_up = 0; 536 oct->link_info.oper_up = 0; 537 538 oct->hw_ops.disable_interrupts(oct); 539 octep_vf_napi_disable(oct); 540 octep_vf_napi_delete(oct); 541 542 octep_vf_clean_irqs(oct); 543 octep_vf_clean_iqs(oct); 544 545 oct->hw_ops.disable_io_queues(oct); 546 oct->hw_ops.reset_io_queues(oct); 547 octep_vf_free_oqs(oct); 548 octep_vf_free_iqs(oct); 549 netdev_info(netdev, "Device stopped !!\n"); 550 return 0; 551} 552 553/** 554 * octep_vf_iq_full_check() - check if a Tx queue is full. 555 * 556 * @iq: Octeon Tx queue data structure. 557 * 558 * Return: 0, if the Tx queue is not full. 559 * 1, if the Tx queue is full. 560 */ 561static int octep_vf_iq_full_check(struct octep_vf_iq *iq) 562{ 563 int ret; 564 565 ret = netif_subqueue_maybe_stop(iq->netdev, iq->q_no, IQ_INSTR_SPACE(iq), 566 OCTEP_VF_WAKE_QUEUE_THRESHOLD, 567 OCTEP_VF_WAKE_QUEUE_THRESHOLD); 568 switch (ret) { 569 case 0: /* Stopped the queue, since IQ is full */ 570 return 1; 571 case -1: /* 572 * Pending updates in write index from 573 * iq_process_completion in other cpus 574 * caused queues to get re-enabled after 575 * being stopped 576 */ 577 iq->stats.restart_cnt++; 578 fallthrough; 579 case 1: /* Queue left enabled, since IQ is not yet full*/ 580 return 0; 581 } 582 583 return 1; 584} 585 586/** 587 * octep_vf_start_xmit() - Enqueue packet to Octoen hardware Tx Queue. 588 * 589 * @skb: packet skbuff pointer. 590 * @netdev: kernel network device. 591 * 592 * Return: NETDEV_TX_BUSY, if Tx Queue is full. 593 * NETDEV_TX_OK, if successfully enqueued to hardware Tx queue. 594 */ 595static netdev_tx_t octep_vf_start_xmit(struct sk_buff *skb, 596 struct net_device *netdev) 597{ 598 struct octep_vf_device *oct = netdev_priv(netdev); 599 netdev_features_t feat = netdev->features; 600 struct octep_vf_tx_sglist_desc *sglist; 601 struct octep_vf_tx_buffer *tx_buffer; 602 struct octep_vf_tx_desc_hw *hw_desc; 603 struct skb_shared_info *shinfo; 604 struct octep_vf_instr_hdr *ih; 605 struct octep_vf_iq *iq; 606 skb_frag_t *frag; 607 u16 nr_frags, si; 608 int xmit_more; 609 u16 q_no, wi; 610 611 if (skb_put_padto(skb, ETH_ZLEN)) 612 return NETDEV_TX_OK; 613 614 q_no = skb_get_queue_mapping(skb); 615 if (q_no >= oct->num_iqs) { 616 netdev_err(netdev, "Invalid Tx skb->queue_mapping=%d\n", q_no); 617 q_no = q_no % oct->num_iqs; 618 } 619 620 iq = oct->iq[q_no]; 621 622 shinfo = skb_shinfo(skb); 623 nr_frags = shinfo->nr_frags; 624 625 wi = iq->host_write_index; 626 hw_desc = &iq->desc_ring[wi]; 627 hw_desc->ih64 = 0; 628 629 tx_buffer = iq->buff_info + wi; 630 tx_buffer->skb = skb; 631 632 ih = &hw_desc->ih; 633 ih->tlen = skb->len; 634 ih->pkind = oct->fw_info.pkind; 635 ih->fsz = oct->fw_info.fsz; 636 ih->tlen = skb->len + ih->fsz; 637 638 if (!nr_frags) { 639 tx_buffer->gather = 0; 640 tx_buffer->dma = dma_map_single(iq->dev, skb->data, 641 skb->len, DMA_TO_DEVICE); 642 if (dma_mapping_error(iq->dev, tx_buffer->dma)) 643 goto dma_map_err; 644 hw_desc->dptr = tx_buffer->dma; 645 } else { 646 /* Scatter/Gather */ 647 dma_addr_t dma; 648 u16 len; 649 650 sglist = tx_buffer->sglist; 651 652 ih->gsz = nr_frags + 1; 653 ih->gather = 1; 654 tx_buffer->gather = 1; 655 656 len = skb_headlen(skb); 657 dma = dma_map_single(iq->dev, skb->data, len, DMA_TO_DEVICE); 658 if (dma_mapping_error(iq->dev, dma)) 659 goto dma_map_err; 660 661 memset(sglist, 0, OCTEP_VF_SGLIST_SIZE_PER_PKT); 662 sglist[0].len[3] = len; 663 sglist[0].dma_ptr[0] = dma; 664 665 si = 1; /* entry 0 is main skb, mapped above */ 666 frag = &shinfo->frags[0]; 667 while (nr_frags--) { 668 len = skb_frag_size(frag); 669 dma = skb_frag_dma_map(iq->dev, frag, 0, 670 len, DMA_TO_DEVICE); 671 if (dma_mapping_error(iq->dev, dma)) 672 goto dma_map_sg_err; 673 674 sglist[si >> 2].len[3 - (si & 3)] = len; 675 sglist[si >> 2].dma_ptr[si & 3] = dma; 676 677 frag++; 678 si++; 679 } 680 hw_desc->dptr = tx_buffer->sglist_dma; 681 } 682 if (oct->fw_info.tx_ol_flags) { 683 if ((feat & (NETIF_F_TSO)) && (skb_is_gso(skb))) { 684 hw_desc->txm.ol_flags = OCTEP_VF_TX_OFFLOAD_CKSUM; 685 hw_desc->txm.ol_flags |= OCTEP_VF_TX_OFFLOAD_TSO; 686 hw_desc->txm.gso_size = skb_shinfo(skb)->gso_size; 687 hw_desc->txm.gso_segs = skb_shinfo(skb)->gso_segs; 688 } else if (feat & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) { 689 hw_desc->txm.ol_flags = OCTEP_VF_TX_OFFLOAD_CKSUM; 690 } 691 /* due to ESR txm will be swapped by hw */ 692 hw_desc->txm64[0] = (__force u64)cpu_to_be64(hw_desc->txm64[0]); 693 } 694 695 xmit_more = netdev_xmit_more(); 696 697 netdev_tx_sent_queue(iq->netdev_q, skb->len); 698 699 skb_tx_timestamp(skb); 700 iq->fill_cnt++; 701 wi++; 702 iq->host_write_index = wi & iq->ring_size_mask; 703 704 /* octep_iq_full_check stops the queue and returns 705 * true if so, in case the queue has become full 706 * by inserting current packet. If so, we can 707 * go ahead and ring doorbell. 708 */ 709 if (!octep_vf_iq_full_check(iq) && xmit_more && 710 iq->fill_cnt < iq->fill_threshold) 711 return NETDEV_TX_OK; 712 713 goto ring_dbell; 714 715dma_map_sg_err: 716 if (si > 0) { 717 dma_unmap_single(iq->dev, sglist[0].dma_ptr[0], 718 sglist[0].len[0], DMA_TO_DEVICE); 719 sglist[0].len[0] = 0; 720 } 721 while (si > 1) { 722 dma_unmap_page(iq->dev, sglist[si >> 2].dma_ptr[si & 3], 723 sglist[si >> 2].len[si & 3], DMA_TO_DEVICE); 724 sglist[si >> 2].len[si & 3] = 0; 725 si--; 726 } 727 tx_buffer->gather = 0; 728dma_map_err: 729 dev_kfree_skb_any(skb); 730ring_dbell: 731 /* Flush the hw descriptors before writing to doorbell */ 732 smp_wmb(); 733 writel(iq->fill_cnt, iq->doorbell_reg); 734 iq->stats.instr_posted += iq->fill_cnt; 735 iq->fill_cnt = 0; 736 return NETDEV_TX_OK; 737} 738 739int octep_vf_get_if_stats(struct octep_vf_device *oct) 740{ 741 struct octep_vf_iface_rxtx_stats vf_stats; 742 int ret, size; 743 744 memset(&vf_stats, 0, sizeof(struct octep_vf_iface_rxtx_stats)); 745 ret = octep_vf_mbox_bulk_read(oct, OCTEP_PFVF_MBOX_CMD_GET_STATS, 746 (u8 *)&vf_stats, &size); 747 748 if (ret) 749 return ret; 750 751 memcpy(&oct->iface_rx_stats, &vf_stats.iface_rx_stats, 752 sizeof(struct octep_vf_iface_rx_stats)); 753 memcpy(&oct->iface_tx_stats, &vf_stats.iface_tx_stats, 754 sizeof(struct octep_vf_iface_tx_stats)); 755 756 return 0; 757} 758 759int octep_vf_get_link_info(struct octep_vf_device *oct) 760{ 761 int ret, size; 762 763 ret = octep_vf_mbox_bulk_read(oct, OCTEP_PFVF_MBOX_CMD_GET_LINK_INFO, 764 (u8 *)&oct->link_info, &size); 765 if (ret) { 766 dev_err(&oct->pdev->dev, "Get VF link info failed via VF Mbox\n"); 767 return ret; 768 } 769 return 0; 770} 771 772/** 773 * octep_vf_get_stats64() - Get Octeon network device statistics. 774 * 775 * @netdev: kernel network device. 776 * @stats: pointer to stats structure to be filled in. 777 */ 778static void octep_vf_get_stats64(struct net_device *netdev, 779 struct rtnl_link_stats64 *stats) 780{ 781 struct octep_vf_device *oct = netdev_priv(netdev); 782 u64 tx_packets, tx_bytes, rx_packets, rx_bytes; 783 int q; 784 785 tx_packets = 0; 786 tx_bytes = 0; 787 rx_packets = 0; 788 rx_bytes = 0; 789 for (q = 0; q < oct->num_oqs; q++) { 790 struct octep_vf_iq *iq = oct->iq[q]; 791 struct octep_vf_oq *oq = oct->oq[q]; 792 793 tx_packets += iq->stats.instr_completed; 794 tx_bytes += iq->stats.bytes_sent; 795 rx_packets += oq->stats.packets; 796 rx_bytes += oq->stats.bytes; 797 } 798 stats->tx_packets = tx_packets; 799 stats->tx_bytes = tx_bytes; 800 stats->rx_packets = rx_packets; 801 stats->rx_bytes = rx_bytes; 802 if (!octep_vf_get_if_stats(oct)) { 803 stats->multicast = oct->iface_rx_stats.mcast_pkts; 804 stats->rx_errors = oct->iface_rx_stats.err_pkts; 805 stats->rx_dropped = oct->iface_rx_stats.dropped_pkts_fifo_full + 806 oct->iface_rx_stats.err_pkts; 807 stats->rx_missed_errors = oct->iface_rx_stats.dropped_pkts_fifo_full; 808 stats->tx_dropped = oct->iface_tx_stats.dropped; 809 } 810} 811 812/** 813 * octep_vf_tx_timeout_task - work queue task to Handle Tx queue timeout. 814 * 815 * @work: pointer to Tx queue timeout work_struct 816 * 817 * Stop and start the device so that it frees up all queue resources 818 * and restarts the queues, that potentially clears a Tx queue timeout 819 * condition. 820 **/ 821static void octep_vf_tx_timeout_task(struct work_struct *work) 822{ 823 struct octep_vf_device *oct = container_of(work, struct octep_vf_device, 824 tx_timeout_task); 825 struct net_device *netdev = oct->netdev; 826 827 rtnl_lock(); 828 if (netif_running(netdev)) { 829 octep_vf_stop(netdev); 830 octep_vf_open(netdev); 831 } 832 rtnl_unlock(); 833 netdev_put(netdev, NULL); 834} 835 836/** 837 * octep_vf_tx_timeout() - Handle Tx Queue timeout. 838 * 839 * @netdev: pointer to kernel network device. 840 * @txqueue: Timed out Tx queue number. 841 * 842 * Schedule a work to handle Tx queue timeout. 843 */ 844static void octep_vf_tx_timeout(struct net_device *netdev, unsigned int txqueue) 845{ 846 struct octep_vf_device *oct = netdev_priv(netdev); 847 848 netdev_hold(netdev, NULL, GFP_ATOMIC); 849 schedule_work(&oct->tx_timeout_task); 850} 851 852static int octep_vf_set_mac(struct net_device *netdev, void *p) 853{ 854 struct octep_vf_device *oct = netdev_priv(netdev); 855 struct sockaddr *addr = (struct sockaddr *)p; 856 int err; 857 858 if (!is_valid_ether_addr(addr->sa_data)) 859 return -EADDRNOTAVAIL; 860 861 err = octep_vf_mbox_set_mac_addr(oct, addr->sa_data); 862 if (err) 863 return err; 864 865 memcpy(oct->mac_addr, addr->sa_data, ETH_ALEN); 866 eth_hw_addr_set(netdev, addr->sa_data); 867 868 return 0; 869} 870 871static int octep_vf_change_mtu(struct net_device *netdev, int new_mtu) 872{ 873 struct octep_vf_device *oct = netdev_priv(netdev); 874 struct octep_vf_iface_link_info *link_info; 875 int err; 876 877 link_info = &oct->link_info; 878 if (link_info->mtu == new_mtu) 879 return 0; 880 881 err = octep_vf_mbox_set_mtu(oct, new_mtu); 882 if (!err) { 883 oct->link_info.mtu = new_mtu; 884 netdev->mtu = new_mtu; 885 } 886 return err; 887} 888 889static int octep_vf_set_features(struct net_device *netdev, 890 netdev_features_t features) 891{ 892 struct octep_vf_device *oct = netdev_priv(netdev); 893 u16 rx_offloads = 0, tx_offloads = 0; 894 int err; 895 896 /* We only support features received from firmware */ 897 if ((features & netdev->hw_features) != features) 898 return -EINVAL; 899 900 if (features & NETIF_F_TSO) 901 tx_offloads |= OCTEP_VF_TX_OFFLOAD_TSO; 902 903 if (features & NETIF_F_TSO6) 904 tx_offloads |= OCTEP_VF_TX_OFFLOAD_TSO; 905 906 if (features & NETIF_F_IP_CSUM) 907 tx_offloads |= OCTEP_VF_TX_OFFLOAD_CKSUM; 908 909 if (features & NETIF_F_IPV6_CSUM) 910 tx_offloads |= OCTEP_VF_TX_OFFLOAD_CKSUM; 911 912 if (features & NETIF_F_RXCSUM) 913 rx_offloads |= OCTEP_VF_RX_OFFLOAD_CKSUM; 914 915 err = octep_vf_mbox_set_offloads(oct, tx_offloads, rx_offloads); 916 if (!err) 917 netdev->features = features; 918 919 return err; 920} 921 922static const struct net_device_ops octep_vf_netdev_ops = { 923 .ndo_open = octep_vf_open, 924 .ndo_stop = octep_vf_stop, 925 .ndo_start_xmit = octep_vf_start_xmit, 926 .ndo_get_stats64 = octep_vf_get_stats64, 927 .ndo_tx_timeout = octep_vf_tx_timeout, 928 .ndo_set_mac_address = octep_vf_set_mac, 929 .ndo_change_mtu = octep_vf_change_mtu, 930 .ndo_set_features = octep_vf_set_features, 931}; 932 933static const char *octep_vf_devid_to_str(struct octep_vf_device *oct) 934{ 935 switch (oct->chip_id) { 936 case OCTEP_PCI_DEVICE_ID_CN93_VF: 937 return "CN93XX"; 938 case OCTEP_PCI_DEVICE_ID_CNF95N_VF: 939 return "CNF95N"; 940 case OCTEP_PCI_DEVICE_ID_CN10KA_VF: 941 return "CN10KA"; 942 case OCTEP_PCI_DEVICE_ID_CNF10KA_VF: 943 return "CNF10KA"; 944 case OCTEP_PCI_DEVICE_ID_CNF10KB_VF: 945 return "CNF10KB"; 946 case OCTEP_PCI_DEVICE_ID_CN10KB_VF: 947 return "CN10KB"; 948 default: 949 return "Unsupported"; 950 } 951} 952 953/** 954 * octep_vf_device_setup() - Setup Octeon Device. 955 * 956 * @oct: Octeon device private data structure. 957 * 958 * Setup Octeon device hardware operations, configuration, etc ... 959 */ 960int octep_vf_device_setup(struct octep_vf_device *oct) 961{ 962 struct pci_dev *pdev = oct->pdev; 963 964 /* allocate memory for oct->conf */ 965 oct->conf = kzalloc(sizeof(*oct->conf), GFP_KERNEL); 966 if (!oct->conf) 967 return -ENOMEM; 968 969 /* Map BAR region 0 */ 970 oct->mmio.hw_addr = ioremap(pci_resource_start(oct->pdev, 0), 971 pci_resource_len(oct->pdev, 0)); 972 if (!oct->mmio.hw_addr) { 973 dev_err(&pdev->dev, 974 "Failed to remap BAR0; start=0x%llx len=0x%llx\n", 975 pci_resource_start(oct->pdev, 0), 976 pci_resource_len(oct->pdev, 0)); 977 goto ioremap_err; 978 } 979 oct->mmio.mapped = 1; 980 981 oct->chip_id = pdev->device; 982 oct->rev_id = pdev->revision; 983 dev_info(&pdev->dev, "chip_id = 0x%x\n", pdev->device); 984 985 switch (oct->chip_id) { 986 case OCTEP_PCI_DEVICE_ID_CN93_VF: 987 case OCTEP_PCI_DEVICE_ID_CNF95N_VF: 988 case OCTEP_PCI_DEVICE_ID_CN98_VF: 989 dev_info(&pdev->dev, "Setting up OCTEON %s VF PASS%d.%d\n", 990 octep_vf_devid_to_str(oct), OCTEP_VF_MAJOR_REV(oct), 991 OCTEP_VF_MINOR_REV(oct)); 992 octep_vf_device_setup_cn93(oct); 993 break; 994 case OCTEP_PCI_DEVICE_ID_CNF10KA_VF: 995 case OCTEP_PCI_DEVICE_ID_CN10KA_VF: 996 case OCTEP_PCI_DEVICE_ID_CNF10KB_VF: 997 case OCTEP_PCI_DEVICE_ID_CN10KB_VF: 998 dev_info(&pdev->dev, "Setting up OCTEON %s VF PASS%d.%d\n", 999 octep_vf_devid_to_str(oct), OCTEP_VF_MAJOR_REV(oct), 1000 OCTEP_VF_MINOR_REV(oct)); 1001 octep_vf_device_setup_cnxk(oct); 1002 break; 1003 default: 1004 dev_err(&pdev->dev, "Unsupported device\n"); 1005 goto unsupported_dev; 1006 } 1007 1008 return 0; 1009 1010unsupported_dev: 1011 iounmap(oct->mmio.hw_addr); 1012ioremap_err: 1013 kfree(oct->conf); 1014 return -EOPNOTSUPP; 1015} 1016 1017/** 1018 * octep_vf_device_cleanup() - Cleanup Octeon Device. 1019 * 1020 * @oct: Octeon device private data structure. 1021 * 1022 * Cleanup Octeon device allocated resources. 1023 */ 1024static void octep_vf_device_cleanup(struct octep_vf_device *oct) 1025{ 1026 dev_info(&oct->pdev->dev, "Cleaning up Octeon Device ...\n"); 1027 1028 if (oct->mmio.mapped) 1029 iounmap(oct->mmio.hw_addr); 1030 1031 kfree(oct->conf); 1032 oct->conf = NULL; 1033} 1034 1035static int octep_vf_get_mac_addr(struct octep_vf_device *oct, u8 *addr) 1036{ 1037 return octep_vf_mbox_get_mac_addr(oct, addr); 1038} 1039 1040/** 1041 * octep_vf_probe() - Octeon PCI device probe handler. 1042 * 1043 * @pdev: PCI device structure. 1044 * @ent: entry in Octeon PCI device ID table. 1045 * 1046 * Initializes and enables the Octeon PCI device for network operations. 1047 * Initializes Octeon private data structure and registers a network device. 1048 */ 1049static int octep_vf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 1050{ 1051 struct octep_vf_device *octep_vf_dev; 1052 struct net_device *netdev; 1053 int err; 1054 1055 err = pci_enable_device(pdev); 1056 if (err) { 1057 dev_err(&pdev->dev, "Failed to enable PCI device\n"); 1058 return err; 1059 } 1060 1061 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1062 if (err) { 1063 dev_err(&pdev->dev, "Failed to set DMA mask !!\n"); 1064 goto disable_pci_device; 1065 } 1066 1067 err = pci_request_mem_regions(pdev, OCTEP_VF_DRV_NAME); 1068 if (err) { 1069 dev_err(&pdev->dev, "Failed to map PCI memory regions\n"); 1070 goto disable_pci_device; 1071 } 1072 1073 pci_set_master(pdev); 1074 1075 netdev = alloc_etherdev_mq(sizeof(struct octep_vf_device), 1076 OCTEP_VF_MAX_QUEUES); 1077 if (!netdev) { 1078 dev_err(&pdev->dev, "Failed to allocate netdev\n"); 1079 err = -ENOMEM; 1080 goto mem_regions_release; 1081 } 1082 SET_NETDEV_DEV(netdev, &pdev->dev); 1083 1084 octep_vf_dev = netdev_priv(netdev); 1085 octep_vf_dev->netdev = netdev; 1086 octep_vf_dev->pdev = pdev; 1087 octep_vf_dev->dev = &pdev->dev; 1088 pci_set_drvdata(pdev, octep_vf_dev); 1089 1090 err = octep_vf_device_setup(octep_vf_dev); 1091 if (err) { 1092 dev_err(&pdev->dev, "Device setup failed\n"); 1093 goto netdevice_free; 1094 } 1095 INIT_WORK(&octep_vf_dev->tx_timeout_task, octep_vf_tx_timeout_task); 1096 1097 netdev->netdev_ops = &octep_vf_netdev_ops; 1098 octep_vf_set_ethtool_ops(netdev); 1099 netif_carrier_off(netdev); 1100 1101 if (octep_vf_setup_mbox(octep_vf_dev)) { 1102 dev_err(&pdev->dev, "VF Mailbox setup failed\n"); 1103 err = -ENOMEM; 1104 goto device_cleanup; 1105 } 1106 1107 if (octep_vf_mbox_version_check(octep_vf_dev)) { 1108 dev_err(&pdev->dev, "PF VF Mailbox version mismatch\n"); 1109 err = -EINVAL; 1110 goto delete_mbox; 1111 } 1112 1113 if (octep_vf_mbox_get_fw_info(octep_vf_dev)) { 1114 dev_err(&pdev->dev, "unable to get fw info\n"); 1115 err = -EINVAL; 1116 goto delete_mbox; 1117 } 1118 1119 netdev->hw_features = NETIF_F_SG; 1120 if (OCTEP_VF_TX_IP_CSUM(octep_vf_dev->fw_info.tx_ol_flags)) 1121 netdev->hw_features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM); 1122 1123 if (OCTEP_VF_RX_IP_CSUM(octep_vf_dev->fw_info.rx_ol_flags)) 1124 netdev->hw_features |= NETIF_F_RXCSUM; 1125 1126 netdev->min_mtu = OCTEP_VF_MIN_MTU; 1127 netdev->max_mtu = OCTEP_VF_MAX_MTU; 1128 netdev->mtu = OCTEP_VF_DEFAULT_MTU; 1129 1130 if (OCTEP_VF_TX_TSO(octep_vf_dev->fw_info.tx_ol_flags)) { 1131 netdev->hw_features |= NETIF_F_TSO; 1132 netif_set_tso_max_size(netdev, netdev->max_mtu); 1133 } 1134 1135 netdev->features |= netdev->hw_features; 1136 octep_vf_get_mac_addr(octep_vf_dev, octep_vf_dev->mac_addr); 1137 eth_hw_addr_set(netdev, octep_vf_dev->mac_addr); 1138 err = register_netdev(netdev); 1139 if (err) { 1140 dev_err(&pdev->dev, "Failed to register netdev\n"); 1141 goto delete_mbox; 1142 } 1143 dev_info(&pdev->dev, "Device probe successful\n"); 1144 return 0; 1145 1146delete_mbox: 1147 octep_vf_delete_mbox(octep_vf_dev); 1148device_cleanup: 1149 octep_vf_device_cleanup(octep_vf_dev); 1150netdevice_free: 1151 free_netdev(netdev); 1152mem_regions_release: 1153 pci_release_mem_regions(pdev); 1154disable_pci_device: 1155 pci_disable_device(pdev); 1156 dev_err(&pdev->dev, "Device probe failed\n"); 1157 return err; 1158} 1159 1160/** 1161 * octep_vf_remove() - Remove Octeon PCI device from driver control. 1162 * 1163 * @pdev: PCI device structure of the Octeon device. 1164 * 1165 * Cleanup all resources allocated for the Octeon device. 1166 * Unregister from network device and disable the PCI device. 1167 */ 1168static void octep_vf_remove(struct pci_dev *pdev) 1169{ 1170 struct octep_vf_device *oct = pci_get_drvdata(pdev); 1171 struct net_device *netdev; 1172 1173 if (!oct) 1174 return; 1175 1176 octep_vf_mbox_dev_remove(oct); 1177 cancel_work_sync(&oct->tx_timeout_task); 1178 netdev = oct->netdev; 1179 if (netdev->reg_state == NETREG_REGISTERED) 1180 unregister_netdev(netdev); 1181 octep_vf_delete_mbox(oct); 1182 octep_vf_device_cleanup(oct); 1183 pci_release_mem_regions(pdev); 1184 free_netdev(netdev); 1185 pci_disable_device(pdev); 1186} 1187 1188static struct pci_driver octep_vf_driver = { 1189 .name = OCTEP_VF_DRV_NAME, 1190 .id_table = octep_vf_pci_id_tbl, 1191 .probe = octep_vf_probe, 1192 .remove = octep_vf_remove, 1193}; 1194 1195/** 1196 * octep_vf_init_module() - Module initialization. 1197 * 1198 * create common resource for the driver and register PCI driver. 1199 */ 1200static int __init octep_vf_init_module(void) 1201{ 1202 int ret; 1203 1204 pr_info("%s: Loading %s ...\n", OCTEP_VF_DRV_NAME, OCTEP_VF_DRV_STRING); 1205 1206 ret = pci_register_driver(&octep_vf_driver); 1207 if (ret < 0) { 1208 pr_err("%s: Failed to register PCI driver; err=%d\n", 1209 OCTEP_VF_DRV_NAME, ret); 1210 return ret; 1211 } 1212 1213 return ret; 1214} 1215 1216/** 1217 * octep_vf_exit_module() - Module exit routine. 1218 * 1219 * unregister the driver with PCI subsystem and cleanup common resources. 1220 */ 1221static void __exit octep_vf_exit_module(void) 1222{ 1223 pr_info("%s: Unloading ...\n", OCTEP_VF_DRV_NAME); 1224 1225 pci_unregister_driver(&octep_vf_driver); 1226 1227 pr_info("%s: Unloading complete\n", OCTEP_VF_DRV_NAME); 1228} 1229 1230module_init(octep_vf_init_module); 1231module_exit(octep_vf_exit_module); 1232