1// SPDX-License-Identifier: GPL-2.0+ 2/* Copyright (c) 2018 Quantenna Communications */ 3 4#include <linux/kernel.h> 5#include <linux/firmware.h> 6#include <linux/pci.h> 7#include <linux/vmalloc.h> 8#include <linux/delay.h> 9#include <linux/interrupt.h> 10#include <linux/sched.h> 11#include <linux/crc32.h> 12#include <linux/completion.h> 13#include <linux/spinlock.h> 14#include <linux/circ_buf.h> 15 16#include "pcie_priv.h" 17#include "topaz_pcie_regs.h" 18#include "topaz_pcie_ipc.h" 19#include "qtn_hw_ids.h" 20#include "core.h" 21#include "bus.h" 22#include "shm_ipc.h" 23#include "debug.h" 24 25#define TOPAZ_TX_BD_SIZE_DEFAULT 128 26#define TOPAZ_RX_BD_SIZE_DEFAULT 256 27 28struct qtnf_topaz_tx_bd { 29 __le32 addr; 30 __le32 info; 31} __packed; 32 33struct qtnf_topaz_rx_bd { 34 __le32 addr; 35 __le32 info; 36} __packed; 37 38struct qtnf_extra_bd_params { 39 __le32 param1; 40 __le32 param2; 41 __le32 param3; 42 __le32 param4; 43} __packed; 44 45#define QTNF_BD_PARAM_OFFSET(n) offsetof(struct qtnf_extra_bd_params, param##n) 46 47struct vmac_pkt_info { 48 __le32 addr; 49 __le32 info; 50}; 51 52struct qtnf_topaz_bda { 53 __le16 bda_len; 54 __le16 bda_version; 55 __le32 bda_bootstate; 56 __le32 bda_dma_mask; 57 __le32 bda_dma_offset; 58 __le32 bda_flags; 59 __le32 bda_img; 60 __le32 bda_img_size; 61 __le32 bda_ep2h_irqstatus; 62 __le32 bda_h2ep_irqstatus; 63 __le32 bda_msi_addr; 64 u8 reserved1[56]; 65 __le32 bda_flashsz; 66 u8 bda_boardname[PCIE_BDA_NAMELEN]; 67 __le32 bda_pci_pre_status; 68 __le32 bda_pci_endian; 69 __le32 bda_pci_post_status; 70 __le32 bda_h2ep_txd_budget; 71 __le32 bda_ep2h_txd_budget; 72 __le32 bda_rc_rx_bd_base; 73 __le32 bda_rc_rx_bd_num; 74 __le32 bda_rc_tx_bd_base; 75 __le32 bda_rc_tx_bd_num; 76 u8 bda_ep_link_state; 77 u8 bda_rc_link_state; 78 u8 bda_rc_msi_enabled; 79 u8 reserved2; 80 __le32 bda_ep_next_pkt; 81 struct vmac_pkt_info request[QTN_PCIE_RC_TX_QUEUE_LEN]; 82 struct qtnf_shm_ipc_region bda_shm_reg1 __aligned(4096); 83 struct qtnf_shm_ipc_region bda_shm_reg2 __aligned(4096); 84} __packed; 85 86struct qtnf_pcie_topaz_state { 87 struct qtnf_pcie_bus_priv base; 88 struct qtnf_topaz_bda __iomem *bda; 89 90 dma_addr_t dma_msi_dummy; 91 u32 dma_msi_imwr; 92 93 struct qtnf_topaz_tx_bd *tx_bd_vbase; 94 struct qtnf_topaz_rx_bd *rx_bd_vbase; 95 96 __le32 __iomem *ep_next_rx_pkt; 97 __le32 __iomem *txqueue_wake; 98 __le32 __iomem *ep_pmstate; 99 100 unsigned long rx_pkt_count; 101}; 102 103static void qtnf_deassert_intx(struct qtnf_pcie_topaz_state *ts) 104{ 105 void __iomem *reg = ts->base.sysctl_bar + TOPAZ_PCIE_CFG0_OFFSET; 106 u32 cfg; 107 108 cfg = readl(reg); 109 cfg &= ~TOPAZ_ASSERT_INTX; 110 qtnf_non_posted_write(cfg, reg); 111} 112 113static inline int qtnf_topaz_intx_asserted(struct qtnf_pcie_topaz_state *ts) 114{ 115 void __iomem *reg = ts->base.sysctl_bar + TOPAZ_PCIE_CFG0_OFFSET; 116 u32 cfg = readl(reg); 117 118 return !!(cfg & TOPAZ_ASSERT_INTX); 119} 120 121static void qtnf_topaz_reset_ep(struct qtnf_pcie_topaz_state *ts) 122{ 123 writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_RST_EP_IRQ), 124 TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar)); 125 msleep(QTN_EP_RESET_WAIT_MS); 126 pci_restore_state(ts->base.pdev); 127} 128 129static void setup_rx_irqs(struct qtnf_pcie_topaz_state *ts) 130{ 131 void __iomem *reg = PCIE_DMA_WR_DONE_IMWR_ADDR_LOW(ts->base.dmareg_bar); 132 133 ts->dma_msi_imwr = readl(reg); 134} 135 136static void enable_rx_irqs(struct qtnf_pcie_topaz_state *ts) 137{ 138 void __iomem *reg = PCIE_DMA_WR_DONE_IMWR_ADDR_LOW(ts->base.dmareg_bar); 139 140 qtnf_non_posted_write(ts->dma_msi_imwr, reg); 141} 142 143static void disable_rx_irqs(struct qtnf_pcie_topaz_state *ts) 144{ 145 void __iomem *reg = PCIE_DMA_WR_DONE_IMWR_ADDR_LOW(ts->base.dmareg_bar); 146 147 qtnf_non_posted_write(QTN_HOST_LO32(ts->dma_msi_dummy), reg); 148} 149 150static void qtnf_topaz_ipc_gen_ep_int(void *arg) 151{ 152 struct qtnf_pcie_topaz_state *ts = arg; 153 154 writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_CTRL_IRQ), 155 TOPAZ_CTL_M2L_INT(ts->base.sysctl_bar)); 156} 157 158static int qtnf_is_state(__le32 __iomem *reg, u32 state) 159{ 160 u32 s = readl(reg); 161 162 return (s == state); 163} 164 165static void qtnf_set_state(__le32 __iomem *reg, u32 state) 166{ 167 qtnf_non_posted_write(state, reg); 168} 169 170static int qtnf_poll_state(__le32 __iomem *reg, u32 state, u32 delay_in_ms) 171{ 172 u32 timeout = 0; 173 174 while ((qtnf_is_state(reg, state) == 0)) { 175 usleep_range(1000, 1200); 176 if (++timeout > delay_in_ms) 177 return -1; 178 } 179 180 return 0; 181} 182 183static int topaz_alloc_bd_table(struct qtnf_pcie_topaz_state *ts, 184 struct qtnf_topaz_bda __iomem *bda) 185{ 186 struct qtnf_extra_bd_params __iomem *extra_params; 187 struct qtnf_pcie_bus_priv *priv = &ts->base; 188 dma_addr_t paddr; 189 void *vaddr; 190 int len; 191 int i; 192 193 /* bd table */ 194 195 len = priv->tx_bd_num * sizeof(struct qtnf_topaz_tx_bd) + 196 priv->rx_bd_num * sizeof(struct qtnf_topaz_rx_bd) + 197 sizeof(struct qtnf_extra_bd_params); 198 199 vaddr = dmam_alloc_coherent(&priv->pdev->dev, len, &paddr, GFP_KERNEL); 200 if (!vaddr) 201 return -ENOMEM; 202 203 /* tx bd */ 204 205 ts->tx_bd_vbase = vaddr; 206 qtnf_non_posted_write(paddr, &bda->bda_rc_tx_bd_base); 207 208 for (i = 0; i < priv->tx_bd_num; i++) 209 ts->tx_bd_vbase[i].info |= cpu_to_le32(QTN_BD_EMPTY); 210 211 pr_debug("TX descriptor table: vaddr=0x%p paddr=%pad\n", vaddr, &paddr); 212 213 priv->tx_bd_r_index = 0; 214 priv->tx_bd_w_index = 0; 215 216 /* rx bd */ 217 218 vaddr = ((struct qtnf_topaz_tx_bd *)vaddr) + priv->tx_bd_num; 219 paddr += priv->tx_bd_num * sizeof(struct qtnf_topaz_tx_bd); 220 221 ts->rx_bd_vbase = vaddr; 222 qtnf_non_posted_write(paddr, &bda->bda_rc_rx_bd_base); 223 224 pr_debug("RX descriptor table: vaddr=0x%p paddr=%pad\n", vaddr, &paddr); 225 226 /* extra shared params */ 227 228 vaddr = ((struct qtnf_topaz_rx_bd *)vaddr) + priv->rx_bd_num; 229 paddr += priv->rx_bd_num * sizeof(struct qtnf_topaz_rx_bd); 230 231 extra_params = (struct qtnf_extra_bd_params __iomem *)vaddr; 232 233 ts->ep_next_rx_pkt = &extra_params->param1; 234 qtnf_non_posted_write(paddr + QTNF_BD_PARAM_OFFSET(1), 235 &bda->bda_ep_next_pkt); 236 ts->txqueue_wake = &extra_params->param2; 237 ts->ep_pmstate = &extra_params->param3; 238 ts->dma_msi_dummy = paddr + QTNF_BD_PARAM_OFFSET(4); 239 240 return 0; 241} 242 243static int 244topaz_skb2rbd_attach(struct qtnf_pcie_topaz_state *ts, u16 index, u32 wrap) 245{ 246 struct qtnf_topaz_rx_bd *rxbd = &ts->rx_bd_vbase[index]; 247 struct sk_buff *skb; 248 dma_addr_t paddr; 249 250 skb = netdev_alloc_skb_ip_align(NULL, SKB_BUF_SIZE); 251 if (!skb) { 252 ts->base.rx_skb[index] = NULL; 253 return -ENOMEM; 254 } 255 256 ts->base.rx_skb[index] = skb; 257 258 paddr = dma_map_single(&ts->base.pdev->dev, skb->data, SKB_BUF_SIZE, 259 DMA_FROM_DEVICE); 260 if (dma_mapping_error(&ts->base.pdev->dev, paddr)) { 261 pr_err("skb mapping error: %pad\n", &paddr); 262 return -ENOMEM; 263 } 264 265 rxbd->addr = cpu_to_le32(QTN_HOST_LO32(paddr)); 266 rxbd->info = cpu_to_le32(QTN_BD_EMPTY | wrap); 267 268 ts->base.rx_bd_w_index = index; 269 270 return 0; 271} 272 273static int topaz_alloc_rx_buffers(struct qtnf_pcie_topaz_state *ts) 274{ 275 u16 i; 276 int ret = 0; 277 278 memset(ts->rx_bd_vbase, 0x0, 279 ts->base.rx_bd_num * sizeof(struct qtnf_topaz_rx_bd)); 280 281 for (i = 0; i < ts->base.rx_bd_num; i++) { 282 ret = topaz_skb2rbd_attach(ts, i, 0); 283 if (ret) 284 break; 285 } 286 287 ts->rx_bd_vbase[ts->base.rx_bd_num - 1].info |= 288 cpu_to_le32(QTN_BD_WRAP); 289 290 return ret; 291} 292 293/* all rx/tx activity should have ceased before calling this function */ 294static void qtnf_topaz_free_xfer_buffers(struct qtnf_pcie_topaz_state *ts) 295{ 296 struct qtnf_pcie_bus_priv *priv = &ts->base; 297 struct qtnf_topaz_rx_bd *rxbd; 298 struct qtnf_topaz_tx_bd *txbd; 299 struct sk_buff *skb; 300 dma_addr_t paddr; 301 int i; 302 303 /* free rx buffers */ 304 for (i = 0; i < priv->rx_bd_num; i++) { 305 if (priv->rx_skb && priv->rx_skb[i]) { 306 rxbd = &ts->rx_bd_vbase[i]; 307 skb = priv->rx_skb[i]; 308 paddr = QTN_HOST_ADDR(0x0, le32_to_cpu(rxbd->addr)); 309 dma_unmap_single(&priv->pdev->dev, paddr, 310 SKB_BUF_SIZE, DMA_FROM_DEVICE); 311 dev_kfree_skb_any(skb); 312 priv->rx_skb[i] = NULL; 313 rxbd->addr = 0; 314 rxbd->info = 0; 315 } 316 } 317 318 /* free tx buffers */ 319 for (i = 0; i < priv->tx_bd_num; i++) { 320 if (priv->tx_skb && priv->tx_skb[i]) { 321 txbd = &ts->tx_bd_vbase[i]; 322 skb = priv->tx_skb[i]; 323 paddr = QTN_HOST_ADDR(0x0, le32_to_cpu(txbd->addr)); 324 dma_unmap_single(&priv->pdev->dev, paddr, 325 SKB_BUF_SIZE, DMA_TO_DEVICE); 326 dev_kfree_skb_any(skb); 327 priv->tx_skb[i] = NULL; 328 txbd->addr = 0; 329 txbd->info = 0; 330 } 331 } 332} 333 334static int qtnf_pcie_topaz_init_xfer(struct qtnf_pcie_topaz_state *ts, 335 unsigned int tx_bd_size, 336 unsigned int rx_bd_size) 337{ 338 struct qtnf_topaz_bda __iomem *bda = ts->bda; 339 struct qtnf_pcie_bus_priv *priv = &ts->base; 340 int ret; 341 342 if (tx_bd_size == 0) 343 tx_bd_size = TOPAZ_TX_BD_SIZE_DEFAULT; 344 345 /* check TX BD queue max length according to struct qtnf_topaz_bda */ 346 if (tx_bd_size > QTN_PCIE_RC_TX_QUEUE_LEN) { 347 pr_warn("TX BD queue cannot exceed %d\n", 348 QTN_PCIE_RC_TX_QUEUE_LEN); 349 tx_bd_size = QTN_PCIE_RC_TX_QUEUE_LEN; 350 } 351 352 priv->tx_bd_num = tx_bd_size; 353 qtnf_non_posted_write(priv->tx_bd_num, &bda->bda_rc_tx_bd_num); 354 355 if (rx_bd_size == 0) 356 rx_bd_size = TOPAZ_RX_BD_SIZE_DEFAULT; 357 358 if (rx_bd_size > TOPAZ_RX_BD_SIZE_DEFAULT) { 359 pr_warn("RX BD queue cannot exceed %d\n", 360 TOPAZ_RX_BD_SIZE_DEFAULT); 361 rx_bd_size = TOPAZ_RX_BD_SIZE_DEFAULT; 362 } 363 364 priv->rx_bd_num = rx_bd_size; 365 qtnf_non_posted_write(priv->rx_bd_num, &bda->bda_rc_rx_bd_num); 366 367 priv->rx_bd_w_index = 0; 368 priv->rx_bd_r_index = 0; 369 370 ret = qtnf_pcie_alloc_skb_array(priv); 371 if (ret) { 372 pr_err("failed to allocate skb array\n"); 373 return ret; 374 } 375 376 ret = topaz_alloc_bd_table(ts, bda); 377 if (ret) { 378 pr_err("failed to allocate bd table\n"); 379 return ret; 380 } 381 382 ret = topaz_alloc_rx_buffers(ts); 383 if (ret) { 384 pr_err("failed to allocate rx buffers\n"); 385 return ret; 386 } 387 388 return ret; 389} 390 391static void qtnf_topaz_data_tx_reclaim(struct qtnf_pcie_topaz_state *ts) 392{ 393 struct qtnf_pcie_bus_priv *priv = &ts->base; 394 struct qtnf_topaz_tx_bd *txbd; 395 struct sk_buff *skb; 396 unsigned long flags; 397 dma_addr_t paddr; 398 u32 tx_done_index; 399 int count = 0; 400 int i; 401 402 spin_lock_irqsave(&priv->tx_reclaim_lock, flags); 403 404 tx_done_index = readl(ts->ep_next_rx_pkt); 405 i = priv->tx_bd_r_index; 406 407 if (CIRC_CNT(priv->tx_bd_w_index, tx_done_index, priv->tx_bd_num)) 408 writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_TX_DONE_IRQ), 409 TOPAZ_LH_IPC4_INT(priv->sysctl_bar)); 410 411 while (CIRC_CNT(tx_done_index, i, priv->tx_bd_num)) { 412 skb = priv->tx_skb[i]; 413 414 if (likely(skb)) { 415 txbd = &ts->tx_bd_vbase[i]; 416 paddr = QTN_HOST_ADDR(0x0, le32_to_cpu(txbd->addr)); 417 dma_unmap_single(&priv->pdev->dev, paddr, skb->len, 418 DMA_TO_DEVICE); 419 420 if (skb->dev) { 421 dev_sw_netstats_tx_add(skb->dev, 1, skb->len); 422 if (unlikely(priv->tx_stopped)) { 423 qtnf_wake_all_queues(skb->dev); 424 priv->tx_stopped = 0; 425 } 426 } 427 428 dev_kfree_skb_any(skb); 429 } 430 431 priv->tx_skb[i] = NULL; 432 count++; 433 434 if (++i >= priv->tx_bd_num) 435 i = 0; 436 } 437 438 priv->tx_reclaim_done += count; 439 priv->tx_reclaim_req++; 440 priv->tx_bd_r_index = i; 441 442 spin_unlock_irqrestore(&priv->tx_reclaim_lock, flags); 443} 444 445static void qtnf_try_stop_xmit(struct qtnf_bus *bus, struct net_device *ndev) 446{ 447 struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus); 448 449 if (ndev) { 450 netif_tx_stop_all_queues(ndev); 451 ts->base.tx_stopped = 1; 452 } 453 454 writel(0x0, ts->txqueue_wake); 455 456 /* sync up tx queue status before generating interrupt */ 457 dma_wmb(); 458 459 /* send irq to card: tx stopped */ 460 writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_TX_STOP_IRQ), 461 TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar)); 462 463 /* schedule reclaim attempt */ 464 tasklet_hi_schedule(&ts->base.reclaim_tq); 465} 466 467static void qtnf_try_wake_xmit(struct qtnf_bus *bus, struct net_device *ndev) 468{ 469 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus); 470 int ready; 471 472 ready = readl(ts->txqueue_wake); 473 if (ready) { 474 netif_wake_queue(ndev); 475 } else { 476 /* re-send irq to card: tx stopped */ 477 writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_TX_STOP_IRQ), 478 TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar)); 479 } 480} 481 482static int qtnf_tx_queue_ready(struct qtnf_pcie_topaz_state *ts) 483{ 484 struct qtnf_pcie_bus_priv *priv = &ts->base; 485 486 if (!CIRC_SPACE(priv->tx_bd_w_index, priv->tx_bd_r_index, 487 priv->tx_bd_num)) { 488 qtnf_topaz_data_tx_reclaim(ts); 489 490 if (!CIRC_SPACE(priv->tx_bd_w_index, priv->tx_bd_r_index, 491 priv->tx_bd_num)) { 492 priv->tx_full_count++; 493 return 0; 494 } 495 } 496 497 return 1; 498} 499 500static int qtnf_pcie_data_tx(struct qtnf_bus *bus, struct sk_buff *skb, 501 unsigned int macid, unsigned int vifid) 502{ 503 struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus); 504 struct qtnf_pcie_bus_priv *priv = &ts->base; 505 struct qtnf_topaz_bda __iomem *bda = ts->bda; 506 struct qtnf_topaz_tx_bd *txbd; 507 dma_addr_t skb_paddr; 508 unsigned long flags; 509 int ret = 0; 510 int len; 511 int i; 512 513 spin_lock_irqsave(&priv->tx_lock, flags); 514 515 if (!qtnf_tx_queue_ready(ts)) { 516 qtnf_try_stop_xmit(bus, skb->dev); 517 spin_unlock_irqrestore(&priv->tx_lock, flags); 518 return NETDEV_TX_BUSY; 519 } 520 521 i = priv->tx_bd_w_index; 522 priv->tx_skb[i] = skb; 523 len = skb->len; 524 525 skb_paddr = dma_map_single(&priv->pdev->dev, skb->data, skb->len, 526 DMA_TO_DEVICE); 527 if (dma_mapping_error(&priv->pdev->dev, skb_paddr)) { 528 ret = -ENOMEM; 529 goto tx_done; 530 } 531 532 txbd = &ts->tx_bd_vbase[i]; 533 txbd->addr = cpu_to_le32(QTN_HOST_LO32(skb_paddr)); 534 535 writel(QTN_HOST_LO32(skb_paddr), &bda->request[i].addr); 536 writel(len | QTN_PCIE_TX_VALID_PKT, &bda->request[i].info); 537 538 /* sync up descriptor updates before generating interrupt */ 539 dma_wmb(); 540 541 /* generate irq to card: tx done */ 542 writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_TX_DONE_IRQ), 543 TOPAZ_LH_IPC4_INT(priv->sysctl_bar)); 544 545 if (++i >= priv->tx_bd_num) 546 i = 0; 547 548 priv->tx_bd_w_index = i; 549 550tx_done: 551 if (ret) { 552 if (skb->dev) 553 skb->dev->stats.tx_dropped++; 554 dev_kfree_skb_any(skb); 555 } 556 557 priv->tx_done_count++; 558 spin_unlock_irqrestore(&priv->tx_lock, flags); 559 560 qtnf_topaz_data_tx_reclaim(ts); 561 562 return NETDEV_TX_OK; 563} 564 565static irqreturn_t qtnf_pcie_topaz_interrupt(int irq, void *data) 566{ 567 struct qtnf_bus *bus = (struct qtnf_bus *)data; 568 struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus); 569 struct qtnf_pcie_bus_priv *priv = &ts->base; 570 571 if (!priv->msi_enabled && !qtnf_topaz_intx_asserted(ts)) 572 return IRQ_NONE; 573 574 if (!priv->msi_enabled) 575 qtnf_deassert_intx(ts); 576 577 priv->pcie_irq_count++; 578 579 qtnf_shm_ipc_irq_handler(&priv->shm_ipc_ep_in); 580 qtnf_shm_ipc_irq_handler(&priv->shm_ipc_ep_out); 581 582 if (napi_schedule_prep(&bus->mux_napi)) { 583 disable_rx_irqs(ts); 584 __napi_schedule(&bus->mux_napi); 585 } 586 587 tasklet_hi_schedule(&priv->reclaim_tq); 588 589 return IRQ_HANDLED; 590} 591 592static int qtnf_rx_data_ready(struct qtnf_pcie_topaz_state *ts) 593{ 594 u16 index = ts->base.rx_bd_r_index; 595 struct qtnf_topaz_rx_bd *rxbd; 596 u32 descw; 597 598 rxbd = &ts->rx_bd_vbase[index]; 599 descw = le32_to_cpu(rxbd->info); 600 601 if (descw & QTN_BD_EMPTY) 602 return 0; 603 604 return 1; 605} 606 607static int qtnf_topaz_rx_poll(struct napi_struct *napi, int budget) 608{ 609 struct qtnf_bus *bus = container_of(napi, struct qtnf_bus, mux_napi); 610 struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus); 611 struct qtnf_pcie_bus_priv *priv = &ts->base; 612 struct net_device *ndev = NULL; 613 struct sk_buff *skb = NULL; 614 int processed = 0; 615 struct qtnf_topaz_rx_bd *rxbd; 616 dma_addr_t skb_paddr; 617 int consume; 618 u32 descw; 619 u32 poffset; 620 u32 psize; 621 u16 r_idx; 622 u16 w_idx; 623 int ret; 624 625 while (processed < budget) { 626 if (!qtnf_rx_data_ready(ts)) 627 goto rx_out; 628 629 r_idx = priv->rx_bd_r_index; 630 rxbd = &ts->rx_bd_vbase[r_idx]; 631 descw = le32_to_cpu(rxbd->info); 632 633 skb = priv->rx_skb[r_idx]; 634 poffset = QTN_GET_OFFSET(descw); 635 psize = QTN_GET_LEN(descw); 636 consume = 1; 637 638 if (descw & QTN_BD_EMPTY) { 639 pr_warn("skip invalid rxbd[%d]\n", r_idx); 640 consume = 0; 641 } 642 643 if (!skb) { 644 pr_warn("skip missing rx_skb[%d]\n", r_idx); 645 consume = 0; 646 } 647 648 if (skb && (skb_tailroom(skb) < psize)) { 649 pr_err("skip packet with invalid length: %u > %u\n", 650 psize, skb_tailroom(skb)); 651 consume = 0; 652 } 653 654 if (skb) { 655 skb_paddr = QTN_HOST_ADDR(0x0, le32_to_cpu(rxbd->addr)); 656 dma_unmap_single(&priv->pdev->dev, skb_paddr, 657 SKB_BUF_SIZE, DMA_FROM_DEVICE); 658 } 659 660 if (consume) { 661 skb_reserve(skb, poffset); 662 skb_put(skb, psize); 663 ndev = qtnf_classify_skb(bus, skb); 664 if (likely(ndev)) { 665 dev_sw_netstats_rx_add(ndev, skb->len); 666 skb->protocol = eth_type_trans(skb, ndev); 667 netif_receive_skb(skb); 668 } else { 669 pr_debug("drop untagged skb\n"); 670 bus->mux_dev.stats.rx_dropped++; 671 dev_kfree_skb_any(skb); 672 } 673 } else { 674 if (skb) { 675 bus->mux_dev.stats.rx_dropped++; 676 dev_kfree_skb_any(skb); 677 } 678 } 679 680 /* notify card about recv packets once per several packets */ 681 if (((++ts->rx_pkt_count) & RX_DONE_INTR_MSK) == 0) 682 writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_RX_DONE_IRQ), 683 TOPAZ_LH_IPC4_INT(priv->sysctl_bar)); 684 685 priv->rx_skb[r_idx] = NULL; 686 if (++r_idx >= priv->rx_bd_num) 687 r_idx = 0; 688 689 priv->rx_bd_r_index = r_idx; 690 691 /* repalce processed buffer by a new one */ 692 w_idx = priv->rx_bd_w_index; 693 while (CIRC_SPACE(priv->rx_bd_w_index, priv->rx_bd_r_index, 694 priv->rx_bd_num) > 0) { 695 if (++w_idx >= priv->rx_bd_num) 696 w_idx = 0; 697 698 ret = topaz_skb2rbd_attach(ts, w_idx, 699 descw & QTN_BD_WRAP); 700 if (ret) { 701 pr_err("failed to allocate new rx_skb[%d]\n", 702 w_idx); 703 break; 704 } 705 } 706 707 processed++; 708 } 709 710rx_out: 711 if (processed < budget) { 712 napi_complete(napi); 713 enable_rx_irqs(ts); 714 } 715 716 return processed; 717} 718 719static void 720qtnf_pcie_data_tx_timeout(struct qtnf_bus *bus, struct net_device *ndev) 721{ 722 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus); 723 724 qtnf_try_wake_xmit(bus, ndev); 725 tasklet_hi_schedule(&ts->base.reclaim_tq); 726} 727 728static void qtnf_pcie_data_rx_start(struct qtnf_bus *bus) 729{ 730 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus); 731 732 napi_enable(&bus->mux_napi); 733 enable_rx_irqs(ts); 734} 735 736static void qtnf_pcie_data_rx_stop(struct qtnf_bus *bus) 737{ 738 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus); 739 740 disable_rx_irqs(ts); 741 napi_disable(&bus->mux_napi); 742} 743 744static struct qtnf_bus_ops qtnf_pcie_topaz_bus_ops = { 745 /* control path methods */ 746 .control_tx = qtnf_pcie_control_tx, 747 748 /* data path methods */ 749 .data_tx = qtnf_pcie_data_tx, 750 .data_tx_timeout = qtnf_pcie_data_tx_timeout, 751 .data_rx_start = qtnf_pcie_data_rx_start, 752 .data_rx_stop = qtnf_pcie_data_rx_stop, 753}; 754 755static int qtnf_dbg_irq_stats(struct seq_file *s, void *data) 756{ 757 struct qtnf_bus *bus = dev_get_drvdata(s->private); 758 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus); 759 760 seq_printf(s, "pcie_irq_count(%u)\n", ts->base.pcie_irq_count); 761 762 return 0; 763} 764 765static int qtnf_dbg_pkt_stats(struct seq_file *s, void *data) 766{ 767 struct qtnf_bus *bus = dev_get_drvdata(s->private); 768 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus); 769 struct qtnf_pcie_bus_priv *priv = &ts->base; 770 u32 tx_done_index = readl(ts->ep_next_rx_pkt); 771 772 seq_printf(s, "tx_full_count(%u)\n", priv->tx_full_count); 773 seq_printf(s, "tx_done_count(%u)\n", priv->tx_done_count); 774 seq_printf(s, "tx_reclaim_done(%u)\n", priv->tx_reclaim_done); 775 seq_printf(s, "tx_reclaim_req(%u)\n", priv->tx_reclaim_req); 776 777 seq_printf(s, "tx_bd_r_index(%u)\n", priv->tx_bd_r_index); 778 seq_printf(s, "tx_done_index(%u)\n", tx_done_index); 779 seq_printf(s, "tx_bd_w_index(%u)\n", priv->tx_bd_w_index); 780 781 seq_printf(s, "tx host queue len(%u)\n", 782 CIRC_CNT(priv->tx_bd_w_index, priv->tx_bd_r_index, 783 priv->tx_bd_num)); 784 seq_printf(s, "tx reclaim queue len(%u)\n", 785 CIRC_CNT(tx_done_index, priv->tx_bd_r_index, 786 priv->tx_bd_num)); 787 seq_printf(s, "tx card queue len(%u)\n", 788 CIRC_CNT(priv->tx_bd_w_index, tx_done_index, 789 priv->tx_bd_num)); 790 791 seq_printf(s, "rx_bd_r_index(%u)\n", priv->rx_bd_r_index); 792 seq_printf(s, "rx_bd_w_index(%u)\n", priv->rx_bd_w_index); 793 seq_printf(s, "rx alloc queue len(%u)\n", 794 CIRC_SPACE(priv->rx_bd_w_index, priv->rx_bd_r_index, 795 priv->rx_bd_num)); 796 797 return 0; 798} 799 800static void qtnf_reset_dma_offset(struct qtnf_pcie_topaz_state *ts) 801{ 802 struct qtnf_topaz_bda __iomem *bda = ts->bda; 803 u32 offset = readl(&bda->bda_dma_offset); 804 805 if ((offset & PCIE_DMA_OFFSET_ERROR_MASK) != PCIE_DMA_OFFSET_ERROR) 806 return; 807 808 writel(0x0, &bda->bda_dma_offset); 809} 810 811static int qtnf_pcie_endian_detect(struct qtnf_pcie_topaz_state *ts) 812{ 813 struct qtnf_topaz_bda __iomem *bda = ts->bda; 814 u32 timeout = 0; 815 u32 endian; 816 int ret = 0; 817 818 writel(QTN_PCI_ENDIAN_DETECT_DATA, &bda->bda_pci_endian); 819 820 /* flush endian modifications before status update */ 821 dma_wmb(); 822 823 writel(QTN_PCI_ENDIAN_VALID_STATUS, &bda->bda_pci_pre_status); 824 825 while (readl(&bda->bda_pci_post_status) != 826 QTN_PCI_ENDIAN_VALID_STATUS) { 827 usleep_range(1000, 1200); 828 if (++timeout > QTN_FW_DL_TIMEOUT_MS) { 829 pr_err("card endianness detection timed out\n"); 830 ret = -ETIMEDOUT; 831 goto endian_out; 832 } 833 } 834 835 /* do not read before status is updated */ 836 dma_rmb(); 837 838 endian = readl(&bda->bda_pci_endian); 839 WARN(endian != QTN_PCI_LITTLE_ENDIAN, 840 "%s: unexpected card endianness", __func__); 841 842endian_out: 843 writel(0, &bda->bda_pci_pre_status); 844 writel(0, &bda->bda_pci_post_status); 845 writel(0, &bda->bda_pci_endian); 846 847 return ret; 848} 849 850static int qtnf_pre_init_ep(struct qtnf_bus *bus) 851{ 852 struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus); 853 struct qtnf_topaz_bda __iomem *bda = ts->bda; 854 u32 flags; 855 int ret; 856 857 ret = qtnf_pcie_endian_detect(ts); 858 if (ret < 0) { 859 pr_err("failed to detect card endianness\n"); 860 return ret; 861 } 862 863 writeb(ts->base.msi_enabled, &ts->bda->bda_rc_msi_enabled); 864 qtnf_reset_dma_offset(ts); 865 866 /* notify card about driver type and boot mode */ 867 flags = readl(&bda->bda_flags) | QTN_BDA_HOST_QLINK_DRV; 868 869 if (ts->base.flashboot) 870 flags |= QTN_BDA_FLASH_BOOT; 871 else 872 flags &= ~QTN_BDA_FLASH_BOOT; 873 874 writel(flags, &bda->bda_flags); 875 876 qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_HOST_RDY); 877 if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_TARGET_RDY, 878 QTN_FW_DL_TIMEOUT_MS)) { 879 pr_err("card is not ready to boot...\n"); 880 return -ETIMEDOUT; 881 } 882 883 return ret; 884} 885 886static int qtnf_post_init_ep(struct qtnf_pcie_topaz_state *ts) 887{ 888 struct pci_dev *pdev = ts->base.pdev; 889 890 setup_rx_irqs(ts); 891 disable_rx_irqs(ts); 892 893 if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_QLINK_DONE, 894 QTN_FW_QLINK_TIMEOUT_MS)) 895 return -ETIMEDOUT; 896 897 enable_irq(pdev->irq); 898 return 0; 899} 900 901static int 902qtnf_ep_fw_load(struct qtnf_pcie_topaz_state *ts, const u8 *fw, u32 fw_size) 903{ 904 struct qtnf_topaz_bda __iomem *bda = ts->bda; 905 struct pci_dev *pdev = ts->base.pdev; 906 u32 remaining = fw_size; 907 u8 *curr = (u8 *)fw; 908 u32 blksize; 909 u32 nblocks; 910 u32 offset; 911 u32 count; 912 u32 size; 913 dma_addr_t paddr; 914 void *data; 915 int ret = 0; 916 917 pr_debug("FW upload started: fw_addr = 0x%p, size=%d\n", fw, fw_size); 918 919 blksize = ts->base.fw_blksize; 920 921 if (blksize < PAGE_SIZE) 922 blksize = PAGE_SIZE; 923 924 while (blksize >= PAGE_SIZE) { 925 pr_debug("allocating %u bytes to upload FW\n", blksize); 926 data = dma_alloc_coherent(&pdev->dev, blksize, 927 &paddr, GFP_KERNEL); 928 if (data) 929 break; 930 blksize /= 2; 931 } 932 933 if (!data) { 934 pr_err("failed to allocate DMA buffer for FW upload\n"); 935 ret = -ENOMEM; 936 goto fw_load_out; 937 } 938 939 nblocks = NBLOCKS(fw_size, blksize); 940 offset = readl(&bda->bda_dma_offset); 941 942 qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_HOST_LOAD); 943 if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_EP_RDY, 944 QTN_FW_DL_TIMEOUT_MS)) { 945 pr_err("card is not ready to download FW\n"); 946 ret = -ETIMEDOUT; 947 goto fw_load_map; 948 } 949 950 for (count = 0 ; count < nblocks; count++) { 951 size = (remaining > blksize) ? blksize : remaining; 952 953 memcpy(data, curr, size); 954 qtnf_non_posted_write(paddr + offset, &bda->bda_img); 955 qtnf_non_posted_write(size, &bda->bda_img_size); 956 957 pr_debug("chunk[%u] VA[0x%p] PA[%pad] sz[%u]\n", 958 count, (void *)curr, &paddr, size); 959 960 qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_RDY); 961 if (qtnf_poll_state(&ts->bda->bda_bootstate, 962 QTN_BDA_FW_BLOCK_DONE, 963 QTN_FW_DL_TIMEOUT_MS)) { 964 pr_err("confirmation for block #%d timed out\n", count); 965 ret = -ETIMEDOUT; 966 goto fw_load_map; 967 } 968 969 remaining = (remaining < size) ? remaining : (remaining - size); 970 curr += size; 971 } 972 973 /* upload completion mark: zero-sized block */ 974 qtnf_non_posted_write(0, &bda->bda_img); 975 qtnf_non_posted_write(0, &bda->bda_img_size); 976 977 qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_RDY); 978 if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_DONE, 979 QTN_FW_DL_TIMEOUT_MS)) { 980 pr_err("confirmation for the last block timed out\n"); 981 ret = -ETIMEDOUT; 982 goto fw_load_map; 983 } 984 985 /* RC is done */ 986 qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_END); 987 if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_LOAD_DONE, 988 QTN_FW_DL_TIMEOUT_MS)) { 989 pr_err("confirmation for FW upload completion timed out\n"); 990 ret = -ETIMEDOUT; 991 goto fw_load_map; 992 } 993 994 pr_debug("FW upload completed: totally sent %d blocks\n", count); 995 996fw_load_map: 997 dma_free_coherent(&pdev->dev, blksize, data, paddr); 998 999fw_load_out: 1000 return ret; 1001} 1002 1003static int qtnf_topaz_fw_upload(struct qtnf_pcie_topaz_state *ts, 1004 const char *fwname) 1005{ 1006 const struct firmware *fw; 1007 struct pci_dev *pdev = ts->base.pdev; 1008 int ret; 1009 1010 if (qtnf_poll_state(&ts->bda->bda_bootstate, 1011 QTN_BDA_FW_LOAD_RDY, 1012 QTN_FW_DL_TIMEOUT_MS)) { 1013 pr_err("%s: card is not ready\n", fwname); 1014 return -1; 1015 } 1016 1017 pr_info("starting firmware upload: %s\n", fwname); 1018 1019 ret = request_firmware(&fw, fwname, &pdev->dev); 1020 if (ret < 0) { 1021 pr_err("%s: request_firmware error %d\n", fwname, ret); 1022 return -1; 1023 } 1024 1025 ret = qtnf_ep_fw_load(ts, fw->data, fw->size); 1026 release_firmware(fw); 1027 1028 if (ret) 1029 pr_err("%s: FW upload error\n", fwname); 1030 1031 return ret; 1032} 1033 1034static void qtnf_topaz_fw_work_handler(struct work_struct *work) 1035{ 1036 struct qtnf_bus *bus = container_of(work, struct qtnf_bus, fw_work); 1037 struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus); 1038 int bootloader_needed = readl(&ts->bda->bda_flags) & QTN_BDA_XMIT_UBOOT; 1039 struct pci_dev *pdev = ts->base.pdev; 1040 int ret; 1041 1042 qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_TARGET_BOOT); 1043 1044 if (bootloader_needed) { 1045 ret = qtnf_topaz_fw_upload(ts, QTN_PCI_TOPAZ_BOOTLD_NAME); 1046 if (ret) 1047 goto fw_load_exit; 1048 1049 ret = qtnf_pre_init_ep(bus); 1050 if (ret) 1051 goto fw_load_exit; 1052 1053 qtnf_set_state(&ts->bda->bda_bootstate, 1054 QTN_BDA_FW_TARGET_BOOT); 1055 } 1056 1057 if (ts->base.flashboot) { 1058 pr_info("booting firmware from flash\n"); 1059 1060 ret = qtnf_poll_state(&ts->bda->bda_bootstate, 1061 QTN_BDA_FW_FLASH_BOOT, 1062 QTN_FW_DL_TIMEOUT_MS); 1063 if (ret) 1064 goto fw_load_exit; 1065 } else { 1066 ret = qtnf_topaz_fw_upload(ts, QTN_PCI_TOPAZ_FW_NAME); 1067 if (ret) 1068 goto fw_load_exit; 1069 1070 qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_START); 1071 ret = qtnf_poll_state(&ts->bda->bda_bootstate, 1072 QTN_BDA_FW_CONFIG, 1073 QTN_FW_QLINK_TIMEOUT_MS); 1074 if (ret) { 1075 pr_err("FW bringup timed out\n"); 1076 goto fw_load_exit; 1077 } 1078 1079 qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_RUN); 1080 ret = qtnf_poll_state(&ts->bda->bda_bootstate, 1081 QTN_BDA_FW_RUNNING, 1082 QTN_FW_QLINK_TIMEOUT_MS); 1083 if (ret) { 1084 pr_err("card bringup timed out\n"); 1085 goto fw_load_exit; 1086 } 1087 } 1088 1089 ret = qtnf_post_init_ep(ts); 1090 if (ret) { 1091 pr_err("FW runtime failure\n"); 1092 goto fw_load_exit; 1093 } 1094 1095 pr_info("firmware is up and running\n"); 1096 1097 ret = qtnf_pcie_fw_boot_done(bus); 1098 if (ret) 1099 goto fw_load_exit; 1100 1101 qtnf_debugfs_add_entry(bus, "pkt_stats", qtnf_dbg_pkt_stats); 1102 qtnf_debugfs_add_entry(bus, "irq_stats", qtnf_dbg_irq_stats); 1103 1104fw_load_exit: 1105 put_device(&pdev->dev); 1106} 1107 1108static void qtnf_reclaim_tasklet_fn(struct tasklet_struct *t) 1109{ 1110 struct qtnf_pcie_topaz_state *ts = from_tasklet(ts, t, base.reclaim_tq); 1111 1112 qtnf_topaz_data_tx_reclaim(ts); 1113} 1114 1115static u64 qtnf_topaz_dma_mask_get(void) 1116{ 1117 return DMA_BIT_MASK(32); 1118} 1119 1120static int qtnf_pcie_topaz_probe(struct qtnf_bus *bus, 1121 unsigned int tx_bd_num, unsigned int rx_bd_num) 1122{ 1123 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus); 1124 struct pci_dev *pdev = ts->base.pdev; 1125 struct qtnf_shm_ipc_int ipc_int; 1126 unsigned long irqflags; 1127 int ret; 1128 1129 bus->bus_ops = &qtnf_pcie_topaz_bus_ops; 1130 INIT_WORK(&bus->fw_work, qtnf_topaz_fw_work_handler); 1131 ts->bda = ts->base.epmem_bar; 1132 1133 /* assign host msi irq before card init */ 1134 if (ts->base.msi_enabled) 1135 irqflags = IRQF_NOBALANCING; 1136 else 1137 irqflags = IRQF_NOBALANCING | IRQF_SHARED; 1138 1139 ret = devm_request_irq(&pdev->dev, pdev->irq, 1140 &qtnf_pcie_topaz_interrupt, 1141 irqflags, "qtnf_topaz_irq", (void *)bus); 1142 if (ret) { 1143 pr_err("failed to request pcie irq %d\n", pdev->irq); 1144 return ret; 1145 } 1146 1147 disable_irq(pdev->irq); 1148 1149 ret = qtnf_pre_init_ep(bus); 1150 if (ret) { 1151 pr_err("failed to init card\n"); 1152 return ret; 1153 } 1154 1155 ret = qtnf_pcie_topaz_init_xfer(ts, tx_bd_num, rx_bd_num); 1156 if (ret) { 1157 pr_err("PCIE xfer init failed\n"); 1158 return ret; 1159 } 1160 1161 tasklet_setup(&ts->base.reclaim_tq, qtnf_reclaim_tasklet_fn); 1162 netif_napi_add_weight(&bus->mux_dev, &bus->mux_napi, 1163 qtnf_topaz_rx_poll, 10); 1164 1165 ipc_int.fn = qtnf_topaz_ipc_gen_ep_int; 1166 ipc_int.arg = ts; 1167 qtnf_pcie_init_shm_ipc(&ts->base, &ts->bda->bda_shm_reg1, 1168 &ts->bda->bda_shm_reg2, &ipc_int); 1169 1170 return 0; 1171} 1172 1173static void qtnf_pcie_topaz_remove(struct qtnf_bus *bus) 1174{ 1175 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus); 1176 1177 qtnf_topaz_reset_ep(ts); 1178 qtnf_topaz_free_xfer_buffers(ts); 1179} 1180 1181#ifdef CONFIG_PM_SLEEP 1182static int qtnf_pcie_topaz_suspend(struct qtnf_bus *bus) 1183{ 1184 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus); 1185 struct pci_dev *pdev = ts->base.pdev; 1186 1187 writel((u32 __force)PCI_D3hot, ts->ep_pmstate); 1188 dma_wmb(); 1189 writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_PM_EP_IRQ), 1190 TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar)); 1191 1192 pci_save_state(pdev); 1193 pci_enable_wake(pdev, PCI_D3hot, 1); 1194 pci_set_power_state(pdev, PCI_D3hot); 1195 1196 return 0; 1197} 1198 1199static int qtnf_pcie_topaz_resume(struct qtnf_bus *bus) 1200{ 1201 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus); 1202 struct pci_dev *pdev = ts->base.pdev; 1203 1204 pci_set_power_state(pdev, PCI_D0); 1205 pci_restore_state(pdev); 1206 pci_enable_wake(pdev, PCI_D0, 0); 1207 1208 writel((u32 __force)PCI_D0, ts->ep_pmstate); 1209 dma_wmb(); 1210 writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_PM_EP_IRQ), 1211 TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar)); 1212 1213 return 0; 1214} 1215#endif 1216 1217struct qtnf_bus *qtnf_pcie_topaz_alloc(struct pci_dev *pdev) 1218{ 1219 struct qtnf_bus *bus; 1220 struct qtnf_pcie_topaz_state *ts; 1221 1222 bus = devm_kzalloc(&pdev->dev, sizeof(*bus) + sizeof(*ts), GFP_KERNEL); 1223 if (!bus) 1224 return NULL; 1225 1226 ts = get_bus_priv(bus); 1227 ts->base.probe_cb = qtnf_pcie_topaz_probe; 1228 ts->base.remove_cb = qtnf_pcie_topaz_remove; 1229 ts->base.dma_mask_get_cb = qtnf_topaz_dma_mask_get; 1230#ifdef CONFIG_PM_SLEEP 1231 ts->base.resume_cb = qtnf_pcie_topaz_resume; 1232 ts->base.suspend_cb = qtnf_pcie_topaz_suspend; 1233#endif 1234 1235 return bus; 1236} 1237