1// SPDX-License-Identifier: GPL-2.0 2/* Copyright (c) 2019, Intel Corporation. */ 3 4#include <linux/bpf_trace.h> 5#include <net/xdp_sock_drv.h> 6#include <net/xdp.h> 7#include "ice.h" 8#include "ice_base.h" 9#include "ice_type.h" 10#include "ice_xsk.h" 11#include "ice_txrx.h" 12#include "ice_txrx_lib.h" 13#include "ice_lib.h" 14 15static struct xdp_buff **ice_xdp_buf(struct ice_rx_ring *rx_ring, u32 idx) 16{ 17 return &rx_ring->xdp_buf[idx]; 18} 19 20/** 21 * ice_qp_reset_stats - Resets all stats for rings of given index 22 * @vsi: VSI that contains rings of interest 23 * @q_idx: ring index in array 24 */ 25static void ice_qp_reset_stats(struct ice_vsi *vsi, u16 q_idx) 26{ 27 struct ice_vsi_stats *vsi_stat; 28 struct ice_pf *pf; 29 30 pf = vsi->back; 31 if (!pf->vsi_stats) 32 return; 33 34 vsi_stat = pf->vsi_stats[vsi->idx]; 35 if (!vsi_stat) 36 return; 37 38 memset(&vsi_stat->rx_ring_stats[q_idx]->rx_stats, 0, 39 sizeof(vsi_stat->rx_ring_stats[q_idx]->rx_stats)); 40 memset(&vsi_stat->tx_ring_stats[q_idx]->stats, 0, 41 sizeof(vsi_stat->tx_ring_stats[q_idx]->stats)); 42 if (ice_is_xdp_ena_vsi(vsi)) 43 memset(&vsi->xdp_rings[q_idx]->ring_stats->stats, 0, 44 sizeof(vsi->xdp_rings[q_idx]->ring_stats->stats)); 45} 46 47/** 48 * ice_qp_clean_rings - Cleans all the rings of a given index 49 * @vsi: VSI that contains rings of interest 50 * @q_idx: ring index in array 51 */ 52static void ice_qp_clean_rings(struct ice_vsi *vsi, u16 q_idx) 53{ 54 ice_clean_tx_ring(vsi->tx_rings[q_idx]); 55 if (ice_is_xdp_ena_vsi(vsi)) { 56 synchronize_rcu(); 57 ice_clean_tx_ring(vsi->xdp_rings[q_idx]); 58 } 59 ice_clean_rx_ring(vsi->rx_rings[q_idx]); 60} 61 62/** 63 * ice_qvec_toggle_napi - Enables/disables NAPI for a given q_vector 64 * @vsi: VSI that has netdev 65 * @q_vector: q_vector that has NAPI context 66 * @enable: true for enable, false for disable 67 */ 68static void 69ice_qvec_toggle_napi(struct ice_vsi *vsi, struct ice_q_vector *q_vector, 70 bool enable) 71{ 72 if (!vsi->netdev || !q_vector) 73 return; 74 75 if (enable) 76 napi_enable(&q_vector->napi); 77 else 78 napi_disable(&q_vector->napi); 79} 80 81/** 82 * ice_qvec_dis_irq - Mask off queue interrupt generation on given ring 83 * @vsi: the VSI that contains queue vector being un-configured 84 * @rx_ring: Rx ring that will have its IRQ disabled 85 * @q_vector: queue vector 86 */ 87static void 88ice_qvec_dis_irq(struct ice_vsi *vsi, struct ice_rx_ring *rx_ring, 89 struct ice_q_vector *q_vector) 90{ 91 struct ice_pf *pf = vsi->back; 92 struct ice_hw *hw = &pf->hw; 93 u16 reg; 94 u32 val; 95 96 /* QINT_TQCTL is being cleared in ice_vsi_stop_tx_ring, so handle 97 * here only QINT_RQCTL 98 */ 99 reg = rx_ring->reg_idx; 100 val = rd32(hw, QINT_RQCTL(reg)); 101 val &= ~QINT_RQCTL_CAUSE_ENA_M; 102 wr32(hw, QINT_RQCTL(reg), val); 103 104 if (q_vector) { 105 wr32(hw, GLINT_DYN_CTL(q_vector->reg_idx), 0); 106 ice_flush(hw); 107 synchronize_irq(q_vector->irq.virq); 108 } 109} 110 111/** 112 * ice_qvec_cfg_msix - Enable IRQ for given queue vector 113 * @vsi: the VSI that contains queue vector 114 * @q_vector: queue vector 115 */ 116static void 117ice_qvec_cfg_msix(struct ice_vsi *vsi, struct ice_q_vector *q_vector) 118{ 119 u16 reg_idx = q_vector->reg_idx; 120 struct ice_pf *pf = vsi->back; 121 struct ice_hw *hw = &pf->hw; 122 struct ice_tx_ring *tx_ring; 123 struct ice_rx_ring *rx_ring; 124 125 ice_cfg_itr(hw, q_vector); 126 127 ice_for_each_tx_ring(tx_ring, q_vector->tx) 128 ice_cfg_txq_interrupt(vsi, tx_ring->reg_idx, reg_idx, 129 q_vector->tx.itr_idx); 130 131 ice_for_each_rx_ring(rx_ring, q_vector->rx) 132 ice_cfg_rxq_interrupt(vsi, rx_ring->reg_idx, reg_idx, 133 q_vector->rx.itr_idx); 134 135 ice_flush(hw); 136} 137 138/** 139 * ice_qvec_ena_irq - Enable IRQ for given queue vector 140 * @vsi: the VSI that contains queue vector 141 * @q_vector: queue vector 142 */ 143static void ice_qvec_ena_irq(struct ice_vsi *vsi, struct ice_q_vector *q_vector) 144{ 145 struct ice_pf *pf = vsi->back; 146 struct ice_hw *hw = &pf->hw; 147 148 ice_irq_dynamic_ena(hw, vsi, q_vector); 149 150 ice_flush(hw); 151} 152 153/** 154 * ice_qp_dis - Disables a queue pair 155 * @vsi: VSI of interest 156 * @q_idx: ring index in array 157 * 158 * Returns 0 on success, negative on failure. 159 */ 160static int ice_qp_dis(struct ice_vsi *vsi, u16 q_idx) 161{ 162 struct ice_txq_meta txq_meta = { }; 163 struct ice_q_vector *q_vector; 164 struct ice_tx_ring *tx_ring; 165 struct ice_rx_ring *rx_ring; 166 int timeout = 50; 167 int err; 168 169 if (q_idx >= vsi->num_rxq || q_idx >= vsi->num_txq) 170 return -EINVAL; 171 172 tx_ring = vsi->tx_rings[q_idx]; 173 rx_ring = vsi->rx_rings[q_idx]; 174 q_vector = rx_ring->q_vector; 175 176 while (test_and_set_bit(ICE_CFG_BUSY, vsi->state)) { 177 timeout--; 178 if (!timeout) 179 return -EBUSY; 180 usleep_range(1000, 2000); 181 } 182 183 ice_qvec_dis_irq(vsi, rx_ring, q_vector); 184 ice_qvec_toggle_napi(vsi, q_vector, false); 185 186 netif_tx_stop_queue(netdev_get_tx_queue(vsi->netdev, q_idx)); 187 188 ice_fill_txq_meta(vsi, tx_ring, &txq_meta); 189 err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, tx_ring, &txq_meta); 190 if (err) 191 return err; 192 if (ice_is_xdp_ena_vsi(vsi)) { 193 struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx]; 194 195 memset(&txq_meta, 0, sizeof(txq_meta)); 196 ice_fill_txq_meta(vsi, xdp_ring, &txq_meta); 197 err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, xdp_ring, 198 &txq_meta); 199 if (err) 200 return err; 201 } 202 err = ice_vsi_ctrl_one_rx_ring(vsi, false, q_idx, true); 203 if (err) 204 return err; 205 206 ice_qp_clean_rings(vsi, q_idx); 207 ice_qp_reset_stats(vsi, q_idx); 208 209 return 0; 210} 211 212/** 213 * ice_qp_ena - Enables a queue pair 214 * @vsi: VSI of interest 215 * @q_idx: ring index in array 216 * 217 * Returns 0 on success, negative on failure. 218 */ 219static int ice_qp_ena(struct ice_vsi *vsi, u16 q_idx) 220{ 221 struct ice_q_vector *q_vector; 222 int err; 223 224 err = ice_vsi_cfg_single_txq(vsi, vsi->tx_rings, q_idx); 225 if (err) 226 return err; 227 228 if (ice_is_xdp_ena_vsi(vsi)) { 229 struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx]; 230 231 err = ice_vsi_cfg_single_txq(vsi, vsi->xdp_rings, q_idx); 232 if (err) 233 return err; 234 ice_set_ring_xdp(xdp_ring); 235 ice_tx_xsk_pool(vsi, q_idx); 236 } 237 238 err = ice_vsi_cfg_single_rxq(vsi, q_idx); 239 if (err) 240 return err; 241 242 q_vector = vsi->rx_rings[q_idx]->q_vector; 243 ice_qvec_cfg_msix(vsi, q_vector); 244 245 err = ice_vsi_ctrl_one_rx_ring(vsi, true, q_idx, true); 246 if (err) 247 return err; 248 249 ice_qvec_toggle_napi(vsi, q_vector, true); 250 ice_qvec_ena_irq(vsi, q_vector); 251 252 netif_tx_start_queue(netdev_get_tx_queue(vsi->netdev, q_idx)); 253 clear_bit(ICE_CFG_BUSY, vsi->state); 254 255 return 0; 256} 257 258/** 259 * ice_xsk_pool_disable - disable a buffer pool region 260 * @vsi: Current VSI 261 * @qid: queue ID 262 * 263 * Returns 0 on success, negative on failure 264 */ 265static int ice_xsk_pool_disable(struct ice_vsi *vsi, u16 qid) 266{ 267 struct xsk_buff_pool *pool = xsk_get_pool_from_qid(vsi->netdev, qid); 268 269 if (!pool) 270 return -EINVAL; 271 272 clear_bit(qid, vsi->af_xdp_zc_qps); 273 xsk_pool_dma_unmap(pool, ICE_RX_DMA_ATTR); 274 275 return 0; 276} 277 278/** 279 * ice_xsk_pool_enable - enable a buffer pool region 280 * @vsi: Current VSI 281 * @pool: pointer to a requested buffer pool region 282 * @qid: queue ID 283 * 284 * Returns 0 on success, negative on failure 285 */ 286static int 287ice_xsk_pool_enable(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid) 288{ 289 int err; 290 291 if (vsi->type != ICE_VSI_PF) 292 return -EINVAL; 293 294 if (qid >= vsi->netdev->real_num_rx_queues || 295 qid >= vsi->netdev->real_num_tx_queues) 296 return -EINVAL; 297 298 err = xsk_pool_dma_map(pool, ice_pf_to_dev(vsi->back), 299 ICE_RX_DMA_ATTR); 300 if (err) 301 return err; 302 303 set_bit(qid, vsi->af_xdp_zc_qps); 304 305 return 0; 306} 307 308/** 309 * ice_realloc_rx_xdp_bufs - reallocate for either XSK or normal buffer 310 * @rx_ring: Rx ring 311 * @pool_present: is pool for XSK present 312 * 313 * Try allocating memory and return ENOMEM, if failed to allocate. 314 * If allocation was successful, substitute buffer with allocated one. 315 * Returns 0 on success, negative on failure 316 */ 317static int 318ice_realloc_rx_xdp_bufs(struct ice_rx_ring *rx_ring, bool pool_present) 319{ 320 size_t elem_size = pool_present ? sizeof(*rx_ring->xdp_buf) : 321 sizeof(*rx_ring->rx_buf); 322 void *sw_ring = kcalloc(rx_ring->count, elem_size, GFP_KERNEL); 323 324 if (!sw_ring) 325 return -ENOMEM; 326 327 if (pool_present) { 328 kfree(rx_ring->rx_buf); 329 rx_ring->rx_buf = NULL; 330 rx_ring->xdp_buf = sw_ring; 331 } else { 332 kfree(rx_ring->xdp_buf); 333 rx_ring->xdp_buf = NULL; 334 rx_ring->rx_buf = sw_ring; 335 } 336 337 return 0; 338} 339 340/** 341 * ice_realloc_zc_buf - reallocate XDP ZC queue pairs 342 * @vsi: Current VSI 343 * @zc: is zero copy set 344 * 345 * Reallocate buffer for rx_rings that might be used by XSK. 346 * XDP requires more memory, than rx_buf provides. 347 * Returns 0 on success, negative on failure 348 */ 349int ice_realloc_zc_buf(struct ice_vsi *vsi, bool zc) 350{ 351 struct ice_rx_ring *rx_ring; 352 unsigned long q; 353 354 for_each_set_bit(q, vsi->af_xdp_zc_qps, 355 max_t(int, vsi->alloc_txq, vsi->alloc_rxq)) { 356 rx_ring = vsi->rx_rings[q]; 357 if (ice_realloc_rx_xdp_bufs(rx_ring, zc)) 358 return -ENOMEM; 359 } 360 361 return 0; 362} 363 364/** 365 * ice_xsk_pool_setup - enable/disable a buffer pool region depending on its state 366 * @vsi: Current VSI 367 * @pool: buffer pool to enable/associate to a ring, NULL to disable 368 * @qid: queue ID 369 * 370 * Returns 0 on success, negative on failure 371 */ 372int ice_xsk_pool_setup(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid) 373{ 374 bool if_running, pool_present = !!pool; 375 int ret = 0, pool_failure = 0; 376 377 if (qid >= vsi->num_rxq || qid >= vsi->num_txq) { 378 netdev_err(vsi->netdev, "Please use queue id in scope of combined queues count\n"); 379 pool_failure = -EINVAL; 380 goto failure; 381 } 382 383 if_running = netif_running(vsi->netdev) && ice_is_xdp_ena_vsi(vsi); 384 385 if (if_running) { 386 struct ice_rx_ring *rx_ring = vsi->rx_rings[qid]; 387 388 ret = ice_qp_dis(vsi, qid); 389 if (ret) { 390 netdev_err(vsi->netdev, "ice_qp_dis error = %d\n", ret); 391 goto xsk_pool_if_up; 392 } 393 394 ret = ice_realloc_rx_xdp_bufs(rx_ring, pool_present); 395 if (ret) 396 goto xsk_pool_if_up; 397 } 398 399 pool_failure = pool_present ? ice_xsk_pool_enable(vsi, pool, qid) : 400 ice_xsk_pool_disable(vsi, qid); 401 402xsk_pool_if_up: 403 if (if_running) { 404 ret = ice_qp_ena(vsi, qid); 405 if (!ret && pool_present) 406 napi_schedule(&vsi->rx_rings[qid]->xdp_ring->q_vector->napi); 407 else if (ret) 408 netdev_err(vsi->netdev, "ice_qp_ena error = %d\n", ret); 409 } 410 411failure: 412 if (pool_failure) { 413 netdev_err(vsi->netdev, "Could not %sable buffer pool, error = %d\n", 414 pool_present ? "en" : "dis", pool_failure); 415 return pool_failure; 416 } 417 418 return ret; 419} 420 421/** 422 * ice_fill_rx_descs - pick buffers from XSK buffer pool and use it 423 * @pool: XSK Buffer pool to pull the buffers from 424 * @xdp: SW ring of xdp_buff that will hold the buffers 425 * @rx_desc: Pointer to Rx descriptors that will be filled 426 * @count: The number of buffers to allocate 427 * 428 * This function allocates a number of Rx buffers from the fill ring 429 * or the internal recycle mechanism and places them on the Rx ring. 430 * 431 * Note that ring wrap should be handled by caller of this function. 432 * 433 * Returns the amount of allocated Rx descriptors 434 */ 435static u16 ice_fill_rx_descs(struct xsk_buff_pool *pool, struct xdp_buff **xdp, 436 union ice_32b_rx_flex_desc *rx_desc, u16 count) 437{ 438 dma_addr_t dma; 439 u16 buffs; 440 int i; 441 442 buffs = xsk_buff_alloc_batch(pool, xdp, count); 443 for (i = 0; i < buffs; i++) { 444 dma = xsk_buff_xdp_get_dma(*xdp); 445 rx_desc->read.pkt_addr = cpu_to_le64(dma); 446 rx_desc->wb.status_error0 = 0; 447 448 /* Put private info that changes on a per-packet basis 449 * into xdp_buff_xsk->cb. 450 */ 451 ice_xdp_meta_set_desc(*xdp, rx_desc); 452 453 rx_desc++; 454 xdp++; 455 } 456 457 return buffs; 458} 459 460/** 461 * __ice_alloc_rx_bufs_zc - allocate a number of Rx buffers 462 * @rx_ring: Rx ring 463 * @count: The number of buffers to allocate 464 * 465 * Place the @count of descriptors onto Rx ring. Handle the ring wrap 466 * for case where space from next_to_use up to the end of ring is less 467 * than @count. Finally do a tail bump. 468 * 469 * Returns true if all allocations were successful, false if any fail. 470 */ 471static bool __ice_alloc_rx_bufs_zc(struct ice_rx_ring *rx_ring, u16 count) 472{ 473 u32 nb_buffs_extra = 0, nb_buffs = 0; 474 union ice_32b_rx_flex_desc *rx_desc; 475 u16 ntu = rx_ring->next_to_use; 476 u16 total_count = count; 477 struct xdp_buff **xdp; 478 479 rx_desc = ICE_RX_DESC(rx_ring, ntu); 480 xdp = ice_xdp_buf(rx_ring, ntu); 481 482 if (ntu + count >= rx_ring->count) { 483 nb_buffs_extra = ice_fill_rx_descs(rx_ring->xsk_pool, xdp, 484 rx_desc, 485 rx_ring->count - ntu); 486 if (nb_buffs_extra != rx_ring->count - ntu) { 487 ntu += nb_buffs_extra; 488 goto exit; 489 } 490 rx_desc = ICE_RX_DESC(rx_ring, 0); 491 xdp = ice_xdp_buf(rx_ring, 0); 492 ntu = 0; 493 count -= nb_buffs_extra; 494 ice_release_rx_desc(rx_ring, 0); 495 } 496 497 nb_buffs = ice_fill_rx_descs(rx_ring->xsk_pool, xdp, rx_desc, count); 498 499 ntu += nb_buffs; 500 if (ntu == rx_ring->count) 501 ntu = 0; 502 503exit: 504 if (rx_ring->next_to_use != ntu) 505 ice_release_rx_desc(rx_ring, ntu); 506 507 return total_count == (nb_buffs_extra + nb_buffs); 508} 509 510/** 511 * ice_alloc_rx_bufs_zc - allocate a number of Rx buffers 512 * @rx_ring: Rx ring 513 * @count: The number of buffers to allocate 514 * 515 * Wrapper for internal allocation routine; figure out how many tail 516 * bumps should take place based on the given threshold 517 * 518 * Returns true if all calls to internal alloc routine succeeded 519 */ 520bool ice_alloc_rx_bufs_zc(struct ice_rx_ring *rx_ring, u16 count) 521{ 522 u16 rx_thresh = ICE_RING_QUARTER(rx_ring); 523 u16 leftover, i, tail_bumps; 524 525 tail_bumps = count / rx_thresh; 526 leftover = count - (tail_bumps * rx_thresh); 527 528 for (i = 0; i < tail_bumps; i++) 529 if (!__ice_alloc_rx_bufs_zc(rx_ring, rx_thresh)) 530 return false; 531 return __ice_alloc_rx_bufs_zc(rx_ring, leftover); 532} 533 534/** 535 * ice_construct_skb_zc - Create an sk_buff from zero-copy buffer 536 * @rx_ring: Rx ring 537 * @xdp: Pointer to XDP buffer 538 * 539 * This function allocates a new skb from a zero-copy Rx buffer. 540 * 541 * Returns the skb on success, NULL on failure. 542 */ 543static struct sk_buff * 544ice_construct_skb_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp) 545{ 546 unsigned int totalsize = xdp->data_end - xdp->data_meta; 547 unsigned int metasize = xdp->data - xdp->data_meta; 548 struct skb_shared_info *sinfo = NULL; 549 struct sk_buff *skb; 550 u32 nr_frags = 0; 551 552 if (unlikely(xdp_buff_has_frags(xdp))) { 553 sinfo = xdp_get_shared_info_from_buff(xdp); 554 nr_frags = sinfo->nr_frags; 555 } 556 net_prefetch(xdp->data_meta); 557 558 skb = __napi_alloc_skb(&rx_ring->q_vector->napi, totalsize, 559 GFP_ATOMIC | __GFP_NOWARN); 560 if (unlikely(!skb)) 561 return NULL; 562 563 memcpy(__skb_put(skb, totalsize), xdp->data_meta, 564 ALIGN(totalsize, sizeof(long))); 565 566 if (metasize) { 567 skb_metadata_set(skb, metasize); 568 __skb_pull(skb, metasize); 569 } 570 571 if (likely(!xdp_buff_has_frags(xdp))) 572 goto out; 573 574 for (int i = 0; i < nr_frags; i++) { 575 struct skb_shared_info *skinfo = skb_shinfo(skb); 576 skb_frag_t *frag = &sinfo->frags[i]; 577 struct page *page; 578 void *addr; 579 580 page = dev_alloc_page(); 581 if (!page) { 582 dev_kfree_skb(skb); 583 return NULL; 584 } 585 addr = page_to_virt(page); 586 587 memcpy(addr, skb_frag_page(frag), skb_frag_size(frag)); 588 589 __skb_fill_page_desc_noacc(skinfo, skinfo->nr_frags++, 590 addr, 0, skb_frag_size(frag)); 591 } 592 593out: 594 xsk_buff_free(xdp); 595 return skb; 596} 597 598/** 599 * ice_clean_xdp_irq_zc - produce AF_XDP descriptors to CQ 600 * @xdp_ring: XDP Tx ring 601 */ 602static u32 ice_clean_xdp_irq_zc(struct ice_tx_ring *xdp_ring) 603{ 604 u16 ntc = xdp_ring->next_to_clean; 605 struct ice_tx_desc *tx_desc; 606 u16 cnt = xdp_ring->count; 607 struct ice_tx_buf *tx_buf; 608 u16 completed_frames = 0; 609 u16 xsk_frames = 0; 610 u16 last_rs; 611 int i; 612 613 last_rs = xdp_ring->next_to_use ? xdp_ring->next_to_use - 1 : cnt - 1; 614 tx_desc = ICE_TX_DESC(xdp_ring, last_rs); 615 if (tx_desc->cmd_type_offset_bsz & 616 cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE)) { 617 if (last_rs >= ntc) 618 completed_frames = last_rs - ntc + 1; 619 else 620 completed_frames = last_rs + cnt - ntc + 1; 621 } 622 623 if (!completed_frames) 624 return 0; 625 626 if (likely(!xdp_ring->xdp_tx_active)) { 627 xsk_frames = completed_frames; 628 goto skip; 629 } 630 631 ntc = xdp_ring->next_to_clean; 632 for (i = 0; i < completed_frames; i++) { 633 tx_buf = &xdp_ring->tx_buf[ntc]; 634 635 if (tx_buf->type == ICE_TX_BUF_XSK_TX) { 636 tx_buf->type = ICE_TX_BUF_EMPTY; 637 xsk_buff_free(tx_buf->xdp); 638 xdp_ring->xdp_tx_active--; 639 } else { 640 xsk_frames++; 641 } 642 643 ntc++; 644 if (ntc >= xdp_ring->count) 645 ntc = 0; 646 } 647skip: 648 tx_desc->cmd_type_offset_bsz = 0; 649 xdp_ring->next_to_clean += completed_frames; 650 if (xdp_ring->next_to_clean >= cnt) 651 xdp_ring->next_to_clean -= cnt; 652 if (xsk_frames) 653 xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames); 654 655 return completed_frames; 656} 657 658/** 659 * ice_xmit_xdp_tx_zc - AF_XDP ZC handler for XDP_TX 660 * @xdp: XDP buffer to xmit 661 * @xdp_ring: XDP ring to produce descriptor onto 662 * 663 * note that this function works directly on xdp_buff, no need to convert 664 * it to xdp_frame. xdp_buff pointer is stored to ice_tx_buf so that cleaning 665 * side will be able to xsk_buff_free() it. 666 * 667 * Returns ICE_XDP_TX for successfully produced desc, ICE_XDP_CONSUMED if there 668 * was not enough space on XDP ring 669 */ 670static int ice_xmit_xdp_tx_zc(struct xdp_buff *xdp, 671 struct ice_tx_ring *xdp_ring) 672{ 673 struct skb_shared_info *sinfo = NULL; 674 u32 size = xdp->data_end - xdp->data; 675 u32 ntu = xdp_ring->next_to_use; 676 struct ice_tx_desc *tx_desc; 677 struct ice_tx_buf *tx_buf; 678 struct xdp_buff *head; 679 u32 nr_frags = 0; 680 u32 free_space; 681 u32 frag = 0; 682 683 free_space = ICE_DESC_UNUSED(xdp_ring); 684 if (free_space < ICE_RING_QUARTER(xdp_ring)) 685 free_space += ice_clean_xdp_irq_zc(xdp_ring); 686 687 if (unlikely(!free_space)) 688 goto busy; 689 690 if (unlikely(xdp_buff_has_frags(xdp))) { 691 sinfo = xdp_get_shared_info_from_buff(xdp); 692 nr_frags = sinfo->nr_frags; 693 if (free_space < nr_frags + 1) 694 goto busy; 695 } 696 697 tx_desc = ICE_TX_DESC(xdp_ring, ntu); 698 tx_buf = &xdp_ring->tx_buf[ntu]; 699 head = xdp; 700 701 for (;;) { 702 dma_addr_t dma; 703 704 dma = xsk_buff_xdp_get_dma(xdp); 705 xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma, size); 706 707 tx_buf->xdp = xdp; 708 tx_buf->type = ICE_TX_BUF_XSK_TX; 709 tx_desc->buf_addr = cpu_to_le64(dma); 710 tx_desc->cmd_type_offset_bsz = ice_build_ctob(0, 0, size, 0); 711 /* account for each xdp_buff from xsk_buff_pool */ 712 xdp_ring->xdp_tx_active++; 713 714 if (++ntu == xdp_ring->count) 715 ntu = 0; 716 717 if (frag == nr_frags) 718 break; 719 720 tx_desc = ICE_TX_DESC(xdp_ring, ntu); 721 tx_buf = &xdp_ring->tx_buf[ntu]; 722 723 xdp = xsk_buff_get_frag(head); 724 size = skb_frag_size(&sinfo->frags[frag]); 725 frag++; 726 } 727 728 xdp_ring->next_to_use = ntu; 729 /* update last descriptor from a frame with EOP */ 730 tx_desc->cmd_type_offset_bsz |= 731 cpu_to_le64(ICE_TX_DESC_CMD_EOP << ICE_TXD_QW1_CMD_S); 732 733 return ICE_XDP_TX; 734 735busy: 736 xdp_ring->ring_stats->tx_stats.tx_busy++; 737 738 return ICE_XDP_CONSUMED; 739} 740 741/** 742 * ice_run_xdp_zc - Executes an XDP program in zero-copy path 743 * @rx_ring: Rx ring 744 * @xdp: xdp_buff used as input to the XDP program 745 * @xdp_prog: XDP program to run 746 * @xdp_ring: ring to be used for XDP_TX action 747 * 748 * Returns any of ICE_XDP_{PASS, CONSUMED, TX, REDIR} 749 */ 750static int 751ice_run_xdp_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp, 752 struct bpf_prog *xdp_prog, struct ice_tx_ring *xdp_ring) 753{ 754 int err, result = ICE_XDP_PASS; 755 u32 act; 756 757 act = bpf_prog_run_xdp(xdp_prog, xdp); 758 759 if (likely(act == XDP_REDIRECT)) { 760 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); 761 if (!err) 762 return ICE_XDP_REDIR; 763 if (xsk_uses_need_wakeup(rx_ring->xsk_pool) && err == -ENOBUFS) 764 result = ICE_XDP_EXIT; 765 else 766 result = ICE_XDP_CONSUMED; 767 goto out_failure; 768 } 769 770 switch (act) { 771 case XDP_PASS: 772 break; 773 case XDP_TX: 774 result = ice_xmit_xdp_tx_zc(xdp, xdp_ring); 775 if (result == ICE_XDP_CONSUMED) 776 goto out_failure; 777 break; 778 case XDP_DROP: 779 result = ICE_XDP_CONSUMED; 780 break; 781 default: 782 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); 783 fallthrough; 784 case XDP_ABORTED: 785 result = ICE_XDP_CONSUMED; 786out_failure: 787 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); 788 break; 789 } 790 791 return result; 792} 793 794static int 795ice_add_xsk_frag(struct ice_rx_ring *rx_ring, struct xdp_buff *first, 796 struct xdp_buff *xdp, const unsigned int size) 797{ 798 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(first); 799 800 if (!size) 801 return 0; 802 803 if (!xdp_buff_has_frags(first)) { 804 sinfo->nr_frags = 0; 805 sinfo->xdp_frags_size = 0; 806 xdp_buff_set_frags_flag(first); 807 } 808 809 if (unlikely(sinfo->nr_frags == MAX_SKB_FRAGS)) { 810 xsk_buff_free(first); 811 return -ENOMEM; 812 } 813 814 __skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++, 815 virt_to_page(xdp->data_hard_start), 816 XDP_PACKET_HEADROOM, size); 817 sinfo->xdp_frags_size += size; 818 xsk_buff_add_frag(xdp); 819 820 return 0; 821} 822 823/** 824 * ice_clean_rx_irq_zc - consumes packets from the hardware ring 825 * @rx_ring: AF_XDP Rx ring 826 * @budget: NAPI budget 827 * 828 * Returns number of processed packets on success, remaining budget on failure. 829 */ 830int ice_clean_rx_irq_zc(struct ice_rx_ring *rx_ring, int budget) 831{ 832 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 833 struct xsk_buff_pool *xsk_pool = rx_ring->xsk_pool; 834 u32 ntc = rx_ring->next_to_clean; 835 u32 ntu = rx_ring->next_to_use; 836 struct xdp_buff *first = NULL; 837 struct ice_tx_ring *xdp_ring; 838 unsigned int xdp_xmit = 0; 839 struct bpf_prog *xdp_prog; 840 u32 cnt = rx_ring->count; 841 bool failure = false; 842 int entries_to_alloc; 843 844 /* ZC patch is enabled only when XDP program is set, 845 * so here it can not be NULL 846 */ 847 xdp_prog = READ_ONCE(rx_ring->xdp_prog); 848 xdp_ring = rx_ring->xdp_ring; 849 850 if (ntc != rx_ring->first_desc) 851 first = *ice_xdp_buf(rx_ring, rx_ring->first_desc); 852 853 while (likely(total_rx_packets < (unsigned int)budget)) { 854 union ice_32b_rx_flex_desc *rx_desc; 855 unsigned int size, xdp_res = 0; 856 struct xdp_buff *xdp; 857 struct sk_buff *skb; 858 u16 stat_err_bits; 859 u16 vlan_tci; 860 861 rx_desc = ICE_RX_DESC(rx_ring, ntc); 862 863 stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S); 864 if (!ice_test_staterr(rx_desc->wb.status_error0, stat_err_bits)) 865 break; 866 867 /* This memory barrier is needed to keep us from reading 868 * any other fields out of the rx_desc until we have 869 * verified the descriptor has been written back. 870 */ 871 dma_rmb(); 872 873 if (unlikely(ntc == ntu)) 874 break; 875 876 xdp = *ice_xdp_buf(rx_ring, ntc); 877 878 size = le16_to_cpu(rx_desc->wb.pkt_len) & 879 ICE_RX_FLX_DESC_PKT_LEN_M; 880 881 xsk_buff_set_size(xdp, size); 882 xsk_buff_dma_sync_for_cpu(xdp, xsk_pool); 883 884 if (!first) { 885 first = xdp; 886 } else if (ice_add_xsk_frag(rx_ring, first, xdp, size)) { 887 break; 888 } 889 890 if (++ntc == cnt) 891 ntc = 0; 892 893 if (ice_is_non_eop(rx_ring, rx_desc)) 894 continue; 895 896 xdp_res = ice_run_xdp_zc(rx_ring, first, xdp_prog, xdp_ring); 897 if (likely(xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR))) { 898 xdp_xmit |= xdp_res; 899 } else if (xdp_res == ICE_XDP_EXIT) { 900 failure = true; 901 first = NULL; 902 rx_ring->first_desc = ntc; 903 break; 904 } else if (xdp_res == ICE_XDP_CONSUMED) { 905 xsk_buff_free(first); 906 } else if (xdp_res == ICE_XDP_PASS) { 907 goto construct_skb; 908 } 909 910 total_rx_bytes += xdp_get_buff_len(first); 911 total_rx_packets++; 912 913 first = NULL; 914 rx_ring->first_desc = ntc; 915 continue; 916 917construct_skb: 918 /* XDP_PASS path */ 919 skb = ice_construct_skb_zc(rx_ring, first); 920 if (!skb) { 921 rx_ring->ring_stats->rx_stats.alloc_buf_failed++; 922 break; 923 } 924 925 first = NULL; 926 rx_ring->first_desc = ntc; 927 928 if (eth_skb_pad(skb)) { 929 skb = NULL; 930 continue; 931 } 932 933 total_rx_bytes += skb->len; 934 total_rx_packets++; 935 936 vlan_tci = ice_get_vlan_tci(rx_desc); 937 938 ice_process_skb_fields(rx_ring, rx_desc, skb); 939 ice_receive_skb(rx_ring, skb, vlan_tci); 940 } 941 942 rx_ring->next_to_clean = ntc; 943 entries_to_alloc = ICE_RX_DESC_UNUSED(rx_ring); 944 if (entries_to_alloc > ICE_RING_QUARTER(rx_ring)) 945 failure |= !ice_alloc_rx_bufs_zc(rx_ring, entries_to_alloc); 946 947 ice_finalize_xdp_rx(xdp_ring, xdp_xmit, 0); 948 ice_update_rx_ring_stats(rx_ring, total_rx_packets, total_rx_bytes); 949 950 if (xsk_uses_need_wakeup(xsk_pool)) { 951 /* ntu could have changed when allocating entries above, so 952 * use rx_ring value instead of stack based one 953 */ 954 if (failure || ntc == rx_ring->next_to_use) 955 xsk_set_rx_need_wakeup(xsk_pool); 956 else 957 xsk_clear_rx_need_wakeup(xsk_pool); 958 959 return (int)total_rx_packets; 960 } 961 962 return failure ? budget : (int)total_rx_packets; 963} 964 965/** 966 * ice_xmit_pkt - produce a single HW Tx descriptor out of AF_XDP descriptor 967 * @xdp_ring: XDP ring to produce the HW Tx descriptor on 968 * @desc: AF_XDP descriptor to pull the DMA address and length from 969 * @total_bytes: bytes accumulator that will be used for stats update 970 */ 971static void ice_xmit_pkt(struct ice_tx_ring *xdp_ring, struct xdp_desc *desc, 972 unsigned int *total_bytes) 973{ 974 struct ice_tx_desc *tx_desc; 975 dma_addr_t dma; 976 977 dma = xsk_buff_raw_get_dma(xdp_ring->xsk_pool, desc->addr); 978 xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma, desc->len); 979 980 tx_desc = ICE_TX_DESC(xdp_ring, xdp_ring->next_to_use++); 981 tx_desc->buf_addr = cpu_to_le64(dma); 982 tx_desc->cmd_type_offset_bsz = ice_build_ctob(xsk_is_eop_desc(desc), 983 0, desc->len, 0); 984 985 *total_bytes += desc->len; 986} 987 988/** 989 * ice_xmit_pkt_batch - produce a batch of HW Tx descriptors out of AF_XDP descriptors 990 * @xdp_ring: XDP ring to produce the HW Tx descriptors on 991 * @descs: AF_XDP descriptors to pull the DMA addresses and lengths from 992 * @total_bytes: bytes accumulator that will be used for stats update 993 */ 994static void ice_xmit_pkt_batch(struct ice_tx_ring *xdp_ring, struct xdp_desc *descs, 995 unsigned int *total_bytes) 996{ 997 u16 ntu = xdp_ring->next_to_use; 998 struct ice_tx_desc *tx_desc; 999 u32 i; 1000 1001 loop_unrolled_for(i = 0; i < PKTS_PER_BATCH; i++) { 1002 dma_addr_t dma; 1003 1004 dma = xsk_buff_raw_get_dma(xdp_ring->xsk_pool, descs[i].addr); 1005 xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma, descs[i].len); 1006 1007 tx_desc = ICE_TX_DESC(xdp_ring, ntu++); 1008 tx_desc->buf_addr = cpu_to_le64(dma); 1009 tx_desc->cmd_type_offset_bsz = ice_build_ctob(xsk_is_eop_desc(&descs[i]), 1010 0, descs[i].len, 0); 1011 1012 *total_bytes += descs[i].len; 1013 } 1014 1015 xdp_ring->next_to_use = ntu; 1016} 1017 1018/** 1019 * ice_fill_tx_hw_ring - produce the number of Tx descriptors onto ring 1020 * @xdp_ring: XDP ring to produce the HW Tx descriptors on 1021 * @descs: AF_XDP descriptors to pull the DMA addresses and lengths from 1022 * @nb_pkts: count of packets to be send 1023 * @total_bytes: bytes accumulator that will be used for stats update 1024 */ 1025static void ice_fill_tx_hw_ring(struct ice_tx_ring *xdp_ring, struct xdp_desc *descs, 1026 u32 nb_pkts, unsigned int *total_bytes) 1027{ 1028 u32 batched, leftover, i; 1029 1030 batched = ALIGN_DOWN(nb_pkts, PKTS_PER_BATCH); 1031 leftover = nb_pkts & (PKTS_PER_BATCH - 1); 1032 for (i = 0; i < batched; i += PKTS_PER_BATCH) 1033 ice_xmit_pkt_batch(xdp_ring, &descs[i], total_bytes); 1034 for (; i < batched + leftover; i++) 1035 ice_xmit_pkt(xdp_ring, &descs[i], total_bytes); 1036} 1037 1038/** 1039 * ice_xmit_zc - take entries from XSK Tx ring and place them onto HW Tx ring 1040 * @xdp_ring: XDP ring to produce the HW Tx descriptors on 1041 * 1042 * Returns true if there is no more work that needs to be done, false otherwise 1043 */ 1044bool ice_xmit_zc(struct ice_tx_ring *xdp_ring) 1045{ 1046 struct xdp_desc *descs = xdp_ring->xsk_pool->tx_descs; 1047 u32 nb_pkts, nb_processed = 0; 1048 unsigned int total_bytes = 0; 1049 int budget; 1050 1051 ice_clean_xdp_irq_zc(xdp_ring); 1052 1053 budget = ICE_DESC_UNUSED(xdp_ring); 1054 budget = min_t(u16, budget, ICE_RING_QUARTER(xdp_ring)); 1055 1056 nb_pkts = xsk_tx_peek_release_desc_batch(xdp_ring->xsk_pool, budget); 1057 if (!nb_pkts) 1058 return true; 1059 1060 if (xdp_ring->next_to_use + nb_pkts >= xdp_ring->count) { 1061 nb_processed = xdp_ring->count - xdp_ring->next_to_use; 1062 ice_fill_tx_hw_ring(xdp_ring, descs, nb_processed, &total_bytes); 1063 xdp_ring->next_to_use = 0; 1064 } 1065 1066 ice_fill_tx_hw_ring(xdp_ring, &descs[nb_processed], nb_pkts - nb_processed, 1067 &total_bytes); 1068 1069 ice_set_rs_bit(xdp_ring); 1070 ice_xdp_ring_update_tail(xdp_ring); 1071 ice_update_tx_ring_stats(xdp_ring, nb_pkts, total_bytes); 1072 1073 if (xsk_uses_need_wakeup(xdp_ring->xsk_pool)) 1074 xsk_set_tx_need_wakeup(xdp_ring->xsk_pool); 1075 1076 return nb_pkts < budget; 1077} 1078 1079/** 1080 * ice_xsk_wakeup - Implements ndo_xsk_wakeup 1081 * @netdev: net_device 1082 * @queue_id: queue to wake up 1083 * @flags: ignored in our case, since we have Rx and Tx in the same NAPI 1084 * 1085 * Returns negative on error, zero otherwise. 1086 */ 1087int 1088ice_xsk_wakeup(struct net_device *netdev, u32 queue_id, 1089 u32 __always_unused flags) 1090{ 1091 struct ice_netdev_priv *np = netdev_priv(netdev); 1092 struct ice_q_vector *q_vector; 1093 struct ice_vsi *vsi = np->vsi; 1094 struct ice_tx_ring *ring; 1095 1096 if (test_bit(ICE_VSI_DOWN, vsi->state)) 1097 return -ENETDOWN; 1098 1099 if (!ice_is_xdp_ena_vsi(vsi)) 1100 return -EINVAL; 1101 1102 if (queue_id >= vsi->num_txq || queue_id >= vsi->num_rxq) 1103 return -EINVAL; 1104 1105 ring = vsi->rx_rings[queue_id]->xdp_ring; 1106 1107 if (!ring->xsk_pool) 1108 return -EINVAL; 1109 1110 /* The idea here is that if NAPI is running, mark a miss, so 1111 * it will run again. If not, trigger an interrupt and 1112 * schedule the NAPI from interrupt context. If NAPI would be 1113 * scheduled here, the interrupt affinity would not be 1114 * honored. 1115 */ 1116 q_vector = ring->q_vector; 1117 if (!napi_if_scheduled_mark_missed(&q_vector->napi)) 1118 ice_trigger_sw_intr(&vsi->back->hw, q_vector); 1119 1120 return 0; 1121} 1122 1123/** 1124 * ice_xsk_any_rx_ring_ena - Checks if Rx rings have AF_XDP buff pool attached 1125 * @vsi: VSI to be checked 1126 * 1127 * Returns true if any of the Rx rings has an AF_XDP buff pool attached 1128 */ 1129bool ice_xsk_any_rx_ring_ena(struct ice_vsi *vsi) 1130{ 1131 int i; 1132 1133 ice_for_each_rxq(vsi, i) { 1134 if (xsk_get_pool_from_qid(vsi->netdev, i)) 1135 return true; 1136 } 1137 1138 return false; 1139} 1140 1141/** 1142 * ice_xsk_clean_rx_ring - clean buffer pool queues connected to a given Rx ring 1143 * @rx_ring: ring to be cleaned 1144 */ 1145void ice_xsk_clean_rx_ring(struct ice_rx_ring *rx_ring) 1146{ 1147 u16 ntc = rx_ring->next_to_clean; 1148 u16 ntu = rx_ring->next_to_use; 1149 1150 while (ntc != ntu) { 1151 struct xdp_buff *xdp = *ice_xdp_buf(rx_ring, ntc); 1152 1153 xsk_buff_free(xdp); 1154 ntc++; 1155 if (ntc >= rx_ring->count) 1156 ntc = 0; 1157 } 1158} 1159 1160/** 1161 * ice_xsk_clean_xdp_ring - Clean the XDP Tx ring and its buffer pool queues 1162 * @xdp_ring: XDP_Tx ring 1163 */ 1164void ice_xsk_clean_xdp_ring(struct ice_tx_ring *xdp_ring) 1165{ 1166 u16 ntc = xdp_ring->next_to_clean, ntu = xdp_ring->next_to_use; 1167 u32 xsk_frames = 0; 1168 1169 while (ntc != ntu) { 1170 struct ice_tx_buf *tx_buf = &xdp_ring->tx_buf[ntc]; 1171 1172 if (tx_buf->type == ICE_TX_BUF_XSK_TX) { 1173 tx_buf->type = ICE_TX_BUF_EMPTY; 1174 xsk_buff_free(tx_buf->xdp); 1175 } else { 1176 xsk_frames++; 1177 } 1178 1179 ntc++; 1180 if (ntc >= xdp_ring->count) 1181 ntc = 0; 1182 } 1183 1184 if (xsk_frames) 1185 xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames); 1186} 1187