1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2/* Copyright (c) 2019 Mellanox Technologies. */ 3 4#include "tx.h" 5#include "pool.h" 6#include "en/xdp.h" 7#include "en/params.h" 8#include <net/xdp_sock_drv.h> 9 10int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) 11{ 12 struct mlx5e_priv *priv = netdev_priv(dev); 13 struct mlx5e_params *params = &priv->channels.params; 14 struct mlx5e_channel *c; 15 16 if (unlikely(!mlx5e_xdp_is_active(priv))) 17 return -ENETDOWN; 18 19 if (unlikely(qid >= params->num_channels)) 20 return -EINVAL; 21 22 c = priv->channels.c[qid]; 23 24 if (!napi_if_scheduled_mark_missed(&c->napi)) { 25 /* To avoid WQE overrun, don't post a NOP if async_icosq is not 26 * active and not polled by NAPI. Return 0, because the upcoming 27 * activate will trigger the IRQ for us. 28 */ 29 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &c->async_icosq.state))) 30 return 0; 31 32 if (test_and_set_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->async_icosq.state)) 33 return 0; 34 35 mlx5e_trigger_napi_icosq(c); 36 } 37 38 return 0; 39} 40 41/* When TX fails (because of the size of the packet), we need to get completions 42 * in order, so post a NOP to get a CQE. Since AF_XDP doesn't distinguish 43 * between successful TX and errors, handling in mlx5e_poll_xdpsq_cq is the 44 * same. 45 */ 46static void mlx5e_xsk_tx_post_err(struct mlx5e_xdpsq *sq, 47 union mlx5e_xdp_info *xdpi) 48{ 49 u16 pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc); 50 struct mlx5e_xdp_wqe_info *wi = &sq->db.wqe_info[pi]; 51 struct mlx5e_tx_wqe *nopwqe; 52 53 wi->num_wqebbs = 1; 54 wi->num_pkts = 1; 55 56 nopwqe = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc); 57 mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, *xdpi); 58 if (xp_tx_metadata_enabled(sq->xsk_pool)) 59 mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, 60 (union mlx5e_xdp_info) { .xsk_meta = {} }); 61 sq->doorbell_cseg = &nopwqe->ctrl; 62} 63 64bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget) 65{ 66 struct xsk_buff_pool *pool = sq->xsk_pool; 67 struct xsk_tx_metadata *meta = NULL; 68 union mlx5e_xdp_info xdpi; 69 bool work_done = true; 70 bool flush = false; 71 72 xdpi.mode = MLX5E_XDP_XMIT_MODE_XSK; 73 74 for (; budget; budget--) { 75 int check_result = INDIRECT_CALL_2(sq->xmit_xdp_frame_check, 76 mlx5e_xmit_xdp_frame_check_mpwqe, 77 mlx5e_xmit_xdp_frame_check, 78 sq); 79 struct mlx5e_xmit_data xdptxd = {}; 80 struct xdp_desc desc; 81 bool ret; 82 83 if (unlikely(check_result < 0)) { 84 work_done = false; 85 break; 86 } 87 88 if (!xsk_tx_peek_desc(pool, &desc)) { 89 /* TX will get stuck until something wakes it up by 90 * triggering NAPI. Currently it's expected that the 91 * application calls sendto() if there are consumed, but 92 * not completed frames. 93 */ 94 break; 95 } 96 97 xdptxd.dma_addr = xsk_buff_raw_get_dma(pool, desc.addr); 98 xdptxd.data = xsk_buff_raw_get_data(pool, desc.addr); 99 xdptxd.len = desc.len; 100 meta = xsk_buff_get_metadata(pool, desc.addr); 101 102 xsk_buff_raw_dma_sync_for_device(pool, xdptxd.dma_addr, xdptxd.len); 103 104 ret = INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe, 105 mlx5e_xmit_xdp_frame, sq, &xdptxd, 106 check_result, meta); 107 if (unlikely(!ret)) { 108 if (sq->mpwqe.wqe) 109 mlx5e_xdp_mpwqe_complete(sq); 110 111 mlx5e_xsk_tx_post_err(sq, &xdpi); 112 } else { 113 mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, xdpi); 114 if (xp_tx_metadata_enabled(sq->xsk_pool)) { 115 struct xsk_tx_metadata_compl compl; 116 117 xsk_tx_metadata_to_compl(meta, &compl); 118 XSK_TX_COMPL_FITS(void *); 119 120 mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, 121 (union mlx5e_xdp_info) 122 { .xsk_meta = compl }); 123 } 124 } 125 126 flush = true; 127 } 128 129 if (flush) { 130 if (sq->mpwqe.wqe) 131 mlx5e_xdp_mpwqe_complete(sq); 132 mlx5e_xmit_xdp_doorbell(sq); 133 134 xsk_tx_release(pool); 135 } 136 137 return !(budget && work_done); 138} 139