1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2/* Copyright (C) 2018 Netronome Systems, Inc */ 3/* Copyright (C) 2021 Corigine, Inc */ 4 5#include <linux/dma-direction.h> 6#include <linux/dma-mapping.h> 7#include <linux/slab.h> 8#include <net/xdp_sock_drv.h> 9#include <trace/events/xdp.h> 10 11#include "nfp_app.h" 12#include "nfp_net.h" 13#include "nfp_net_dp.h" 14#include "nfp_net_xsk.h" 15 16static void 17nfp_net_xsk_rx_bufs_stash(struct nfp_net_rx_ring *rx_ring, unsigned int idx, 18 struct xdp_buff *xdp) 19{ 20 unsigned int headroom; 21 22 headroom = xsk_pool_get_headroom(rx_ring->r_vec->xsk_pool); 23 24 rx_ring->rxds[idx].fld.reserved = 0; 25 rx_ring->rxds[idx].fld.meta_len_dd = 0; 26 27 rx_ring->xsk_rxbufs[idx].xdp = xdp; 28 rx_ring->xsk_rxbufs[idx].dma_addr = 29 xsk_buff_xdp_get_frame_dma(xdp) + headroom; 30} 31 32void nfp_net_xsk_rx_unstash(struct nfp_net_xsk_rx_buf *rxbuf) 33{ 34 rxbuf->dma_addr = 0; 35 rxbuf->xdp = NULL; 36} 37 38void nfp_net_xsk_rx_free(struct nfp_net_xsk_rx_buf *rxbuf) 39{ 40 if (rxbuf->xdp) 41 xsk_buff_free(rxbuf->xdp); 42 43 nfp_net_xsk_rx_unstash(rxbuf); 44} 45 46void nfp_net_xsk_rx_bufs_free(struct nfp_net_rx_ring *rx_ring) 47{ 48 unsigned int i; 49 50 if (!rx_ring->cnt) 51 return; 52 53 for (i = 0; i < rx_ring->cnt - 1; i++) 54 nfp_net_xsk_rx_free(&rx_ring->xsk_rxbufs[i]); 55} 56 57void nfp_net_xsk_rx_ring_fill_freelist(struct nfp_net_rx_ring *rx_ring) 58{ 59 struct nfp_net_r_vector *r_vec = rx_ring->r_vec; 60 struct xsk_buff_pool *pool = r_vec->xsk_pool; 61 unsigned int wr_idx, wr_ptr_add = 0; 62 struct xdp_buff *xdp; 63 64 while (nfp_net_rx_space(rx_ring)) { 65 wr_idx = D_IDX(rx_ring, rx_ring->wr_p); 66 67 xdp = xsk_buff_alloc(pool); 68 if (!xdp) 69 break; 70 71 nfp_net_xsk_rx_bufs_stash(rx_ring, wr_idx, xdp); 72 73 /* DMA address is expanded to 48-bit width in freelist for NFP3800, 74 * so the *_48b macro is used accordingly, it's also OK to fill 75 * a 40-bit address since the top 8 bits are get set to 0. 76 */ 77 nfp_desc_set_dma_addr_48b(&rx_ring->rxds[wr_idx].fld, 78 rx_ring->xsk_rxbufs[wr_idx].dma_addr); 79 80 rx_ring->wr_p++; 81 wr_ptr_add++; 82 } 83 84 /* Ensure all records are visible before incrementing write counter. */ 85 wmb(); 86 nfp_qcp_wr_ptr_add(rx_ring->qcp_fl, wr_ptr_add); 87} 88 89void nfp_net_xsk_rx_drop(struct nfp_net_r_vector *r_vec, 90 struct nfp_net_xsk_rx_buf *xrxbuf) 91{ 92 u64_stats_update_begin(&r_vec->rx_sync); 93 r_vec->rx_drops++; 94 u64_stats_update_end(&r_vec->rx_sync); 95 96 nfp_net_xsk_rx_free(xrxbuf); 97} 98 99static void nfp_net_xsk_pool_unmap(struct device *dev, 100 struct xsk_buff_pool *pool) 101{ 102 return xsk_pool_dma_unmap(pool, 0); 103} 104 105static int nfp_net_xsk_pool_map(struct device *dev, struct xsk_buff_pool *pool) 106{ 107 return xsk_pool_dma_map(pool, dev, 0); 108} 109 110int nfp_net_xsk_setup_pool(struct net_device *netdev, 111 struct xsk_buff_pool *pool, u16 queue_id) 112{ 113 struct nfp_net *nn = netdev_priv(netdev); 114 115 struct xsk_buff_pool *prev_pool; 116 struct nfp_net_dp *dp; 117 int err; 118 119 /* NFDK doesn't implement xsk yet. */ 120 if (nn->dp.ops->version == NFP_NFD_VER_NFDK) 121 return -EOPNOTSUPP; 122 123 /* Reject on old FWs so we can drop some checks on datapath. */ 124 if (nn->dp.rx_offset != NFP_NET_CFG_RX_OFFSET_DYNAMIC) 125 return -EOPNOTSUPP; 126 if (!nn->dp.chained_metadata_format) 127 return -EOPNOTSUPP; 128 129 /* Install */ 130 if (pool) { 131 err = nfp_net_xsk_pool_map(nn->dp.dev, pool); 132 if (err) 133 return err; 134 } 135 136 /* Reconfig/swap */ 137 dp = nfp_net_clone_dp(nn); 138 if (!dp) { 139 err = -ENOMEM; 140 goto err_unmap; 141 } 142 143 prev_pool = dp->xsk_pools[queue_id]; 144 dp->xsk_pools[queue_id] = pool; 145 146 err = nfp_net_ring_reconfig(nn, dp, NULL); 147 if (err) 148 goto err_unmap; 149 150 /* Uninstall */ 151 if (prev_pool) 152 nfp_net_xsk_pool_unmap(nn->dp.dev, prev_pool); 153 154 return 0; 155err_unmap: 156 if (pool) 157 nfp_net_xsk_pool_unmap(nn->dp.dev, pool); 158 159 return err; 160} 161 162int nfp_net_xsk_wakeup(struct net_device *netdev, u32 queue_id, u32 flags) 163{ 164 struct nfp_net *nn = netdev_priv(netdev); 165 166 /* queue_id comes from a zero-copy socket, installed with XDP_SETUP_XSK_POOL, 167 * so it must be within our vector range. Moreover, our napi structs 168 * are statically allocated, so we can always kick them without worrying 169 * if reconfig is in progress or interface down. 170 */ 171 napi_schedule(&nn->r_vecs[queue_id].napi); 172 173 return 0; 174} 175