1/* SPDX-License-Identifier: GPL-2.0 */ 2/* Copyright(c) 2020 Intel Corporation. */ 3 4#ifndef XSK_BUFF_POOL_H_ 5#define XSK_BUFF_POOL_H_ 6 7#include <linux/if_xdp.h> 8#include <linux/types.h> 9#include <linux/dma-mapping.h> 10#include <linux/bpf.h> 11#include <net/xdp.h> 12 13struct xsk_buff_pool; 14struct xdp_rxq_info; 15struct xsk_cb_desc; 16struct xsk_queue; 17struct xdp_desc; 18struct xdp_umem; 19struct xdp_sock; 20struct device; 21struct page; 22 23#define XSK_PRIV_MAX 24 24 25struct xdp_buff_xsk { 26 struct xdp_buff xdp; 27 u8 cb[XSK_PRIV_MAX]; 28 dma_addr_t dma; 29 dma_addr_t frame_dma; 30 struct xsk_buff_pool *pool; 31 u64 orig_addr; 32 struct list_head free_list_node; 33 struct list_head xskb_list_node; 34}; 35 36#define XSK_CHECK_PRIV_TYPE(t) BUILD_BUG_ON(sizeof(t) > offsetofend(struct xdp_buff_xsk, cb)) 37#define XSK_TX_COMPL_FITS(t) BUILD_BUG_ON(sizeof(struct xsk_tx_metadata_compl) > sizeof(t)) 38 39struct xsk_dma_map { 40 dma_addr_t *dma_pages; 41 struct device *dev; 42 struct net_device *netdev; 43 refcount_t users; 44 struct list_head list; /* Protected by the RTNL_LOCK */ 45 u32 dma_pages_cnt; 46 bool dma_need_sync; 47}; 48 49struct xsk_buff_pool { 50 /* Members only used in the control path first. */ 51 struct device *dev; 52 struct net_device *netdev; 53 struct list_head xsk_tx_list; 54 /* Protects modifications to the xsk_tx_list */ 55 spinlock_t xsk_tx_list_lock; 56 refcount_t users; 57 struct xdp_umem *umem; 58 struct work_struct work; 59 struct list_head free_list; 60 struct list_head xskb_list; 61 u32 heads_cnt; 62 u16 queue_id; 63 64 /* Data path members as close to free_heads at the end as possible. */ 65 struct xsk_queue *fq ____cacheline_aligned_in_smp; 66 struct xsk_queue *cq; 67 /* For performance reasons, each buff pool has its own array of dma_pages 68 * even when they are identical. 69 */ 70 dma_addr_t *dma_pages; 71 struct xdp_buff_xsk *heads; 72 struct xdp_desc *tx_descs; 73 u64 chunk_mask; 74 u64 addrs_cnt; 75 u32 free_list_cnt; 76 u32 dma_pages_cnt; 77 u32 free_heads_cnt; 78 u32 headroom; 79 u32 chunk_size; 80 u32 chunk_shift; 81 u32 frame_len; 82 u8 tx_metadata_len; /* inherited from umem */ 83 u8 cached_need_wakeup; 84 bool uses_need_wakeup; 85 bool dma_need_sync; 86 bool unaligned; 87 bool tx_sw_csum; 88 void *addrs; 89 /* Mutual exclusion of the completion ring in the SKB mode. Two cases to protect: 90 * NAPI TX thread and sendmsg error paths in the SKB destructor callback and when 91 * sockets share a single cq when the same netdev and queue id is shared. 92 */ 93 spinlock_t cq_lock; 94 struct xdp_buff_xsk *free_heads[]; 95}; 96 97/* Masks for xdp_umem_page flags. 98 * The low 12-bits of the addr will be 0 since this is the page address, so we 99 * can use them for flags. 100 */ 101#define XSK_NEXT_PG_CONTIG_SHIFT 0 102#define XSK_NEXT_PG_CONTIG_MASK BIT_ULL(XSK_NEXT_PG_CONTIG_SHIFT) 103 104/* AF_XDP core. */ 105struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs, 106 struct xdp_umem *umem); 107int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *dev, 108 u16 queue_id, u16 flags); 109int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_sock *umem_xs, 110 struct net_device *dev, u16 queue_id); 111int xp_alloc_tx_descs(struct xsk_buff_pool *pool, struct xdp_sock *xs); 112void xp_destroy(struct xsk_buff_pool *pool); 113void xp_get_pool(struct xsk_buff_pool *pool); 114bool xp_put_pool(struct xsk_buff_pool *pool); 115void xp_clear_dev(struct xsk_buff_pool *pool); 116void xp_add_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs); 117void xp_del_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs); 118 119/* AF_XDP, and XDP core. */ 120void xp_free(struct xdp_buff_xsk *xskb); 121 122static inline void xp_init_xskb_addr(struct xdp_buff_xsk *xskb, struct xsk_buff_pool *pool, 123 u64 addr) 124{ 125 xskb->orig_addr = addr; 126 xskb->xdp.data_hard_start = pool->addrs + addr + pool->headroom; 127} 128 129static inline void xp_init_xskb_dma(struct xdp_buff_xsk *xskb, struct xsk_buff_pool *pool, 130 dma_addr_t *dma_pages, u64 addr) 131{ 132 xskb->frame_dma = (dma_pages[addr >> PAGE_SHIFT] & ~XSK_NEXT_PG_CONTIG_MASK) + 133 (addr & ~PAGE_MASK); 134 xskb->dma = xskb->frame_dma + pool->headroom + XDP_PACKET_HEADROOM; 135} 136 137/* AF_XDP ZC drivers, via xdp_sock_buff.h */ 138void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq); 139void xp_fill_cb(struct xsk_buff_pool *pool, struct xsk_cb_desc *desc); 140int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev, 141 unsigned long attrs, struct page **pages, u32 nr_pages); 142void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs); 143struct xdp_buff *xp_alloc(struct xsk_buff_pool *pool); 144u32 xp_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max); 145bool xp_can_alloc(struct xsk_buff_pool *pool, u32 count); 146void *xp_raw_get_data(struct xsk_buff_pool *pool, u64 addr); 147dma_addr_t xp_raw_get_dma(struct xsk_buff_pool *pool, u64 addr); 148static inline dma_addr_t xp_get_dma(struct xdp_buff_xsk *xskb) 149{ 150 return xskb->dma; 151} 152 153static inline dma_addr_t xp_get_frame_dma(struct xdp_buff_xsk *xskb) 154{ 155 return xskb->frame_dma; 156} 157 158void xp_dma_sync_for_cpu_slow(struct xdp_buff_xsk *xskb); 159static inline void xp_dma_sync_for_cpu(struct xdp_buff_xsk *xskb) 160{ 161 xp_dma_sync_for_cpu_slow(xskb); 162} 163 164void xp_dma_sync_for_device_slow(struct xsk_buff_pool *pool, dma_addr_t dma, 165 size_t size); 166static inline void xp_dma_sync_for_device(struct xsk_buff_pool *pool, 167 dma_addr_t dma, size_t size) 168{ 169 if (!pool->dma_need_sync) 170 return; 171 172 xp_dma_sync_for_device_slow(pool, dma, size); 173} 174 175/* Masks for xdp_umem_page flags. 176 * The low 12-bits of the addr will be 0 since this is the page address, so we 177 * can use them for flags. 178 */ 179#define XSK_NEXT_PG_CONTIG_SHIFT 0 180#define XSK_NEXT_PG_CONTIG_MASK BIT_ULL(XSK_NEXT_PG_CONTIG_SHIFT) 181 182static inline bool xp_desc_crosses_non_contig_pg(struct xsk_buff_pool *pool, 183 u64 addr, u32 len) 184{ 185 bool cross_pg = (addr & (PAGE_SIZE - 1)) + len > PAGE_SIZE; 186 187 if (likely(!cross_pg)) 188 return false; 189 190 return pool->dma_pages && 191 !(pool->dma_pages[addr >> PAGE_SHIFT] & XSK_NEXT_PG_CONTIG_MASK); 192} 193 194static inline bool xp_mb_desc(struct xdp_desc *desc) 195{ 196 return desc->options & XDP_PKT_CONTD; 197} 198 199static inline u64 xp_aligned_extract_addr(struct xsk_buff_pool *pool, u64 addr) 200{ 201 return addr & pool->chunk_mask; 202} 203 204static inline u64 xp_unaligned_extract_addr(u64 addr) 205{ 206 return addr & XSK_UNALIGNED_BUF_ADDR_MASK; 207} 208 209static inline u64 xp_unaligned_extract_offset(u64 addr) 210{ 211 return addr >> XSK_UNALIGNED_BUF_OFFSET_SHIFT; 212} 213 214static inline u64 xp_unaligned_add_offset_to_addr(u64 addr) 215{ 216 return xp_unaligned_extract_addr(addr) + 217 xp_unaligned_extract_offset(addr); 218} 219 220static inline u32 xp_aligned_extract_idx(struct xsk_buff_pool *pool, u64 addr) 221{ 222 return xp_aligned_extract_addr(pool, addr) >> pool->chunk_shift; 223} 224 225static inline void xp_release(struct xdp_buff_xsk *xskb) 226{ 227 if (xskb->pool->unaligned) 228 xskb->pool->free_heads[xskb->pool->free_heads_cnt++] = xskb; 229} 230 231static inline u64 xp_get_handle(struct xdp_buff_xsk *xskb) 232{ 233 u64 offset = xskb->xdp.data - xskb->xdp.data_hard_start; 234 235 offset += xskb->pool->headroom; 236 if (!xskb->pool->unaligned) 237 return xskb->orig_addr + offset; 238 return xskb->orig_addr + (offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT); 239} 240 241static inline bool xp_tx_metadata_enabled(const struct xsk_buff_pool *pool) 242{ 243 return pool->tx_metadata_len > 0; 244} 245 246#endif /* XSK_BUFF_POOL_H_ */ 247