1/* SPDX-License-Identifier: GPL-2.0 */ 2/* AF_XDP internal functions 3 * Copyright(c) 2018 Intel Corporation. 4 */ 5 6#ifndef _LINUX_XDP_SOCK_H 7#define _LINUX_XDP_SOCK_H 8 9#include <linux/bpf.h> 10#include <linux/workqueue.h> 11#include <linux/if_xdp.h> 12#include <linux/mutex.h> 13#include <linux/spinlock.h> 14#include <linux/mm.h> 15#include <net/sock.h> 16 17#define XDP_UMEM_SG_FLAG (1 << 1) 18 19struct net_device; 20struct xsk_queue; 21struct xdp_buff; 22 23struct xdp_umem { 24 void *addrs; 25 u64 size; 26 u32 headroom; 27 u32 chunk_size; 28 u32 chunks; 29 u32 npgs; 30 struct user_struct *user; 31 refcount_t users; 32 u8 flags; 33 u8 tx_metadata_len; 34 bool zc; 35 struct page **pgs; 36 int id; 37 struct list_head xsk_dma_list; 38 struct work_struct work; 39}; 40 41struct xsk_map { 42 struct bpf_map map; 43 spinlock_t lock; /* Synchronize map updates */ 44 atomic_t count; 45 struct xdp_sock __rcu *xsk_map[]; 46}; 47 48struct xdp_sock { 49 /* struct sock must be the first member of struct xdp_sock */ 50 struct sock sk; 51 struct xsk_queue *rx ____cacheline_aligned_in_smp; 52 struct net_device *dev; 53 struct xdp_umem *umem; 54 struct list_head flush_node; 55 struct xsk_buff_pool *pool; 56 u16 queue_id; 57 bool zc; 58 bool sg; 59 enum { 60 XSK_READY = 0, 61 XSK_BOUND, 62 XSK_UNBOUND, 63 } state; 64 65 struct xsk_queue *tx ____cacheline_aligned_in_smp; 66 struct list_head tx_list; 67 /* record the number of tx descriptors sent by this xsk and 68 * when it exceeds MAX_PER_SOCKET_BUDGET, an opportunity needs 69 * to be given to other xsks for sending tx descriptors, thereby 70 * preventing other XSKs from being starved. 71 */ 72 u32 tx_budget_spent; 73 74 /* Protects generic receive. */ 75 spinlock_t rx_lock; 76 77 /* Statistics */ 78 u64 rx_dropped; 79 u64 rx_queue_full; 80 81 /* When __xsk_generic_xmit() must return before it sees the EOP descriptor for the current 82 * packet, the partially built skb is saved here so that packet building can resume in next 83 * call of __xsk_generic_xmit(). 84 */ 85 struct sk_buff *skb; 86 87 struct list_head map_list; 88 /* Protects map_list */ 89 spinlock_t map_list_lock; 90 /* Protects multiple processes in the control path */ 91 struct mutex mutex; 92 struct xsk_queue *fq_tmp; /* Only as tmp storage before bind */ 93 struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */ 94}; 95 96/* 97 * AF_XDP TX metadata hooks for network devices. 98 * The following hooks can be defined; unless noted otherwise, they are 99 * optional and can be filled with a null pointer. 100 * 101 * void (*tmo_request_timestamp)(void *priv) 102 * Called when AF_XDP frame requested egress timestamp. 103 * 104 * u64 (*tmo_fill_timestamp)(void *priv) 105 * Called when AF_XDP frame, that had requested egress timestamp, 106 * received a completion. The hook needs to return the actual HW timestamp. 107 * 108 * void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv) 109 * Called when AF_XDP frame requested HW checksum offload. csum_start 110 * indicates position where checksumming should start. 111 * csum_offset indicates position where checksum should be stored. 112 * 113 */ 114struct xsk_tx_metadata_ops { 115 void (*tmo_request_timestamp)(void *priv); 116 u64 (*tmo_fill_timestamp)(void *priv); 117 void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv); 118}; 119 120#ifdef CONFIG_XDP_SOCKETS 121 122int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp); 123int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp); 124void __xsk_map_flush(void); 125 126/** 127 * xsk_tx_metadata_to_compl - Save enough relevant metadata information 128 * to perform tx completion in the future. 129 * @meta: pointer to AF_XDP metadata area 130 * @compl: pointer to output struct xsk_tx_metadata_to_compl 131 * 132 * This function should be called by the networking device when 133 * it prepares AF_XDP egress packet. The value of @compl should be stored 134 * and passed to xsk_tx_metadata_complete upon TX completion. 135 */ 136static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta, 137 struct xsk_tx_metadata_compl *compl) 138{ 139 if (!meta) 140 return; 141 142 if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP) 143 compl->tx_timestamp = &meta->completion.tx_timestamp; 144 else 145 compl->tx_timestamp = NULL; 146} 147 148/** 149 * xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission 150 * and call appropriate xsk_tx_metadata_ops operation. 151 * @meta: pointer to AF_XDP metadata area 152 * @ops: pointer to struct xsk_tx_metadata_ops 153 * @priv: pointer to driver-private aread 154 * 155 * This function should be called by the networking device when 156 * it prepares AF_XDP egress packet. 157 */ 158static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta, 159 const struct xsk_tx_metadata_ops *ops, 160 void *priv) 161{ 162 if (!meta) 163 return; 164 165 if (ops->tmo_request_timestamp) 166 if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP) 167 ops->tmo_request_timestamp(priv); 168 169 if (ops->tmo_request_checksum) 170 if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) 171 ops->tmo_request_checksum(meta->request.csum_start, 172 meta->request.csum_offset, priv); 173} 174 175/** 176 * xsk_tx_metadata_complete - Evaluate AF_XDP TX metadata at completion 177 * and call appropriate xsk_tx_metadata_ops operation. 178 * @compl: pointer to completion metadata produced from xsk_tx_metadata_to_compl 179 * @ops: pointer to struct xsk_tx_metadata_ops 180 * @priv: pointer to driver-private aread 181 * 182 * This function should be called by the networking device upon 183 * AF_XDP egress completion. 184 */ 185static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl, 186 const struct xsk_tx_metadata_ops *ops, 187 void *priv) 188{ 189 if (!compl) 190 return; 191 if (!compl->tx_timestamp) 192 return; 193 194 *compl->tx_timestamp = ops->tmo_fill_timestamp(priv); 195} 196 197#else 198 199static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) 200{ 201 return -ENOTSUPP; 202} 203 204static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp) 205{ 206 return -EOPNOTSUPP; 207} 208 209static inline void __xsk_map_flush(void) 210{ 211} 212 213static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta, 214 struct xsk_tx_metadata_compl *compl) 215{ 216} 217 218static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta, 219 const struct xsk_tx_metadata_ops *ops, 220 void *priv) 221{ 222} 223 224static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl, 225 const struct xsk_tx_metadata_ops *ops, 226 void *priv) 227{ 228} 229 230#endif /* CONFIG_XDP_SOCKETS */ 231 232#if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET) 233bool xsk_map_check_flush(void); 234#else 235static inline bool xsk_map_check_flush(void) 236{ 237 return false; 238} 239#endif 240 241#endif /* _LINUX_XDP_SOCK_H */ 242