net/xdp/xsk.c

29 #include <net/xdp.h>
157 static int xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
159 	struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
160 	u32 frags = xdp_buff_has_frags(xdp);
179 		len = pos->xdp.data_end - pos->xdp.data;
188 	xsk_buff_free(xdp);
228 static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
231 	void *copy_from = xsk_copy_xdp_start(xdp), *copy_to;
237 	from_len = xdp->data_end - copy_from;
238 	meta_len = xdp->data - copy_from;
241 	if (len <= frame_size && !xdp_buff_has_frags(xdp)) {
250 		xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
271 	if (xdp_buff_has_frags(xdp)) {
274 		sinfo = xdp_get_shared_info_from_buff(xdp);
288 		xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
314 static int xsk_rcv_check(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
316 	struct net_device *dev = xdp->rxq->dev;
317 	u32 qid = xdp->rxq->queue_index;
330 	sk_mark_napi_id_once_xdp(&xs->sk, xdp);
341 int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
343 	u32 len = xdp_get_buff_len(xdp);
347 	err = xsk_rcv_check(xs, xdp, len);
349 		err = __xsk_rcv(xs, xdp, len);
356 static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
358 	u32 len = xdp_get_buff_len(xdp);
361 	err = xsk_rcv_check(xs, xdp, len);
365 	if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) {
366 		len = xdp->data_end - xdp->data;
367 		return xsk_rcv_zc(xs, xdp, len);
370 	err = __xsk_rcv(xs, xdp, len);
372 		xdp_return_buff(xdp);
376 int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
381 	err = xsk_rcv(xs, xdp);
1049 	/* Wait for driver to stop using the xdp socket. */
1115 	mutex_lock(&net->xdp.lock);
1117 	mutex_unlock(&net->xdp.lock);
1642 		mutex_lock(&net->xdp.lock);
1643 		sk_for_each(sk, &net->xdp.list) {
1659 		mutex_unlock(&net->xdp.lock);
1740 	mutex_lock(&net->xdp.lock);
1741 	sk_add_node_rcu(sk, &net->xdp.list);
1742 	mutex_unlock(&net->xdp.lock);
1761 	mutex_init(&net->xdp.lock);
1762 	INIT_HLIST_HEAD(&net->xdp.list);
1768 	WARN_ON_ONCE(!hlist_empty(&net->xdp.list));