include/net/xdp.h

2 /* include/net/xdp.h
74 	XDP_FLAGS_HAS_FRAGS		= BIT(0), /* non-linear xdp buff */
75 	XDP_FLAGS_FRAGS_PF_MEMALLOC	= BIT(1), /* xdp paged memory is under
91 static __always_inline bool xdp_buff_has_frags(struct xdp_buff *xdp)
93 	return !!(xdp->flags & XDP_FLAGS_HAS_FRAGS);
96 static __always_inline void xdp_buff_set_frags_flag(struct xdp_buff *xdp)
98 	xdp->flags |= XDP_FLAGS_HAS_FRAGS;
101 static __always_inline void xdp_buff_clear_frags_flag(struct xdp_buff *xdp)
103 	xdp->flags &= ~XDP_FLAGS_HAS_FRAGS;
106 static __always_inline bool xdp_buff_is_frag_pfmemalloc(struct xdp_buff *xdp)
108 	return !!(xdp->flags & XDP_FLAGS_FRAGS_PF_MEMALLOC);
111 static __always_inline void xdp_buff_set_frag_pfmemalloc(struct xdp_buff *xdp)
113 	xdp->flags |= XDP_FLAGS_FRAGS_PF_MEMALLOC;
117 xdp_init_buff(struct xdp_buff *xdp, u32 frame_sz, struct xdp_rxq_info *rxq)
119 	xdp->frame_sz = frame_sz;
120 	xdp->rxq = rxq;
121 	xdp->flags = 0;
125 xdp_prepare_buff(struct xdp_buff *xdp, unsigned char *hard_start,
130 	xdp->data_hard_start = hard_start;
131 	xdp->data = data;
132 	xdp->data_end = data + data_len;
133 	xdp->data_meta = meta_valid ? data : data + 1;
142 #define xdp_data_hard_end(xdp)				\
143 	((xdp)->data_hard_start + (xdp)->frame_sz -	\
147 xdp_get_shared_info_from_buff(struct xdp_buff *xdp)
149 	return (struct skb_shared_info *)xdp_data_hard_end(xdp);
152 static __always_inline unsigned int xdp_get_buff_len(struct xdp_buff *xdp)
154 	unsigned int len = xdp->data_end - xdp->data;
157 	if (likely(!xdp_buff_has_frags(xdp)))
160 	sinfo = xdp_get_shared_info_from_buff(xdp);
242 struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp);
252 void xdp_convert_frame_to_buff(struct xdp_frame *frame, struct xdp_buff *xdp)
254 	xdp->data_hard_start = frame->data - frame->headroom - sizeof(*frame);
255 	xdp->data = frame->data;
256 	xdp->data_end = frame->data + frame->len;
257 	xdp->data_meta = frame->data - frame->metasize;
258 	xdp->frame_sz = frame->frame_sz;
259 	xdp->flags = frame->flags;
263 int xdp_update_frame_from_buff(struct xdp_buff *xdp,
269 	headroom = xdp->data - xdp->data_hard_start;
270 	metasize = xdp->data - xdp->data_meta;
276 	if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) {
281 	xdp_frame->data = xdp->data;
282 	xdp_frame->len  = xdp->data_end - xdp->data;
285 	xdp_frame->frame_sz = xdp->frame_sz;
286 	xdp_frame->flags = xdp->flags;
293 struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp)
297 	if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL)
298 		return xdp_convert_zc_to_xdp_frame(xdp);
301 	xdp_frame = xdp->data_hard_start;
302 	if (unlikely(xdp_update_frame_from_buff(xdp, xdp_frame) < 0))
306 	xdp_frame->mem = xdp->rxq->mem;
312 		  struct xdp_buff *xdp);
315 void xdp_return_buff(struct xdp_buff *xdp);
359 xdp_set_data_meta_invalid(struct xdp_buff *xdp)
361 	xdp->data_meta = xdp->data + 1;
365 xdp_data_meta_unsupported(const struct xdp_buff *xdp)
367 	return unlikely(xdp->data_meta > xdp->data);
391 /* Define the relationship between xdp-rx-metadata kfunc and
508 					    struct xdp_buff *xdp)
514 	u32 act = __bpf_prog_run(prog, xdp, BPF_DISPATCHER_FUNC(xdp));
517 		if (act == XDP_TX && netif_is_bond_slave(xdp->rxq->dev))
518 			act = xdp_master_redirect(xdp);