intel/ice/ice_txrx.c

12 #include <net/xdp.h>
385 	struct xdp_buff *xdp = &rx_ring->xdp;
399 	if (xdp->data) {
400 		xdp_return_buff(xdp);
401 		xdp->data = NULL;
551  * @xdp: xdp_buff used as input to the XDP program
560 ice_run_xdp(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp,
570 	ice_xdp_meta_set_desc(xdp, eop_desc);
572 	act = bpf_prog_run_xdp(xdp_prog, xdp);
579 		ret = __ice_xmit_xdp_ring(xdp, xdp_ring, false);
586 		if (xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog))
601 	ice_set_rx_bufs_act(xdp, rx_ring, ret);
612 	struct xdp_buff xdp;
614 	xdp.data_hard_start = (void *)xdpf;
615 	xdp.data = xdpf->data;
616 	xdp.data_end = xdp.data + xdpf->len;
617 	xdp.frame_sz = xdpf->frame_sz;
618 	xdp.flags = xdpf->flags;
620 	return __ice_xmit_xdp_ring(&xdp, xdp_ring, true);
864  * ice_add_xdp_frag - Add contents of Rx buffer to xdp buf as a frag
866  * @xdp: xdp buff to place the data into
870  * This function will add the data contained in rx_buf->page to the xdp buf.
874 ice_add_xdp_frag(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp,
877 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
882 	if (!xdp_buff_has_frags(xdp)) {
885 		xdp_buff_set_frags_flag(xdp);
889 		ice_set_rx_bufs_act(xdp, rx_ring, ICE_XDP_CONSUMED);
902 		xdp_buff_set_frag_pfmemalloc(xdp);
976  * @xdp: xdp_buff pointing to the data
983 ice_build_skb(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp)
985 	u8 metasize = xdp->data - xdp->data_meta;
990 	if (unlikely(xdp_buff_has_frags(xdp))) {
991 		sinfo = xdp_get_shared_info_from_buff(xdp);
995 	/* Prefetch first cache line of first page. If xdp->data_meta
996 	 * is unused, this points exactly as xdp->data, otherwise we
1000 	net_prefetch(xdp->data_meta);
1002 	skb = napi_build_skb(xdp->data_hard_start, xdp->frame_sz);
1012 	skb_reserve(skb, xdp->data - xdp->data_hard_start);
1013 	__skb_put(skb, xdp->data_end - xdp->data);
1017 	if (unlikely(xdp_buff_has_frags(xdp)))
1020 					   nr_frags * xdp->frame_sz,
1021 					   xdp_buff_is_frag_pfmemalloc(xdp));
1029  * @xdp: xdp_buff pointing to the data
1036 ice_construct_skb(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp)
1038 	unsigned int size = xdp->data_end - xdp->data;
1046 	net_prefetch(xdp->data);
1048 	if (unlikely(xdp_buff_has_frags(xdp))) {
1049 		sinfo = xdp_get_shared_info_from_buff(xdp);
1063 		headlen = eth_get_headlen(skb->dev, xdp->data, ICE_RX_HDR_SIZE);
1066 	memcpy(__skb_put(skb, headlen), xdp->data, ALIGN(headlen,
1082 				xdp->frame_sz);
1092 	if (unlikely(xdp_buff_has_frags(xdp))) {
1100 					   nr_frags * xdp->frame_sz,
1101 					   xdp_buff_is_frag_pfmemalloc(xdp));
1152 	struct xdp_buff *xdp = &rx_ring->xdp;
1165 	xdp->frame_sz = ice_rx_frame_truesize(rx_ring, 0);
1220 		if (!xdp->data) {
1225 			xdp_prepare_buff(xdp, hard_start, offset, size, !!offset);
1228 			xdp->frame_sz = ice_rx_frame_truesize(rx_ring, size);
1230 			xdp_buff_clear_frags_flag(xdp);
1231 		} else if (ice_add_xdp_frag(rx_ring, xdp, rx_buf, size)) {
1241 		ice_run_xdp(rx_ring, xdp, xdp_prog, xdp_ring, rx_buf, rx_desc);
1244 		total_rx_bytes += xdp_get_buff_len(xdp);
1247 		xdp->data = NULL;
1253 			skb = ice_build_skb(rx_ring, xdp);
1255 			skb = ice_construct_skb(rx_ring, xdp);
1260 			if (unlikely(xdp_buff_has_frags(xdp)))
1261 				ice_set_rx_bufs_act(xdp, rx_ring,
1263 			xdp->data = NULL;
1268 		xdp->data = NULL;
1304 			ice_rx_buf_adjust_pg_offset(buf, xdp->frame_sz);
1309 			ice_rx_buf_adjust_pg_offset(buf, xdp->frame_sz);