drivers/net/virtio_net.c

24 #include <net/xdp.h>
394 	/* xdp_queue_pairs may be 0, when xdp is already loaded. So add this. */
1075 /* when vi->curr_queue_pairs > nr_cpu_ids, the txq/sq is only used for xdp tx on
1172 static void put_xdp_frags(struct xdp_buff *xdp)
1178 	if (xdp_buff_has_frags(xdp)) {
1179 		shinfo = xdp_get_shared_info_from_buff(xdp);
1187 static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
1196 	act = bpf_prog_run_xdp(xdp_prog, xdp);
1205 		xdpf = xdp_convert_buff_to_frame(xdp);
1207 			netdev_dbg(dev, "convert buff to frame failed for xdp\n");
1223 		err = xdp_do_redirect(dev, xdp, xdp_prog);
1355 	struct xdp_buff xdp;
1387 	xdp_init_buff(&xdp, buflen, &rq->xdp_rxq);
1388 	xdp_prepare_buff(&xdp, buf + VIRTNET_RX_PAD + vi->hdr_len,
1391 	act = virtnet_xdp_handler(xdp_prog, &xdp, dev, xdp_xmit, stats);
1396 		len = xdp.data_end - xdp.data;
1397 		metasize = xdp.data - xdp.data_meta;
1408 	skb = virtnet_build_skb(buf, buflen, xdp.data - buf, len);
1519  * XDP core assumes that xdp frags are PAGE_SIZE in length, while in
1521  *  1. The size of the prefilled buffer is not fixed before xdp is set.
1527 					       struct xdp_buff *xdp,
1530 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
1536 	if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) {
1537 		pr_debug("Error building skb as missing reserved tailroom for xdp");
1541 	if (unlikely(xdp_buff_has_frags(xdp)))
1544 	skb = build_skb(xdp->data_hard_start, xdp->frame_sz);
1548 	headroom = xdp->data - xdp->data_hard_start;
1549 	data_len = xdp->data_end - xdp->data;
1553 	metasize = xdp->data - xdp->data_meta;
1558 	if (unlikely(xdp_buff_has_frags(xdp)))
1562 					   xdp_buff_is_frag_pfmemalloc(xdp));
1567 /* TODO: build xdp in big mode */
1571 				      struct xdp_buff *xdp,
1589 	xdp_init_buff(xdp, frame_sz, &rq->xdp_rxq);
1590 	xdp_prepare_buff(xdp, buf - VIRTIO_XDP_HEADROOM,
1597 		/* If we want to build multi-buffer xdp, we need
1601 		if (!xdp_buff_has_frags(xdp))
1602 			xdp_buff_set_frags_flag(xdp);
1604 		shinfo = xdp_get_shared_info_from_buff(xdp);
1644 			xdp_buff_set_frag_pfmemalloc(xdp);
1653 	put_xdp_frags(xdp);
1683 	 * hole mechanism for xdp. See add_recvbuf_mergeable().
1698 	 * using xdp normally.
1748 	struct xdp_buff xdp;
1758 	err = virtnet_build_xdp_buff_mrg(dev, vi, rq, &xdp, data, len, frame_sz,
1763 	act = virtnet_xdp_handler(xdp_prog, &xdp, dev, xdp_xmit, stats);
1767 		head_skb = build_skb_from_xdp_buff(dev, vi, &xdp, xdp_frags_truesz);
1780 	put_xdp_frags(&xdp);
5034 static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp)
5036 	switch (xdp->command) {
5038 		return virtnet_xdp_set(dev, xdp->prog, xdp->extack);
5585 	const struct xdp_buff *xdp = (void *)_ctx;
5590 	if (!(xdp->rxq->dev->features & NETIF_F_RXHASH))
5593 	vi = netdev_priv(xdp->rxq->dev);
5594 	hdr_hash = (struct virtio_net_hdr_v1_hash *)(xdp->data - vi->hdr_len);