mlx5/core/en_rx.c

64 mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
68 mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
375 				     struct mlx5e_wqe_frag_info *wi)
379 	for (i = 0; i < rq->wqe.info.num_frags; i++, wi++)
380 		mlx5e_put_rx_frag(rq, wi);
383 static void mlx5e_xsk_free_rx_wqe(struct mlx5e_wqe_frag_info *wi)
385 	if (!(wi->flags & BIT(MLX5E_WQE_FRAG_SKIP_RELEASE)))
386 		xsk_buff_free(*wi->xskp);
391 	struct mlx5e_wqe_frag_info *wi = get_frag(rq, ix);
394 		mlx5e_xsk_free_rx_wqe(wi);
396 		mlx5e_free_rx_wqe(rq, wi);
402 		for (int i = 0; i < rq->wqe.info.num_frags; i++, wi++)
403 			wi->flags |= BIT(MLX5E_WQE_FRAG_SKIP_RELEASE);
414 		struct mlx5e_wqe_frag_info *wi;
416 		wi = get_frag(rq, j);
421 		mlx5e_xsk_free_rx_wqe(wi);
432 		struct mlx5e_wqe_frag_info *wi;
434 		wi = get_frag(rq, j);
435 		mlx5e_free_rx_wqe(rq, wi);
552 mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi)
558 	if (bitmap_full(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe))
561 	no_xdp_xmit = bitmap_empty(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe);
564 		struct xdp_buff **xsk_buffs = wi->alloc_units.xsk_buffs;
571 			if (no_xdp_xmit || !test_bit(i, wi->skip_release_bitmap))
575 			if (no_xdp_xmit || !test_bit(i, wi->skip_release_bitmap)) {
578 				frag_page = &wi->alloc_units.frag_pages[i];
761 	struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, ix);
781 	frag_page = &wi->alloc_units.frag_pages[0];
806 	bitmap_zero(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe);
807 	wi->consumed_strides = 0;
834 	bitmap_fill(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe);
884 	struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, ix);
886 	mlx5e_free_rx_mpwqe(rq, wi);
891 	bitmap_fill(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe);
954 		struct mlx5e_icosq_wqe_info *wi;
958 		wi = &sq->db.wqe_info[ci];
959 		sqcc += wi->num_wqebbs;
961 		switch (wi->wqe_type) {
963 			mlx5e_ktls_handle_ctx_completion(wi);
966 			mlx5e_ktls_handle_get_psv_completion(wi, sq);
1025 			struct mlx5e_icosq_wqe_info *wi;
1031 			wi = &sq->db.wqe_info[ci];
1032 			sqcc += wi->num_wqebbs;
1040 				mlx5_wq_cyc_wqe_dump(&sq->wq, ci, wi->num_wqebbs);
1046 			switch (wi->wqe_type) {
1048 				wi->umr.rq->mpwqe.umr_completed++;
1053 				mlx5e_handle_shampo_hd_umr(wi->shampo, sq);
1059 				mlx5e_ktls_handle_ctx_completion(wi);
1062 				mlx5e_ktls_handle_get_psv_completion(wi, sq);
1068 						 wi->wqe_type);
1112 		struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, head);
1115 		mlx5e_free_rx_mpwqe(rq, wi);
1672 mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi,
1675 	struct mlx5e_frag_page *frag_page = wi->frag_page;
1684 	va             = page_address(frag_page->page) + wi->offset;
1689 	dma_sync_single_range_for_cpu(rq->pdev, addr, wi->offset,
1720 mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi,
1724 	struct mlx5e_wqe_frag_info *head_wi = wi;
1736 	frag_page = wi->frag_page;
1738 	va = page_address(frag_page->page) + wi->offset;
1742 	dma_sync_single_range_for_cpu(rq->pdev, addr, wi->offset,
1754 	wi++;
1757 		frag_page = wi->frag_page;
1762 					       wi->offset, frag_consumed_bytes);
1767 		wi++;
1775 			for (pwi = head_wi; pwi < wi; pwi++)
1793 		xdp_update_skb_shared_info(skb, wi - head_wi - 1,
1797 		for (struct mlx5e_wqe_frag_info *pwi = head_wi + 1; pwi < wi; pwi++)
1825 	struct mlx5e_wqe_frag_info *wi;
1831 	wi       = get_frag(rq, ci);
1843 			      rq, wi, cqe, cqe_bcnt);
1847 			wi->frag_page->frags++;
1873 	struct mlx5e_wqe_frag_info *wi;
1879 	wi       = get_frag(rq, ci);
1890 			      rq, wi, cqe, cqe_bcnt);
1894 			wi->frag_page->frags++;
1913 	struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, wqe_id);
1923 	wi->consumed_strides += cstrides;
1943 			      rq, wi, cqe, cqe_bcnt, head_offset, page_idx);
1952 	if (likely(wi->consumed_strides < rq->mpwqe.num_strides))
1994 mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
1998 	struct mlx5e_frag_page *frag_page = &wi->alloc_units.frag_pages[page_idx];
2018 		if (unlikely(mlx5e_page_alloc_fragmented(rq, &wi->linear_page))) {
2022 		va = page_address(wi->linear_page.page);
2078 				wi->linear_page.frags++;
2080 			mlx5e_page_release_fragmented(rq, &wi->linear_page);
2089 			mlx5e_page_release_fragmented(rq, &wi->linear_page);
2094 		wi->linear_page.frags++;
2095 		mlx5e_page_release_fragmented(rq, &wi->linear_page);
2139 mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
2143 	struct mlx5e_frag_page *frag_page = &wi->alloc_units.frag_pages[page_idx];
2197 mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
2313 	struct mlx5e_mpw_info *wi;
2316 	wi = mlx5e_get_mpw_info(rq, wqe_id);
2317 	wi->consumed_strides += cstrides;
2339 			*skb = mlx5e_skb_from_cqe_shampo(rq, wi, cqe, header_index);
2341 			*skb = mlx5e_skb_from_cqe_mpwrq_nonlinear(rq, wi, cqe, cqe_bcnt,
2364 		frag_page = &wi->alloc_units.frag_pages[page_idx];
2374 	if (likely(wi->consumed_strides < rq->mpwqe.num_strides))
2386 	struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, wqe_id);
2396 	wi->consumed_strides += cstrides;
2417 			      rq, wi, cqe, cqe_bcnt, head_offset,
2433 	if (likely(wi->consumed_strides < rq->mpwqe.num_strides))
2645 	struct mlx5e_wqe_frag_info *wi;
2651 	wi       = get_frag(rq, ci);
2662 			      rq, wi, cqe, cqe_bcnt);
2735 	struct mlx5e_wqe_frag_info *wi;
2743 	wi       = get_frag(rq, ci);
2751 	skb = mlx5e_skb_from_cqe_nonlinear(rq, wi, cqe, cqe_bcnt);