nfp/nfd3/dp.c

31 static int nfp_nfd3_tx_ring_should_wake(struct nfp_net_tx_ring *tx_ring)
33 	return !nfp_net_tx_full(tx_ring, MAX_SKB_FRAGS * 4);
36 static int nfp_nfd3_tx_ring_should_stop(struct nfp_net_tx_ring *tx_ring)
38 	return nfp_net_tx_full(tx_ring, MAX_SKB_FRAGS + 1);
44  * @tx_ring: driver tx queue structure
52 		      struct nfp_net_tx_ring *tx_ring)
58 	if (unlikely(nfp_nfd3_tx_ring_should_wake(tx_ring)))
259 	struct nfp_net_tx_ring *tx_ring;
274 	tx_ring = &dp->tx_rings[qidx];
275 	r_vec = tx_ring->r_vec;
279 	if (unlikely(nfp_net_tx_full(tx_ring, nr_frags + 1))) {
281 			   qidx, tx_ring->wr_p, tx_ring->rd_p);
284 		nfp_net_tx_xmit_more_flush(tx_ring);
293 		nfp_net_tx_xmit_more_flush(tx_ring);
307 	wr_idx = D_IDX(tx_ring, tx_ring->wr_p);
310 	txbuf = &tx_ring->txbufs[wr_idx];
318 	txd = &tx_ring->txds[wr_idx];
355 			wr_idx = D_IDX(tx_ring, wr_idx + 1);
356 			tx_ring->txbufs[wr_idx].skb = skb;
357 			tx_ring->txbufs[wr_idx].dma_addr = dma_addr;
358 			tx_ring->txbufs[wr_idx].fidx = f;
360 			txd = &tx_ring->txds[wr_idx];
375 	nd_q = netdev_get_tx_queue(dp->netdev, tx_ring->idx);
377 	tx_ring->wr_p += nr_frags + 1;
378 	if (nfp_nfd3_tx_ring_should_stop(tx_ring))
379 		nfp_nfd3_tx_ring_stop(nd_q, tx_ring);
381 	tx_ring->wr_ptr_add += nr_frags + 1;
383 		nfp_net_tx_xmit_more_flush(tx_ring);
390 		dma_unmap_page(dp->dev, tx_ring->txbufs[wr_idx].dma_addr,
392 		tx_ring->txbufs[wr_idx].skb = NULL;
393 		tx_ring->txbufs[wr_idx].dma_addr = 0;
394 		tx_ring->txbufs[wr_idx].fidx = -2;
397 			wr_idx += tx_ring->cnt;
399 	dma_unmap_single(dp->dev, tx_ring->txbufs[wr_idx].dma_addr,
401 	tx_ring->txbufs[wr_idx].skb = NULL;
402 	tx_ring->txbufs[wr_idx].dma_addr = 0;
403 	tx_ring->txbufs[wr_idx].fidx = -2;
407 	nfp_net_tx_xmit_more_flush(tx_ring);
418  * @tx_ring:	TX ring structure
421 void nfp_nfd3_tx_complete(struct nfp_net_tx_ring *tx_ring, int budget)
423 	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;
430 	if (tx_ring->wr_p == tx_ring->rd_p)
434 	qcp_rd_p = nfp_net_read_tx_cmpl(tx_ring, dp);
436 	if (qcp_rd_p == tx_ring->qcp_rd_p)
439 	todo = D_IDX(tx_ring, qcp_rd_p - tx_ring->qcp_rd_p);
448 		idx = D_IDX(tx_ring, tx_ring->rd_p++);
449 		tx_buf = &tx_ring->txbufs[idx];
481 	tx_ring->qcp_rd_p = qcp_rd_p;
491 	nd_q = netdev_get_tx_queue(dp->netdev, tx_ring->idx);
493 	if (nfp_nfd3_tx_ring_should_wake(tx_ring)) {
494 		/* Make sure TX thread will see updated tx_ring->rd_p */
501 	WARN_ONCE(tx_ring->wr_p - tx_ring->rd_p > tx_ring->cnt,
503 		  tx_ring->rd_p, tx_ring->wr_p, tx_ring->cnt);
506 static bool nfp_nfd3_xdp_complete(struct nfp_net_tx_ring *tx_ring)
508 	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;
516 	qcp_rd_p = nfp_net_read_tx_cmpl(tx_ring, dp);
518 	if (qcp_rd_p == tx_ring->qcp_rd_p)
521 	todo = D_IDX(tx_ring, qcp_rd_p - tx_ring->qcp_rd_p);
526 	tx_ring->qcp_rd_p = D_IDX(tx_ring, tx_ring->qcp_rd_p + todo);
530 		idx = D_IDX(tx_ring, tx_ring->rd_p);
531 		tx_ring->rd_p++;
533 		done_bytes += tx_ring->txbufs[idx].real_len;
541 	WARN_ONCE(tx_ring->wr_p - tx_ring->rd_p > tx_ring->cnt,
543 		  tx_ring->rd_p, tx_ring->wr_p, tx_ring->cnt);
837 		    struct nfp_net_tx_ring *tx_ring,
850 	if (unlikely(nfp_net_tx_full(tx_ring, 1))) {
852 			nfp_nfd3_xdp_complete(tx_ring);
856 		if (unlikely(nfp_net_tx_full(tx_ring, 1))) {
863 	wr_idx = D_IDX(tx_ring, tx_ring->wr_p);
866 	txbuf = &tx_ring->txbufs[wr_idx];
880 	txd = &tx_ring->txds[wr_idx];
890 	tx_ring->wr_p++;
891 	tx_ring->wr_ptr_add++;
910 	struct nfp_net_tx_ring *tx_ring;
922 	tx_ring = r_vec->xdp_ring;
1029 								  tx_ring,
1137 		if (tx_ring->wr_ptr_add)
1138 			nfp_net_tx_xmit_more_flush(tx_ring);
1139 		else if (unlikely(tx_ring->wr_p != tx_ring->rd_p) &&
1141 			if (!nfp_nfd3_xdp_complete(tx_ring))
1161 	if (r_vec->tx_ring)
1162 		nfp_nfd3_tx_complete(r_vec->tx_ring, budget);
1185 	if (r_vec->nfp_net->tx_coalesce_adapt_on && r_vec->tx_ring) {
1211 	struct nfp_net_tx_ring *tx_ring;
1219 	tx_ring = r_vec->tx_ring;
1226 	if (unlikely(nfp_net_tx_full(tx_ring, 1))) {
1253 	wr_idx = D_IDX(tx_ring, tx_ring->wr_p);
1256 	txbuf = &tx_ring->txbufs[wr_idx];
1264 	txd = &tx_ring->txds[wr_idx];
1274 	tx_ring->wr_p++;
1275 	tx_ring->wr_ptr_add++;
1276 	nfp_net_tx_xmit_more_flush(tx_ring);
1410 	nfp_nfd3_tx_complete(r_vec->tx_ring, 0);