nfp/nfdk/dp.c

18 static int nfp_nfdk_tx_ring_should_wake(struct nfp_net_tx_ring *tx_ring)
20 	return !nfp_net_tx_full(tx_ring, NFDK_TX_DESC_STOP_CNT * 2);
23 static int nfp_nfdk_tx_ring_should_stop(struct nfp_net_tx_ring *tx_ring)
25 	return nfp_net_tx_full(tx_ring, NFDK_TX_DESC_STOP_CNT);
29 				  struct nfp_net_tx_ring *tx_ring)
35 	if (unlikely(nfp_nfdk_tx_ring_should_wake(tx_ring)))
119 nfp_nfdk_tx_maybe_close_block(struct nfp_net_tx_ring *tx_ring,
151 	if (round_down(tx_ring->wr_p, NFDK_TX_DESC_BLOCK_CNT) !=
152 	    round_down(tx_ring->wr_p + n_descs, NFDK_TX_DESC_BLOCK_CNT))
155 	if ((u32)tx_ring->data_pending + skb->len > NFDK_TX_MAX_DATA_PER_BLOCK)
161 	wr_p = tx_ring->wr_p;
164 	wr_idx = D_IDX(tx_ring, wr_p);
165 	tx_ring->ktxbufs[wr_idx].skb = NULL;
166 	txd = &tx_ring->ktxds[wr_idx];
170 	tx_ring->data_pending = 0;
171 	tx_ring->wr_p += nop_slots;
172 	tx_ring->wr_ptr_add += nop_slots;
258 	struct nfp_net_tx_ring *tx_ring;
273 	tx_ring = &dp->tx_rings[qidx];
274 	r_vec = tx_ring->r_vec;
278 	if (unlikely(nfp_net_tx_full(tx_ring, NFDK_TX_DESC_STOP_CNT))) {
280 			   qidx, tx_ring->wr_p, tx_ring->rd_p);
282 		nfp_net_tx_xmit_more_flush(tx_ring);
293 	if (nfp_nfdk_tx_maybe_close_block(tx_ring, skb))
301 	wr_idx = D_IDX(tx_ring, tx_ring->wr_p);
302 	txd = &tx_ring->ktxds[wr_idx];
303 	txbuf = &tx_ring->ktxbufs[wr_idx];
409 	cnt = txd - tx_ring->ktxds - wr_idx;
416 	tx_ring->wr_p += cnt;
417 	if (tx_ring->wr_p % NFDK_TX_DESC_BLOCK_CNT)
418 		tx_ring->data_pending += skb->len;
420 		tx_ring->data_pending = 0;
422 	if (nfp_nfdk_tx_ring_should_stop(tx_ring))
423 		nfp_nfdk_tx_ring_stop(nd_q, tx_ring);
425 	tx_ring->wr_ptr_add += cnt;
427 		nfp_net_tx_xmit_more_flush(tx_ring);
440 	txbuf = &tx_ring->ktxbufs[wr_idx + 1];
458 	nfp_net_tx_xmit_more_flush(tx_ring);
468  * @tx_ring:	TX ring structure
471 static void nfp_nfdk_tx_complete(struct nfp_net_tx_ring *tx_ring, int budget)
473 	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;
482 	rd_p = tx_ring->rd_p;
483 	if (tx_ring->wr_p == rd_p)
487 	qcp_rd_p = nfp_net_read_tx_cmpl(tx_ring, dp);
489 	if (qcp_rd_p == tx_ring->qcp_rd_p)
492 	todo = D_IDX(tx_ring, qcp_rd_p - tx_ring->qcp_rd_p);
493 	ktxbufs = tx_ring->ktxbufs;
501 		txbuf = &ktxbufs[D_IDX(tx_ring, rd_p)];
544 	tx_ring->rd_p = rd_p;
545 	tx_ring->qcp_rd_p = qcp_rd_p;
555 	nd_q = netdev_get_tx_queue(dp->netdev, tx_ring->idx);
557 	if (nfp_nfdk_tx_ring_should_wake(tx_ring)) {
558 		/* Make sure TX thread will see updated tx_ring->rd_p */
565 	WARN_ONCE(tx_ring->wr_p - tx_ring->rd_p > tx_ring->cnt,
567 		  tx_ring->rd_p, tx_ring->wr_p, tx_ring->cnt);
835 static bool nfp_nfdk_xdp_complete(struct nfp_net_tx_ring *tx_ring)
837 	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;
845 	qcp_rd_p = nfp_net_read_tx_cmpl(tx_ring, dp);
846 	if (qcp_rd_p == tx_ring->qcp_rd_p)
849 	todo = D_IDX(tx_ring, qcp_rd_p - tx_ring->qcp_rd_p);
856 		int idx = D_IDX(tx_ring, tx_ring->rd_p + done);
860 		txbuf = &tx_ring->ktxbufs[idx];
888 	tx_ring->qcp_rd_p = D_IDX(tx_ring, tx_ring->qcp_rd_p + done);
889 	tx_ring->rd_p += done;
891 	WARN_ONCE(tx_ring->wr_p - tx_ring->rd_p > tx_ring->cnt,
893 		  tx_ring->rd_p, tx_ring->wr_p, tx_ring->cnt);
900 		    struct nfp_net_tx_ring *tx_ring,
918 	 * won't wrap around the tx_ring.
920 	if (unlikely(nfp_net_tx_full(tx_ring, NFDK_TX_DESC_STOP_CNT))) {
922 			nfp_nfdk_xdp_complete(tx_ring);
926 		if (unlikely(nfp_net_tx_full(tx_ring, NFDK_TX_DESC_STOP_CNT))) {
935 	if ((round_down(tx_ring->wr_p, NFDK_TX_DESC_BLOCK_CNT) !=
936 	     round_down(tx_ring->wr_p + n_descs, NFDK_TX_DESC_BLOCK_CNT)) ||
937 	    ((u32)tx_ring->data_pending + pkt_len >
939 		unsigned int nop_slots = D_BLOCK_CPL(tx_ring->wr_p);
941 		wr_idx = D_IDX(tx_ring, tx_ring->wr_p);
942 		txd = &tx_ring->ktxds[wr_idx];
946 		tx_ring->data_pending = 0;
947 		tx_ring->wr_p += nop_slots;
948 		tx_ring->wr_ptr_add += nop_slots;
951 	wr_idx = D_IDX(tx_ring, tx_ring->wr_p);
953 	txbuf = &tx_ring->ktxbufs[wr_idx];
963 	txd = &tx_ring->ktxds[wr_idx];
1005 	cnt = txd - tx_ring->ktxds - wr_idx;
1006 	tx_ring->wr_p += cnt;
1007 	if (tx_ring->wr_p % NFDK_TX_DESC_BLOCK_CNT)
1008 		tx_ring->data_pending += pkt_len;
1010 		tx_ring->data_pending = 0;
1012 	tx_ring->wr_ptr_add += cnt;
1031 	struct nfp_net_tx_ring *tx_ring;
1044 	tx_ring = r_vec->xdp_ring;
1148 								  tx_ring,
1247 		if (tx_ring->wr_ptr_add)
1248 			nfp_net_tx_xmit_more_flush(tx_ring);
1249 		else if (unlikely(tx_ring->wr_p != tx_ring->rd_p) &&
1251 			if (!nfp_nfdk_xdp_complete(tx_ring))
1271 	if (r_vec->tx_ring)
1272 		nfp_nfdk_tx_complete(r_vec->tx_ring, budget);
1295 	if (r_vec->nfp_net->tx_coalesce_adapt_on && r_vec->tx_ring) {
1321 	struct nfp_net_tx_ring *tx_ring;
1331 	tx_ring = r_vec->tx_ring;
1339 	if (unlikely(nfp_net_tx_full(tx_ring, NFDK_TX_DESC_STOP_CNT))) {
1363 	if (nfp_nfdk_tx_maybe_close_block(tx_ring, skb))
1367 	wr_idx = D_IDX(tx_ring, tx_ring->wr_p);
1368 	txd = &tx_ring->ktxds[wr_idx];
1369 	txbuf = &tx_ring->ktxbufs[wr_idx];
1419 	cnt = txd - tx_ring->ktxds - wr_idx;
1424 	tx_ring->wr_p += cnt;
1425 	if (tx_ring->wr_p % NFDK_TX_DESC_BLOCK_CNT)
1426 		tx_ring->data_pending += skb->len;
1428 		tx_ring->data_pending = 0;
1430 	tx_ring->wr_ptr_add += cnt;
1431 	nfp_net_tx_xmit_more_flush(tx_ring);
1572 	nfp_nfdk_tx_complete(r_vec->tx_ring, 0);