mlx5/mlx5_en/mlx5_en_tx.c

32 mlx5e_do_send_cqe(struct mlx5e_sq *sq)
34 	sq->cev_counter++;
36 	if (sq->cev_counter >= sq->cev_factor) {
37 		sq->cev_counter = 0;
44 mlx5e_send_nop(struct mlx5e_sq *sq, u32 ds_cnt)
46 	u16 pi = sq->pc & sq->wq.sz_m1;
47 	struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
51 	wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP);
52 	wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
53 	if (mlx5e_do_send_cqe(sq))
59 	memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32));
61 	sq->mbuf[pi].mbuf = NULL;
62 	sq->mbuf[pi].num_bytes = 0;
63 	sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
64 	sq->pc += sq->mbuf[pi].num_wqebbs;
84 	struct mlx5e_sq *sq;
124 	sq = &priv->channel[ch].sq[tc];
125 	if (likely(READ_ONCE(sq->running) != 0))
126 		return (sq);
131 mlx5e_get_l2_header_size(struct mlx5e_sq *sq, struct mbuf *mb)
173 	return (MIN(mb->m_pkthdr.len, sq->max_inline));
266 mlx5e_sq_xmit(struct mlx5e_sq *sq, struct mbuf **mbp)
282 	if (unlikely(!mlx5e_sq_has_room_for(sq, 2 * MLX5_SEND_WQE_MAX_WQEBBS))) {
283 		sq->stats.enobuf++;
288 	pi = ((~sq->pc) & sq->wq.sz_m1);
291 		mlx5e_send_nop(sq, (pi + 1) * MLX5_SEND_WQEBB_NUM_DS);
292 		pi = ((~sq->pc) & sq->wq.sz_m1);
294 			sq->stats.enobuf++;
300 	pi = sq->pc & sq->wq.sz_m1;
301 	wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
302 	ifp = sq->ifp;
317 		sq->stats.csum_offload_none++;
336 		sq->mbuf[pi].num_bytes = payload_len + (num_pkts * ihs);
338 		sq->stats.tso_packets++;
339 		sq->stats.tso_bytes += payload_len;
343 		switch (sq->min_inline_mode) {
348 				ihs = mlx5e_get_l2_header_size(sq, mb);
351 			ihs = mlx5e_get_l2_header_size(sq, mb);
357 			    (sq->min_insert_caps & MLX5E_INSERT_VLAN) != 0) {
363 				   (sq->min_insert_caps & MLX5E_INSERT_NON_VLAN) != 0) {
368 				ihs = mlx5e_get_l2_header_size(sq, mb);
372 		sq->mbuf[pi].num_bytes = max_t (unsigned int,
383 		if (unlikely(ihs > (sq->max_inline - ETHER_VLAN_ENCAP_LEN))) {
388 			ihs = (sq->max_inline - ETHER_VLAN_ENCAP_LEN);
407 		if (unlikely(ihs > sq->max_inline)) {
412 			ihs = sq->max_inline;
426 	err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map,
430 		sq->stats.defragged++;
438 		err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map,
447 		bus_dmamap_sync(sq->dma_tag, sq->mbuf[pi].dma_map,
451 		bus_dmamap_unload(sq->dma_tag, sq->mbuf[pi].dma_map);
460 		dseg->lkey = sq->mkey_be;
467 	wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode);
468 	wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
469 	if (mlx5e_do_send_cqe(sq))
475 	memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32));
478 	sq->mbuf[pi].mbuf = mb;
479 	sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
480 	sq->pc += sq->mbuf[pi].num_wqebbs;
483 	sq->stats.packets++;
484 	sq->stats.bytes += sq->mbuf[pi].num_bytes;
490 	sq->stats.dropped++;
497 mlx5e_poll_tx_cq(struct mlx5e_sq *sq, int budget)
502 	 * sq->cc must be updated only after mlx5_cqwq_update_db_record(),
505 	sqcc = sq->cc;
513 		cqe = mlx5e_get_cqe(&sq->cq);
517 		mlx5_cqwq_pop(&sq->cq.wq);
520 		budget -= sq->cev_factor;
522 		for (x = 0; x != sq->cev_factor; x++) {
523 			ci = sqcc & sq->wq.sz_m1;
524 			mb = sq->mbuf[ci].mbuf;
525 			sq->mbuf[ci].mbuf = NULL;
528 				if (sq->mbuf[ci].num_bytes == 0) {
530 					sq->stats.nop++;
533 				bus_dmamap_sync(sq->dma_tag, sq->mbuf[ci].dma_map,
535 				bus_dmamap_unload(sq->dma_tag, sq->mbuf[ci].dma_map);
540 			sqcc += sq->mbuf[ci].num_wqebbs;
544 	mlx5_cqwq_update_db_record(&sq->cq.wq);
549 	sq->cc = sqcc;
553 mlx5e_xmit_locked(struct ifnet *ifp, struct mlx5e_sq *sq, struct mbuf *mb)
558 	    READ_ONCE(sq->running) == 0)) {
564 	if (mlx5e_sq_xmit(sq, &mb) != 0) {
571 	if (likely(sq->doorbell.d64 != 0)) {
572 		mlx5e_tx_notify_hw(sq, sq->doorbell.d32, 0);
573 		sq->doorbell.d64 = 0;
580 	if (unlikely(sq->cev_next_state == MLX5E_CEV_STATE_INITIAL &&
581 	    sq->cev_factor != 1)) {
583 		mlx5e_sq_cev_timeout(sq);
586 		sq->cev_next_state = MLX5E_CEV_STATE_HOLD_NOPS;
594 	struct mlx5e_sq *sq;
597 	sq = mlx5e_select_queue(ifp, mb);
598 	if (unlikely(sq == NULL)) {
604 	mtx_lock(&sq->lock);
605 	ret = mlx5e_xmit_locked(ifp, sq, mb);
606 	mtx_unlock(&sq->lock);
614 	struct mlx5e_sq *sq = container_of(mcq, struct mlx5e_sq, cq.mcq);
616 	mtx_lock(&sq->comp_lock);
617 	mlx5e_poll_tx_cq(sq, MLX5E_BUDGET_MAX);
618 	mlx5e_cq_arm(&sq->cq, MLX5_GET_DOORBELL_LOCK(&sq->priv->doorbell_lock));
619 	mtx_unlock(&sq->comp_lock);