mlx5_en_tx.c revision 291184 
1/*-
2 * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 *    notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 *    notice, this list of conditions and the following disclaimer in the
11 *    documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 * $FreeBSD: stable/10/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c 291184 2015-11-23 09:32:32Z hselasky $
26 */
27
28#include "en.h"
29#include <machine/atomic.h>
30
31void
32mlx5e_send_nop(struct mlx5e_sq *sq, u32 ds_cnt, bool notify_hw)
33{
34	u16 pi = sq->pc & sq->wq.sz_m1;
35	struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
36
37	memset(&wqe->ctrl, 0, sizeof(wqe->ctrl));
38
39	wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP);
40	wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
41	wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
42
43	sq->mbuf[pi].mbuf = NULL;
44	sq->mbuf[pi].num_bytes = 0;
45	sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
46	sq->pc += sq->mbuf[pi].num_wqebbs;
47	if (notify_hw)
48		mlx5e_tx_notify_hw(sq, wqe, 0);
49}
50
51#if (__FreeBSD_version >= 1100000)
52static uint32_t mlx5e_hash_value;
53
54static void
55mlx5e_hash_init(void *arg)
56{
57	mlx5e_hash_value = m_ether_tcpip_hash_init();
58}
59
60/* Make kernel call mlx5e_hash_init after the random stack finished initializing */
61SYSINIT(mlx5e_hash_init, SI_SUB_RANDOM, SI_ORDER_ANY, &mlx5e_hash_init, NULL);
62#endif
63
64static struct mlx5e_sq *
65mlx5e_select_queue(struct ifnet *ifp, struct mbuf *mb)
66{
67	struct mlx5e_priv *priv = ifp->if_softc;
68	u32 ch;
69	u32 tc;
70
71	/* check if channels are successfully opened */
72	if (unlikely(priv->channel == NULL))
73		return (NULL);
74
75	/* obtain VLAN information if present */
76	if (mb->m_flags & M_VLANTAG) {
77		tc = (mb->m_pkthdr.ether_vtag >> 13);
78		if (tc >= priv->num_tc)
79			tc = priv->default_vlan_prio;
80	} else {
81		tc = priv->default_vlan_prio;
82	}
83
84	ch = priv->params.num_channels;
85
86	/* check if flowid is set */
87	if (M_HASHTYPE_GET(mb) != M_HASHTYPE_NONE) {
88		ch = (mb->m_pkthdr.flowid % 128) % ch;
89	} else {
90#if (__FreeBSD_version >= 1100000)
91		ch = m_ether_tcpip_hash(MBUF_HASHFLAG_L3 |
92		    MBUF_HASHFLAG_L4, mb, mlx5e_hash_value) % ch;
93#else
94		/*
95		 * m_ether_tcpip_hash not present in stable, so just
96		 * throw unhashed mbufs on queue 0
97		 */
98		ch = 0;
99#endif
100	}
101
102	/* check if channel is allocated */
103	if (unlikely(priv->channel[ch] == NULL))
104		return (NULL);
105
106	return (&priv->channel[ch]->sq[tc]);
107}
108
109static inline u16
110mlx5e_get_inline_hdr_size(struct mlx5e_sq *sq, struct mbuf *mb)
111{
112	return (MIN(MLX5E_MAX_TX_INLINE, mb->m_len));
113}
114
115static int
116mlx5e_get_header_size(struct mbuf *mb)
117{
118	struct ether_vlan_header *eh;
119	struct tcphdr *th;
120	struct ip *ip;
121	int ip_hlen, tcp_hlen;
122	struct ip6_hdr *ip6;
123	uint16_t eth_type;
124	int eth_hdr_len;
125
126	eh = mtod(mb, struct ether_vlan_header *);
127	if (mb->m_len < ETHER_HDR_LEN)
128		return (0);
129	if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
130		eth_type = ntohs(eh->evl_proto);
131		eth_hdr_len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
132	} else {
133		eth_type = ntohs(eh->evl_encap_proto);
134		eth_hdr_len = ETHER_HDR_LEN;
135	}
136	if (mb->m_len < eth_hdr_len)
137		return (0);
138	switch (eth_type) {
139	case ETHERTYPE_IP:
140		ip = (struct ip *)(mb->m_data + eth_hdr_len);
141		if (mb->m_len < eth_hdr_len + sizeof(*ip))
142			return (0);
143		if (ip->ip_p != IPPROTO_TCP)
144			return (0);
145		ip_hlen = ip->ip_hl << 2;
146		eth_hdr_len += ip_hlen;
147		break;
148	case ETHERTYPE_IPV6:
149		ip6 = (struct ip6_hdr *)(mb->m_data + eth_hdr_len);
150		if (mb->m_len < eth_hdr_len + sizeof(*ip6))
151			return (0);
152		if (ip6->ip6_nxt != IPPROTO_TCP)
153			return (0);
154		eth_hdr_len += sizeof(*ip6);
155		break;
156	default:
157		return (0);
158	}
159	if (mb->m_len < eth_hdr_len + sizeof(*th))
160		return (0);
161	th = (struct tcphdr *)(mb->m_data + eth_hdr_len);
162	tcp_hlen = th->th_off << 2;
163	eth_hdr_len += tcp_hlen;
164	if (mb->m_len < eth_hdr_len)
165		return (0);
166	return (eth_hdr_len);
167}
168
169/*
170 * The return value is not going back to the stack because of
171 * the drbr
172 */
173static int
174mlx5e_sq_xmit(struct mlx5e_sq *sq, struct mbuf **mbp)
175{
176	bus_dma_segment_t segs[MLX5E_MAX_TX_MBUF_FRAGS];
177	struct mlx5_wqe_data_seg *dseg;
178	struct mlx5e_tx_wqe *wqe;
179	struct ifnet *ifp;
180	int nsegs;
181	int err;
182	int x;
183	struct mbuf *mb = *mbp;
184	u16 ds_cnt;
185	u16 ihs;
186	u16 pi;
187	u8 opcode;
188
189	/*
190	 * Return ENOBUFS if the queue is full, this may trigger reinsertion
191	 * of the mbuf into the drbr (see mlx5e_xmit_locked)
192	 */
193	if (unlikely(!mlx5e_sq_has_room_for(sq, 2 * MLX5_SEND_WQE_MAX_WQEBBS))) {
194		return (ENOBUFS);
195	}
196
197	/* Align SQ edge with NOPs to avoid WQE wrap around */
198	pi = ((~sq->pc) & sq->wq.sz_m1);
199	if (pi < (MLX5_SEND_WQE_MAX_WQEBBS - 1)) {
200		/* Send one multi NOP message instead of many */
201		mlx5e_send_nop(sq, (pi + 1) * MLX5_SEND_WQEBB_NUM_DS, false);
202		pi = ((~sq->pc) & sq->wq.sz_m1);
203		if (pi < (MLX5_SEND_WQE_MAX_WQEBBS - 1)) {
204			m_freem(mb);
205			return (ENOMEM);
206		}
207	}
208
209	/* Setup local variables */
210	pi = sq->pc & sq->wq.sz_m1;
211	wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
212	ifp = sq->channel->ifp;
213
214	memset(wqe, 0, sizeof(*wqe));
215
216	/* Send a copy of the frame to the BPF listener, if any */
217	if (ifp != NULL && ifp->if_bpf != NULL)
218		ETHER_BPF_MTAP(ifp, mb);
219
220	if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)) {
221		wqe->eth.cs_flags |= MLX5_ETH_WQE_L3_CSUM;
222	}
223	if (mb->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) {
224		wqe->eth.cs_flags |= MLX5_ETH_WQE_L4_CSUM;
225	}
226	if (wqe->eth.cs_flags == 0) {
227		sq->stats.csum_offload_none++;
228	}
229	if (mb->m_pkthdr.csum_flags & CSUM_TSO) {
230		u32 payload_len;
231		u32 mss = mb->m_pkthdr.tso_segsz;
232		u32 num_pkts;
233
234		wqe->eth.mss = cpu_to_be16(mss);
235		opcode = MLX5_OPCODE_LSO;
236		ihs = mlx5e_get_header_size(mb);
237		payload_len = mb->m_pkthdr.len - ihs;
238		if (payload_len == 0)
239			num_pkts = 1;
240		else
241			num_pkts = DIV_ROUND_UP(payload_len, mss);
242		sq->mbuf[pi].num_bytes = payload_len + (num_pkts * ihs);
243
244		sq->stats.tso_packets++;
245		sq->stats.tso_bytes += payload_len;
246	} else {
247		opcode = MLX5_OPCODE_SEND;
248		ihs = mlx5e_get_inline_hdr_size(sq, mb);
249		sq->mbuf[pi].num_bytes = max_t (unsigned int,
250		    mb->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN);
251	}
252	if (mb->m_flags & M_VLANTAG) {
253		struct ether_vlan_header *eh =
254		    (struct ether_vlan_header *)wqe->eth.inline_hdr_start;
255
256		/* Range checks */
257		if (ihs > (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN))
258			ihs = (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN);
259		else if (ihs < ETHER_HDR_LEN) {
260			err = EINVAL;
261			goto tx_drop;
262		}
263		m_copydata(mb, 0, ETHER_HDR_LEN, (caddr_t)eh);
264		m_adj(mb, ETHER_HDR_LEN);
265		/* Insert 4 bytes VLAN tag into data stream */
266		eh->evl_proto = eh->evl_encap_proto;
267		eh->evl_encap_proto = htons(ETHERTYPE_VLAN);
268		eh->evl_tag = htons(mb->m_pkthdr.ether_vtag);
269		/* Copy rest of header data, if any */
270		m_copydata(mb, 0, ihs - ETHER_HDR_LEN, (caddr_t)(eh + 1));
271		m_adj(mb, ihs - ETHER_HDR_LEN);
272		/* Extend header by 4 bytes */
273		ihs += ETHER_VLAN_ENCAP_LEN;
274	} else {
275		m_copydata(mb, 0, ihs, wqe->eth.inline_hdr_start);
276		m_adj(mb, ihs);
277	}
278
279	wqe->eth.inline_hdr_sz = cpu_to_be16(ihs);
280
281	ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
282	if (likely(ihs > sizeof(wqe->eth.inline_hdr_start))) {
283		ds_cnt += DIV_ROUND_UP(ihs - sizeof(wqe->eth.inline_hdr_start),
284		    MLX5_SEND_WQE_DS);
285	}
286	dseg = ((struct mlx5_wqe_data_seg *)&wqe->ctrl) + ds_cnt;
287
288	/* Trim off empty mbufs */
289	while (mb->m_len == 0) {
290		mb = m_free(mb);
291		/* Check if all data has been inlined */
292		if (mb == NULL)
293			goto skip_dma;
294	}
295
296	err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map,
297	    mb, segs, &nsegs, BUS_DMA_NOWAIT);
298	if (err == EFBIG) {
299		/*
300		 * Update *mbp before defrag in case it was trimmed in the
301		 * loop above
302		 */
303		*mbp = mb;
304		/* Update statistics */
305		sq->stats.defragged++;
306		/* Too many mbuf fragments */
307		mb = m_defrag(*mbp, M_NOWAIT);
308		if (mb == NULL) {
309			mb = *mbp;
310			goto tx_drop;
311		}
312		/* Try again */
313		err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map,
314		    mb, segs, &nsegs, BUS_DMA_NOWAIT);
315	}
316	/* Catch errors */
317	if (err != 0) {
318		goto tx_drop;
319	}
320	*mbp = mb;
321
322	for (x = 0; x != nsegs; x++) {
323		if (segs[x].ds_len == 0)
324			continue;
325		dseg->addr = cpu_to_be64((uint64_t)segs[x].ds_addr);
326		dseg->lkey = sq->mkey_be;
327		dseg->byte_count = cpu_to_be32((uint32_t)segs[x].ds_len);
328		dseg++;
329	}
330skip_dma:
331	ds_cnt = (dseg - ((struct mlx5_wqe_data_seg *)&wqe->ctrl));
332
333	wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode);
334	wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
335	wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
336
337	/* Store pointer to mbuf */
338	sq->mbuf[pi].mbuf = mb;
339	sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
340	sq->pc += sq->mbuf[pi].num_wqebbs;
341
342	/* Make sure all mbuf data is written to RAM */
343	if (mb != NULL)
344		bus_dmamap_sync(sq->dma_tag, sq->mbuf[pi].dma_map, BUS_DMASYNC_PREWRITE);
345
346	mlx5e_tx_notify_hw(sq, wqe, 0);
347
348	sq->stats.packets++;
349	return (0);
350
351tx_drop:
352	sq->stats.dropped++;
353	*mbp = NULL;
354	m_freem(mb);
355	return err;
356}
357
358static void
359mlx5e_poll_tx_cq(struct mlx5e_sq *sq, int budget)
360{
361	u16 sqcc;
362
363	/*
364	 * sq->cc must be updated only after mlx5_cqwq_update_db_record(),
365	 * otherwise a cq overrun may occur
366	 */
367	sqcc = sq->cc;
368
369	while (budget--) {
370		struct mlx5_cqe64 *cqe;
371		struct mbuf *mb;
372		u16 ci;
373
374		cqe = mlx5e_get_cqe(&sq->cq);
375		if (!cqe)
376			break;
377
378		ci = sqcc & sq->wq.sz_m1;
379		mb = sq->mbuf[ci].mbuf;
380		sq->mbuf[ci].mbuf = NULL;	/* Safety clear */
381
382		if (mb == NULL) {
383			if (sq->mbuf[ci].num_bytes == 0) {
384				/* NOP */
385				sq->stats.nop++;
386			}
387		} else {
388			bus_dmamap_sync(sq->dma_tag, sq->mbuf[ci].dma_map,
389			    BUS_DMASYNC_POSTWRITE);
390			bus_dmamap_unload(sq->dma_tag, sq->mbuf[ci].dma_map);
391
392			/* Free transmitted mbuf */
393			m_freem(mb);
394		}
395		sqcc += sq->mbuf[ci].num_wqebbs;
396	}
397
398	mlx5_cqwq_update_db_record(&sq->cq.wq);
399
400	/* Ensure cq space is freed before enabling more cqes */
401	wmb();
402
403	sq->cc = sqcc;
404
405	if (atomic_cmpset_int(&sq->queue_state, MLX5E_SQ_FULL, MLX5E_SQ_READY))
406		taskqueue_enqueue(sq->sq_tq, &sq->sq_task);
407}
408
409static int
410mlx5e_xmit_locked(struct ifnet *ifp, struct mlx5e_sq *sq, struct mbuf *mb)
411{
412	struct mbuf *next;
413	int err = 0;
414
415	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
416		if (mb)
417			err = drbr_enqueue(ifp, sq->br, mb);
418		return (err);
419	}
420
421	if (mb != NULL)
422		/*
423		 * If we can't insert mbuf into drbr, try to xmit anyway.
424		 * We keep the error we got so we could return that after xmit.
425		 */
426		err = drbr_enqueue(ifp, sq->br, mb);
427
428	/* Process the queue */
429	while ((next = drbr_peek(ifp, sq->br)) != NULL) {
430		if (mlx5e_sq_xmit(sq, &next) != 0) {
431			if (next == NULL) {
432				drbr_advance(ifp, sq->br);
433			} else {
434				drbr_putback(ifp, sq->br, next);
435				atomic_store_rel_int(&sq->queue_state, MLX5E_SQ_FULL);
436			}
437			break;
438		}
439		drbr_advance(ifp, sq->br);
440		if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
441			break;
442	}
443	return (err);
444}
445
446int
447mlx5e_xmit(struct ifnet *ifp, struct mbuf *mb)
448{
449	struct mlx5e_sq *sq;
450	int ret;
451
452	sq = mlx5e_select_queue(ifp, mb);
453	if (unlikely(sq == NULL)) {
454		/* Invalid send queue */
455		m_freem(mb);
456		return (ENXIO);
457	}
458	if (mtx_trylock(&sq->lock)) {
459		ret = mlx5e_xmit_locked(ifp, sq, mb);
460		mtx_unlock(&sq->lock);
461	} else {
462		ret = drbr_enqueue(ifp, sq->br, mb);
463		taskqueue_enqueue(sq->sq_tq, &sq->sq_task);
464	}
465
466	return (ret);
467}
468
469void
470mlx5e_tx_cq_comp(struct mlx5_core_cq *mcq)
471{
472	struct mlx5e_sq *sq = container_of(mcq, struct mlx5e_sq, cq.mcq);
473
474	mtx_lock(&sq->comp_lock);
475	mlx5e_poll_tx_cq(sq, MLX5E_BUDGET_MAX);
476	mlx5e_cq_arm(&sq->cq);
477	mtx_unlock(&sq->comp_lock);
478}
479
480void
481mlx5e_tx_que(void *context, int pending)
482{
483	struct mlx5e_sq *sq = context;
484	struct ifnet *ifp = sq->channel->ifp;
485
486	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
487		mtx_lock(&sq->lock);
488		if (!drbr_empty(ifp, sq->br))
489			mlx5e_xmit_locked(ifp, sq, NULL);
490		mtx_unlock(&sq->lock);
491	}
492}
493