mlx5_en_rx.c revision 322537 
1/*-
2 * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 *    notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 *    notice, this list of conditions and the following disclaimer in the
11 *    documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 * $FreeBSD: stable/11/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c 322537 2017-08-15 12:30:11Z hselasky $
26 */
27
28#include "en.h"
29#include <machine/in_cksum.h>
30
31static inline int
32mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq,
33    struct mlx5e_rx_wqe *wqe, u16 ix)
34{
35	bus_dma_segment_t segs[1];
36	struct mbuf *mb;
37	int nsegs;
38	int err;
39
40	if (rq->mbuf[ix].mbuf != NULL)
41		return (0);
42
43	mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rq->wqe_sz);
44	if (unlikely(!mb))
45		return (-ENOMEM);
46
47	/* set initial mbuf length */
48	mb->m_pkthdr.len = mb->m_len = rq->wqe_sz;
49
50	/* get IP header aligned */
51	m_adj(mb, MLX5E_NET_IP_ALIGN);
52
53	err = -bus_dmamap_load_mbuf_sg(rq->dma_tag, rq->mbuf[ix].dma_map,
54	    mb, segs, &nsegs, BUS_DMA_NOWAIT);
55	if (err != 0)
56		goto err_free_mbuf;
57	if (unlikely(nsegs != 1)) {
58		bus_dmamap_unload(rq->dma_tag, rq->mbuf[ix].dma_map);
59		err = -ENOMEM;
60		goto err_free_mbuf;
61	}
62	wqe->data.addr = cpu_to_be64(segs[0].ds_addr);
63
64	rq->mbuf[ix].mbuf = mb;
65	rq->mbuf[ix].data = mb->m_data;
66
67	bus_dmamap_sync(rq->dma_tag, rq->mbuf[ix].dma_map,
68	    BUS_DMASYNC_PREREAD);
69	return (0);
70
71err_free_mbuf:
72	m_freem(mb);
73	return (err);
74}
75
76static void
77mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
78{
79	if (unlikely(rq->enabled == 0))
80		return;
81
82	while (!mlx5_wq_ll_is_full(&rq->wq)) {
83		struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, rq->wq.head);
84
85		if (unlikely(mlx5e_alloc_rx_wqe(rq, wqe, rq->wq.head))) {
86			callout_reset_curcpu(&rq->watchdog, 1, (void *)&mlx5e_post_rx_wqes, rq);
87			break;
88		}
89		mlx5_wq_ll_push(&rq->wq, be16_to_cpu(wqe->next.next_wqe_index));
90	}
91
92	/* ensure wqes are visible to device before updating doorbell record */
93	wmb();
94
95	mlx5_wq_ll_update_db_record(&rq->wq);
96}
97
98static void
99mlx5e_lro_update_hdr(struct mbuf *mb, struct mlx5_cqe64 *cqe)
100{
101	/* TODO: consider vlans, ip options, ... */
102	struct ether_header *eh;
103	uint16_t eh_type;
104	uint16_t tot_len;
105	struct ip6_hdr *ip6 = NULL;
106	struct ip *ip4 = NULL;
107	struct tcphdr *th;
108	uint32_t *ts_ptr;
109	uint8_t l4_hdr_type;
110	int tcp_ack;
111
112	eh = mtod(mb, struct ether_header *);
113	eh_type = ntohs(eh->ether_type);
114
115	l4_hdr_type = get_cqe_l4_hdr_type(cqe);
116	tcp_ack = ((CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA == l4_hdr_type) ||
117	    (CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA == l4_hdr_type));
118
119	/* TODO: consider vlan */
120	tot_len = be32_to_cpu(cqe->byte_cnt) - ETHER_HDR_LEN;
121
122	switch (eh_type) {
123	case ETHERTYPE_IP:
124		ip4 = (struct ip *)(eh + 1);
125		th = (struct tcphdr *)(ip4 + 1);
126		break;
127	case ETHERTYPE_IPV6:
128		ip6 = (struct ip6_hdr *)(eh + 1);
129		th = (struct tcphdr *)(ip6 + 1);
130		break;
131	default:
132		return;
133	}
134
135	ts_ptr = (uint32_t *)(th + 1);
136
137	if (get_cqe_lro_tcppsh(cqe))
138		th->th_flags |= TH_PUSH;
139
140	if (tcp_ack) {
141		th->th_flags |= TH_ACK;
142		th->th_ack = cqe->lro_ack_seq_num;
143		th->th_win = cqe->lro_tcp_win;
144
145		/*
146		 * FreeBSD handles only 32bit aligned timestamp right after
147		 * the TCP hdr
148		 * +--------+--------+--------+--------+
149		 * |   NOP  |  NOP   |  TSopt |   10   |
150		 * +--------+--------+--------+--------+
151		 * |          TSval   timestamp        |
152		 * +--------+--------+--------+--------+
153		 * |          TSecr   timestamp        |
154		 * +--------+--------+--------+--------+
155		 */
156		if (get_cqe_lro_timestamp_valid(cqe) &&
157		    (__predict_true(*ts_ptr) == ntohl(TCPOPT_NOP << 24 |
158		    TCPOPT_NOP << 16 | TCPOPT_TIMESTAMP << 8 |
159		    TCPOLEN_TIMESTAMP))) {
160			/*
161			 * cqe->timestamp is 64bit long.
162			 * [0-31] - timestamp.
163			 * [32-64] - timestamp echo replay.
164			 */
165			ts_ptr[1] = *(uint32_t *)&cqe->timestamp;
166			ts_ptr[2] = *((uint32_t *)&cqe->timestamp + 1);
167		}
168	}
169	if (ip4) {
170		ip4->ip_ttl = cqe->lro_min_ttl;
171		ip4->ip_len = cpu_to_be16(tot_len);
172		ip4->ip_sum = 0;
173		ip4->ip_sum = in_cksum(mb, ip4->ip_hl << 2);
174	} else {
175		ip6->ip6_hlim = cqe->lro_min_ttl;
176		ip6->ip6_plen = cpu_to_be16(tot_len -
177		    sizeof(struct ip6_hdr));
178	}
179	/* TODO: handle tcp checksum */
180}
181
182static inline void
183mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe,
184    struct mlx5e_rq *rq, struct mbuf *mb,
185    u32 cqe_bcnt)
186{
187	struct ifnet *ifp = rq->ifp;
188	int lro_num_seg;	/* HW LRO session aggregated packets counter */
189
190	lro_num_seg = be32_to_cpu(cqe->srqn) >> 24;
191	if (lro_num_seg > 1) {
192		mlx5e_lro_update_hdr(mb, cqe);
193		rq->stats.lro_packets++;
194		rq->stats.lro_bytes += cqe_bcnt;
195	}
196
197	mb->m_pkthdr.len = mb->m_len = cqe_bcnt;
198	/* check if a Toeplitz hash was computed */
199	if (cqe->rss_hash_type != 0) {
200		mb->m_pkthdr.flowid = be32_to_cpu(cqe->rss_hash_result);
201#ifdef RSS
202		/* decode the RSS hash type */
203		switch (cqe->rss_hash_type &
204		    (CQE_RSS_DST_HTYPE_L4 | CQE_RSS_DST_HTYPE_IP)) {
205		/* IPv4 */
206		case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV4):
207			M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV4);
208			break;
209		case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV4):
210			M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV4);
211			break;
212		case CQE_RSS_DST_HTYPE_IPV4:
213			M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV4);
214			break;
215		/* IPv6 */
216		case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV6):
217			M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV6);
218			break;
219		case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV6):
220			M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV6);
221			break;
222		case CQE_RSS_DST_HTYPE_IPV6:
223			M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV6);
224			break;
225		default:	/* Other */
226			M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE_HASH);
227			break;
228		}
229#else
230		M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE_HASH);
231#endif
232	} else {
233		mb->m_pkthdr.flowid = rq->ix;
234		M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE);
235	}
236	mb->m_pkthdr.rcvif = ifp;
237
238	if (likely(ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) &&
239	    ((cqe->hds_ip_ext & (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK)) ==
240	    (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK))) {
241		mb->m_pkthdr.csum_flags =
242		    CSUM_IP_CHECKED | CSUM_IP_VALID |
243		    CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
244		mb->m_pkthdr.csum_data = htons(0xffff);
245	} else {
246		rq->stats.csum_none++;
247	}
248
249	if (cqe_has_vlan(cqe)) {
250		mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->vlan_info);
251		mb->m_flags |= M_VLANTAG;
252	}
253}
254
255static inline void
256mlx5e_read_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data)
257{
258	memcpy(data, mlx5_cqwq_get_wqe(&cq->wq, (cc & cq->wq.sz_m1)),
259	    sizeof(struct mlx5_cqe64));
260}
261
262static inline void
263mlx5e_write_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data)
264{
265	memcpy(mlx5_cqwq_get_wqe(&cq->wq, cc & cq->wq.sz_m1),
266	    data, sizeof(struct mlx5_cqe64));
267}
268
269static inline void
270mlx5e_decompress_cqe(struct mlx5e_cq *cq, struct mlx5_cqe64 *title,
271    struct mlx5_mini_cqe8 *mini,
272    u16 wqe_counter, int i)
273{
274	/*
275	 * NOTE: The fields which are not set here are copied from the
276	 * initial and common title. See memcpy() in
277	 * mlx5e_write_cqe_slot().
278	 */
279	title->byte_cnt = mini->byte_cnt;
280	title->wqe_counter = cpu_to_be16((wqe_counter + i) & cq->wq.sz_m1);
281	title->check_sum = mini->checksum;
282	title->op_own = (title->op_own & 0xf0) |
283	    (((cq->wq.cc + i) >> cq->wq.log_sz) & 1);
284}
285
286#define MLX5E_MINI_ARRAY_SZ 8
287/* Make sure structs are not packet differently */
288CTASSERT(sizeof(struct mlx5_cqe64) ==
289    sizeof(struct mlx5_mini_cqe8) * MLX5E_MINI_ARRAY_SZ);
290static void
291mlx5e_decompress_cqes(struct mlx5e_cq *cq)
292{
293	struct mlx5_mini_cqe8 mini_array[MLX5E_MINI_ARRAY_SZ];
294	struct mlx5_cqe64 title;
295	u32 cqe_count;
296	u32 i = 0;
297	u16 title_wqe_counter;
298
299	mlx5e_read_cqe_slot(cq, cq->wq.cc, &title);
300	title_wqe_counter = be16_to_cpu(title.wqe_counter);
301	cqe_count = be32_to_cpu(title.byte_cnt);
302
303	/* Make sure we won't overflow */
304	KASSERT(cqe_count <= cq->wq.sz_m1,
305	    ("%s: cqe_count %u > cq->wq.sz_m1 %u", __func__,
306	    cqe_count, cq->wq.sz_m1));
307
308	mlx5e_read_cqe_slot(cq, cq->wq.cc + 1, mini_array);
309	while (true) {
310		mlx5e_decompress_cqe(cq, &title,
311		    &mini_array[i % MLX5E_MINI_ARRAY_SZ],
312		    title_wqe_counter, i);
313		mlx5e_write_cqe_slot(cq, cq->wq.cc + i, &title);
314		i++;
315
316		if (i == cqe_count)
317			break;
318		if (i % MLX5E_MINI_ARRAY_SZ == 0)
319			mlx5e_read_cqe_slot(cq, cq->wq.cc + i, mini_array);
320	}
321}
322
323static int
324mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget)
325{
326	int i;
327
328	for (i = 0; i < budget; i++) {
329		struct mlx5e_rx_wqe *wqe;
330		struct mlx5_cqe64 *cqe;
331		struct mbuf *mb;
332		__be16 wqe_counter_be;
333		u16 wqe_counter;
334		u32 byte_cnt;
335
336		cqe = mlx5e_get_cqe(&rq->cq);
337		if (!cqe)
338			break;
339
340		if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED)
341			mlx5e_decompress_cqes(&rq->cq);
342
343		mlx5_cqwq_pop(&rq->cq.wq);
344
345		wqe_counter_be = cqe->wqe_counter;
346		wqe_counter = be16_to_cpu(wqe_counter_be);
347		wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter);
348		byte_cnt = be32_to_cpu(cqe->byte_cnt);
349
350		bus_dmamap_sync(rq->dma_tag,
351		    rq->mbuf[wqe_counter].dma_map,
352		    BUS_DMASYNC_POSTREAD);
353
354		if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) {
355			rq->stats.wqe_err++;
356			goto wq_ll_pop;
357		}
358		if ((MHLEN - MLX5E_NET_IP_ALIGN) >= byte_cnt &&
359		    (mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) {
360			/* get IP header aligned */
361			mb->m_data += MLX5E_NET_IP_ALIGN;
362
363			bcopy(rq->mbuf[wqe_counter].data, mtod(mb, caddr_t),
364			    byte_cnt);
365		} else {
366			mb = rq->mbuf[wqe_counter].mbuf;
367			rq->mbuf[wqe_counter].mbuf = NULL;	/* safety clear */
368
369			bus_dmamap_unload(rq->dma_tag,
370			    rq->mbuf[wqe_counter].dma_map);
371		}
372
373		mlx5e_build_rx_mbuf(cqe, rq, mb, byte_cnt);
374		rq->stats.packets++;
375
376#if !defined(HAVE_TCP_LRO_RX)
377		tcp_lro_queue_mbuf(&rq->lro, mb);
378#else
379		if (mb->m_pkthdr.csum_flags == 0 ||
380		    (rq->ifp->if_capenable & IFCAP_LRO) == 0 ||
381		    rq->lro.lro_cnt == 0 ||
382		    tcp_lro_rx(&rq->lro, mb, 0) != 0) {
383			rq->ifp->if_input(rq->ifp, mb);
384		}
385#endif
386wq_ll_pop:
387		mlx5_wq_ll_pop(&rq->wq, wqe_counter_be,
388		    &wqe->next.next_wqe_index);
389	}
390
391	mlx5_cqwq_update_db_record(&rq->cq.wq);
392
393	/* ensure cq space is freed before enabling more cqes */
394	wmb();
395	return (i);
396}
397
398void
399mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq)
400{
401	struct mlx5e_rq *rq = container_of(mcq, struct mlx5e_rq, cq.mcq);
402	int i = 0;
403
404#ifdef HAVE_PER_CQ_EVENT_PACKET
405	struct mbuf *mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rq->wqe_sz);
406
407	if (mb != NULL) {
408		/* this code is used for debugging purpose only */
409		mb->m_pkthdr.len = mb->m_len = 15;
410		memset(mb->m_data, 255, 14);
411		mb->m_data[14] = rq->ix;
412		mb->m_pkthdr.rcvif = rq->ifp;
413		rq->ifp->if_input(rq->ifp, mb);
414	}
415#endif
416
417	mtx_lock(&rq->mtx);
418
419	/*
420	 * Polling the entire CQ without posting new WQEs results in
421	 * lack of receive WQEs during heavy traffic scenarios.
422	 */
423	while (1) {
424		if (mlx5e_poll_rx_cq(rq, MLX5E_RX_BUDGET_MAX) !=
425		    MLX5E_RX_BUDGET_MAX)
426			break;
427		i += MLX5E_RX_BUDGET_MAX;
428		if (i >= MLX5E_BUDGET_MAX)
429			break;
430		mlx5e_post_rx_wqes(rq);
431	}
432	mlx5e_post_rx_wqes(rq);
433	mlx5e_cq_arm(&rq->cq);
434	tcp_lro_flush_all(&rq->lro);
435	mtx_unlock(&rq->mtx);
436}
437