Deleted Added
sdiff udiff text old ( 284961 ) new ( 294327 )
full compact
tcp_lro.c (284961) tcp_lro.c (294327)
1/*-
2 * Copyright (c) 2007, Myricom Inc.
3 * Copyright (c) 2008, Intel Corporation.
4 * Copyright (c) 2012 The FreeBSD Foundation
1/*-
2 * Copyright (c) 2007, Myricom Inc.
3 * Copyright (c) 2008, Intel Corporation.
4 * Copyright (c) 2012 The FreeBSD Foundation
5 * Copyright (c) 2016 Mellanox Technologies.
5 * All rights reserved.
6 *
7 * Portions of this software were developed by Bjoern Zeeb
8 * under sponsorship from the FreeBSD Foundation.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:

--- 12 unchanged lines hidden (view full) ---

25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32#include <sys/cdefs.h>
6 * All rights reserved.
7 *
8 * Portions of this software were developed by Bjoern Zeeb
9 * under sponsorship from the FreeBSD Foundation.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:

--- 12 unchanged lines hidden (view full) ---

26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33#include <sys/cdefs.h>
33__FBSDID("$FreeBSD: head/sys/netinet/tcp_lro.c 284961 2015-06-30 17:19:58Z np $");
34__FBSDID("$FreeBSD: head/sys/netinet/tcp_lro.c 294327 2016-01-19 15:33:28Z hselasky $");
34
35#include "opt_inet.h"
36#include "opt_inet6.h"
37
38#include <sys/param.h>
39#include <sys/systm.h>
40#include <sys/mbuf.h>
41#include <sys/kernel.h>

--- 11 unchanged lines hidden (view full) ---

53#include <netinet/ip_var.h>
54#include <netinet/tcp.h>
55#include <netinet/tcp_lro.h>
56
57#include <netinet6/ip6_var.h>
58
59#include <machine/in_cksum.h>
60
35
36#include "opt_inet.h"
37#include "opt_inet6.h"
38
39#include <sys/param.h>
40#include <sys/systm.h>
41#include <sys/mbuf.h>
42#include <sys/kernel.h>

--- 11 unchanged lines hidden (view full) ---

54#include <netinet/ip_var.h>
55#include <netinet/tcp.h>
56#include <netinet/tcp_lro.h>
57
58#include <netinet6/ip6_var.h>
59
60#include <machine/in_cksum.h>
61
61#ifndef LRO_ENTRIES
62#define LRO_ENTRIES 8 /* # of LRO entries per RX queue. */
63#endif
62static MALLOC_DEFINE(M_LRO, "LRO", "LRO control structures");
64
65#define TCP_LRO_UPDATE_CSUM 1
66#ifndef TCP_LRO_UPDATE_CSUM
67#define TCP_LRO_INVALID_CSUM 0x0000
68#endif
69
70int
71tcp_lro_init(struct lro_ctrl *lc)
72{
63
64#define TCP_LRO_UPDATE_CSUM 1
65#ifndef TCP_LRO_UPDATE_CSUM
66#define TCP_LRO_INVALID_CSUM 0x0000
67#endif
68
69int
70tcp_lro_init(struct lro_ctrl *lc)
71{
72 return (tcp_lro_init_args(lc, NULL, TCP_LRO_ENTRIES, 0));
73}
74
75int
76tcp_lro_init_args(struct lro_ctrl *lc, struct ifnet *ifp,
77 unsigned lro_entries, unsigned lro_mbufs)
78{
73 struct lro_entry *le;
79 struct lro_entry *le;
74 int error, i;
80 size_t size;
81 unsigned i;
75
76 lc->lro_bad_csum = 0;
77 lc->lro_queued = 0;
78 lc->lro_flushed = 0;
79 lc->lro_cnt = 0;
82
83 lc->lro_bad_csum = 0;
84 lc->lro_queued = 0;
85 lc->lro_flushed = 0;
86 lc->lro_cnt = 0;
87 lc->lro_mbuf_count = 0;
88 lc->lro_mbuf_max = lro_mbufs;
89 lc->lro_cnt = lro_entries;
90 lc->ifp = ifp;
80 SLIST_INIT(&lc->lro_free);
81 SLIST_INIT(&lc->lro_active);
82
91 SLIST_INIT(&lc->lro_free);
92 SLIST_INIT(&lc->lro_active);
93
83 error = 0;
84 for (i = 0; i < LRO_ENTRIES; i++) {
85 le = (struct lro_entry *)malloc(sizeof(*le), M_DEVBUF,
86 M_NOWAIT | M_ZERO);
87 if (le == NULL) {
88 if (i == 0)
89 error = ENOMEM;
90 break;
91 }
92 lc->lro_cnt = i + 1;
93 SLIST_INSERT_HEAD(&lc->lro_free, le, next);
94 }
94 /* compute size to allocate */
95 size = (lro_mbufs * sizeof(struct mbuf *)) +
96 (lro_entries * sizeof(*le));
97 lc->lro_mbuf_data = (struct mbuf **)
98 malloc(size, M_LRO, M_NOWAIT | M_ZERO);
95
99
96 return (error);
100 /* check for out of memory */
101 if (lc->lro_mbuf_data == NULL) {
102 memset(lc, 0, sizeof(*lc));
103 return (ENOMEM);
104 }
105 /* compute offset for LRO entries */
106 le = (struct lro_entry *)
107 (lc->lro_mbuf_data + lro_mbufs);
108
109 /* setup linked list */
110 for (i = 0; i != lro_entries; i++)
111 SLIST_INSERT_HEAD(&lc->lro_free, le + i, next);
112
113 return (0);
97}
98
99void
100tcp_lro_free(struct lro_ctrl *lc)
101{
102 struct lro_entry *le;
114}
115
116void
117tcp_lro_free(struct lro_ctrl *lc)
118{
119 struct lro_entry *le;
120 unsigned x;
103
121
104 while (!SLIST_EMPTY(&lc->lro_free)) {
105 le = SLIST_FIRST(&lc->lro_free);
106 SLIST_REMOVE_HEAD(&lc->lro_free, next);
107 free(le, M_DEVBUF);
122 /* reset LRO free list */
123 SLIST_INIT(&lc->lro_free);
124
125 /* free active mbufs, if any */
126 while ((le = SLIST_FIRST(&lc->lro_active)) != NULL) {
127 SLIST_REMOVE_HEAD(&lc->lro_active, next);
128 m_freem(le->m_head);
108 }
129 }
130
131 /* free mbuf array, if any */
132 for (x = 0; x != lc->lro_mbuf_count; x++)
133 m_freem(lc->lro_mbuf_data[x]);
134 lc->lro_mbuf_count = 0;
135
136 /* free allocated memory, if any */
137 free(lc->lro_mbuf_data, M_LRO);
138 lc->lro_mbuf_data = NULL;
109}
110
111#ifdef TCP_LRO_UPDATE_CSUM
112static uint16_t
113tcp_lro_csum_th(struct tcphdr *th)
114{
115 uint32_t ch;
116 uint16_t *p, l;

--- 183 unchanged lines hidden (view full) ---

300
301 (*lc->ifp->if_input)(lc->ifp, le->m_head);
302 lc->lro_queued += le->append_cnt + 1;
303 lc->lro_flushed++;
304 bzero(le, sizeof(*le));
305 SLIST_INSERT_HEAD(&lc->lro_free, le, next);
306}
307
139}
140
141#ifdef TCP_LRO_UPDATE_CSUM
142static uint16_t
143tcp_lro_csum_th(struct tcphdr *th)
144{
145 uint32_t ch;
146 uint16_t *p, l;

--- 183 unchanged lines hidden (view full) ---

330
331 (*lc->ifp->if_input)(lc->ifp, le->m_head);
332 lc->lro_queued += le->append_cnt + 1;
333 lc->lro_flushed++;
334 bzero(le, sizeof(*le));
335 SLIST_INSERT_HEAD(&lc->lro_free, le, next);
336}
337
338static int
339tcp_lro_mbuf_compare_header(const void *ppa, const void *ppb)
340{
341 const struct mbuf *ma = *((const struct mbuf * const *)ppa);
342 const struct mbuf *mb = *((const struct mbuf * const *)ppb);
343 int ret;
344
345 ret = M_HASHTYPE_GET(ma) - M_HASHTYPE_GET(mb);
346 if (ret != 0)
347 goto done;
348
349 ret = ma->m_pkthdr.flowid - mb->m_pkthdr.flowid;
350 if (ret != 0)
351 goto done;
352
353 ret = TCP_LRO_SEQUENCE(ma) - TCP_LRO_SEQUENCE(mb);
354done:
355 return (ret);
356}
357
358void
359tcp_lro_flush_all(struct lro_ctrl *lc)
360{
361 struct lro_entry *le;
362 uint32_t hashtype;
363 uint32_t flowid;
364 unsigned x;
365
366 /* check if no mbufs to flush */
367 if (__predict_false(lc->lro_mbuf_count == 0))
368 goto done;
369
370 /* sort all mbufs according to stream */
371 qsort(lc->lro_mbuf_data, lc->lro_mbuf_count, sizeof(struct mbuf *),
372 &tcp_lro_mbuf_compare_header);
373
374 /* input data into LRO engine, stream by stream */
375 flowid = 0;
376 hashtype = M_HASHTYPE_NONE;
377 for (x = 0; x != lc->lro_mbuf_count; x++) {
378 struct mbuf *mb;
379
380 mb = lc->lro_mbuf_data[x];
381
382 /* check for new stream */
383 if (mb->m_pkthdr.flowid != flowid ||
384 M_HASHTYPE_GET(mb) != hashtype) {
385 flowid = mb->m_pkthdr.flowid;
386 hashtype = M_HASHTYPE_GET(mb);
387
388 /* flush active streams */
389 while ((le = SLIST_FIRST(&lc->lro_active)) != NULL) {
390 SLIST_REMOVE_HEAD(&lc->lro_active, next);
391 tcp_lro_flush(lc, le);
392 }
393 }
394#ifdef TCP_LRO_RESET_SEQUENCE
395 /* reset sequence number */
396 TCP_LRO_SEQUENCE(mb) = 0;
397#endif
398 /* add packet to LRO engine */
399 if (tcp_lro_rx(lc, mb, 0) != 0) {
400 /* input packet to network layer */
401 (*lc->ifp->if_input)(lc->ifp, mb);
402 lc->lro_queued++;
403 lc->lro_flushed++;
404 }
405 }
406done:
407 /* flush active streams */
408 while ((le = SLIST_FIRST(&lc->lro_active)) != NULL) {
409 SLIST_REMOVE_HEAD(&lc->lro_active, next);
410 tcp_lro_flush(lc, le);
411 }
412 lc->lro_mbuf_count = 0;
413}
414
308#ifdef INET6
309static int
310tcp_lro_rx_ipv6(struct lro_ctrl *lc, struct mbuf *m, struct ip6_hdr *ip6,
311 struct tcphdr **th)
312{
313
314 /* XXX-BZ we should check the flow-label. */
315

--- 312 unchanged lines hidden (view full) ---

628#endif
629
630 le->m_head = m;
631 le->m_tail = m_last(m);
632
633 return (0);
634}
635
415#ifdef INET6
416static int
417tcp_lro_rx_ipv6(struct lro_ctrl *lc, struct mbuf *m, struct ip6_hdr *ip6,
418 struct tcphdr **th)
419{
420
421 /* XXX-BZ we should check the flow-label. */
422

--- 312 unchanged lines hidden (view full) ---

735#endif
736
737 le->m_head = m;
738 le->m_tail = m_last(m);
739
740 return (0);
741}
742
743void
744tcp_lro_queue_mbuf(struct lro_ctrl *lc, struct mbuf *mb)
745{
746 /* sanity checks */
747 if (__predict_false(lc->ifp == NULL || lc->lro_mbuf_data == NULL ||
748 lc->lro_mbuf_max == 0)) {
749 /* packet drop */
750 m_freem(mb);
751 return;
752 }
753
754 /* check if packet is not LRO capable */
755 if (__predict_false(mb->m_pkthdr.csum_flags == 0 ||
756 (lc->ifp->if_capenable & IFCAP_LRO) == 0)) {
757 lc->lro_flushed++;
758 lc->lro_queued++;
759
760 /* input packet to network layer */
761 (*lc->ifp->if_input) (lc->ifp, mb);
762 return;
763 }
764
765 /* check if array is full */
766 if (__predict_false(lc->lro_mbuf_count == lc->lro_mbuf_max))
767 tcp_lro_flush_all(lc);
768
769 /* store sequence number */
770 TCP_LRO_SEQUENCE(mb) = lc->lro_mbuf_count;
771
772 /* enter mbuf */
773 lc->lro_mbuf_data[lc->lro_mbuf_count++] = mb;
774}
775
636/* end */
776/* end */