1/*-
2 * Copyright (c) 2007, Myricom Inc.
3 * Copyright (c) 2008, Intel Corporation.
4 * Copyright (c) 2012 The FreeBSD Foundation
5 * Copyright (c) 2016 Mellanox Technologies.
6 * All rights reserved.
7 *
8 * Portions of this software were developed by Bjoern Zeeb
9 * under sponsorship from the FreeBSD Foundation.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33#include <sys/cdefs.h>
| 1/*-
2 * Copyright (c) 2007, Myricom Inc.
3 * Copyright (c) 2008, Intel Corporation.
4 * Copyright (c) 2012 The FreeBSD Foundation
5 * Copyright (c) 2016 Mellanox Technologies.
6 * All rights reserved.
7 *
8 * Portions of this software were developed by Bjoern Zeeb
9 * under sponsorship from the FreeBSD Foundation.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33#include <sys/cdefs.h>
|
34__FBSDID("$FreeBSD: head/sys/netinet/tcp_lro.c 300731 2016-05-26 11:10:31Z hselasky $");
| 34__FBSDID("$FreeBSD: head/sys/netinet/tcp_lro.c 301249 2016-06-03 08:35:07Z hselasky $");
|
35
36#include "opt_inet.h"
37#include "opt_inet6.h"
38
39#include <sys/param.h>
40#include <sys/systm.h>
41#include <sys/kernel.h>
42#include <sys/malloc.h>
43#include <sys/mbuf.h>
44#include <sys/socket.h>
45
46#include <net/if.h>
47#include <net/if_var.h>
48#include <net/ethernet.h>
49#include <net/vnet.h>
50
51#include <netinet/in_systm.h>
52#include <netinet/in.h>
53#include <netinet/ip6.h>
54#include <netinet/ip.h>
55#include <netinet/ip_var.h>
56#include <netinet/tcp.h>
57#include <netinet/tcp_lro.h>
58
59#include <netinet6/ip6_var.h>
60
61#include <machine/in_cksum.h>
62
63static MALLOC_DEFINE(M_LRO, "LRO", "LRO control structures");
64
65#define TCP_LRO_UPDATE_CSUM 1
66#ifndef TCP_LRO_UPDATE_CSUM
67#define TCP_LRO_INVALID_CSUM 0x0000
68#endif
69
70static void tcp_lro_rx_done(struct lro_ctrl *lc);
71
72static __inline void
73tcp_lro_active_insert(struct lro_ctrl *lc, struct lro_entry *le)
74{
75
76 LIST_INSERT_HEAD(&lc->lro_active, le, next);
77}
78
79static __inline void
80tcp_lro_active_remove(struct lro_entry *le)
81{
82
83 LIST_REMOVE(le, next);
84}
85
86int
87tcp_lro_init(struct lro_ctrl *lc)
88{
89 return (tcp_lro_init_args(lc, NULL, TCP_LRO_ENTRIES, 0));
90}
91
92int
93tcp_lro_init_args(struct lro_ctrl *lc, struct ifnet *ifp,
94 unsigned lro_entries, unsigned lro_mbufs)
95{
96 struct lro_entry *le;
97 size_t size;
98 unsigned i;
99
100 lc->lro_bad_csum = 0;
101 lc->lro_queued = 0;
102 lc->lro_flushed = 0;
103 lc->lro_cnt = 0;
104 lc->lro_mbuf_count = 0;
105 lc->lro_mbuf_max = lro_mbufs;
106 lc->lro_cnt = lro_entries;
107 lc->lro_ackcnt_lim = TCP_LRO_ACKCNT_MAX;
108 lc->lro_length_lim = TCP_LRO_LENGTH_MAX;
109 lc->ifp = ifp;
110 LIST_INIT(&lc->lro_free);
111 LIST_INIT(&lc->lro_active);
112
113 /* compute size to allocate */
114 size = (lro_mbufs * sizeof(struct lro_mbuf_sort)) +
115 (lro_entries * sizeof(*le));
116 lc->lro_mbuf_data = (struct lro_mbuf_sort *)
117 malloc(size, M_LRO, M_NOWAIT | M_ZERO);
118
119 /* check for out of memory */
120 if (lc->lro_mbuf_data == NULL) {
121 memset(lc, 0, sizeof(*lc));
122 return (ENOMEM);
123 }
124 /* compute offset for LRO entries */
125 le = (struct lro_entry *)
126 (lc->lro_mbuf_data + lro_mbufs);
127
128 /* setup linked list */
129 for (i = 0; i != lro_entries; i++)
130 LIST_INSERT_HEAD(&lc->lro_free, le + i, next);
131
132 return (0);
133}
134
135void
136tcp_lro_free(struct lro_ctrl *lc)
137{
138 struct lro_entry *le;
139 unsigned x;
140
141 /* reset LRO free list */
142 LIST_INIT(&lc->lro_free);
143
144 /* free active mbufs, if any */
145 while ((le = LIST_FIRST(&lc->lro_active)) != NULL) {
146 tcp_lro_active_remove(le);
147 m_freem(le->m_head);
148 }
149
150 /* free mbuf array, if any */
151 for (x = 0; x != lc->lro_mbuf_count; x++)
152 m_freem(lc->lro_mbuf_data[x].mb);
153 lc->lro_mbuf_count = 0;
154
155 /* free allocated memory, if any */
156 free(lc->lro_mbuf_data, M_LRO);
157 lc->lro_mbuf_data = NULL;
158}
159
160#ifdef TCP_LRO_UPDATE_CSUM
161static uint16_t
162tcp_lro_csum_th(struct tcphdr *th)
163{
164 uint32_t ch;
165 uint16_t *p, l;
166
167 ch = th->th_sum = 0x0000;
168 l = th->th_off;
169 p = (uint16_t *)th;
170 while (l > 0) {
171 ch += *p;
172 p++;
173 ch += *p;
174 p++;
175 l--;
176 }
177 while (ch > 0xffff)
178 ch = (ch >> 16) + (ch & 0xffff);
179
180 return (ch & 0xffff);
181}
182
183static uint16_t
184tcp_lro_rx_csum_fixup(struct lro_entry *le, void *l3hdr, struct tcphdr *th,
185 uint16_t tcp_data_len, uint16_t csum)
186{
187 uint32_t c;
188 uint16_t cs;
189
190 c = csum;
191
192 /* Remove length from checksum. */
193 switch (le->eh_type) {
194#ifdef INET6
195 case ETHERTYPE_IPV6:
196 {
197 struct ip6_hdr *ip6;
198
199 ip6 = (struct ip6_hdr *)l3hdr;
200 if (le->append_cnt == 0)
201 cs = ip6->ip6_plen;
202 else {
203 uint32_t cx;
204
205 cx = ntohs(ip6->ip6_plen);
206 cs = in6_cksum_pseudo(ip6, cx, ip6->ip6_nxt, 0);
207 }
208 break;
209 }
210#endif
211#ifdef INET
212 case ETHERTYPE_IP:
213 {
214 struct ip *ip4;
215
216 ip4 = (struct ip *)l3hdr;
217 if (le->append_cnt == 0)
218 cs = ip4->ip_len;
219 else {
220 cs = in_addword(ntohs(ip4->ip_len) - sizeof(*ip4),
221 IPPROTO_TCP);
222 cs = in_pseudo(ip4->ip_src.s_addr, ip4->ip_dst.s_addr,
223 htons(cs));
224 }
225 break;
226 }
227#endif
228 default:
229 cs = 0; /* Keep compiler happy. */
230 }
231
232 cs = ~cs;
233 c += cs;
234
235 /* Remove TCP header csum. */
236 cs = ~tcp_lro_csum_th(th);
237 c += cs;
238 while (c > 0xffff)
239 c = (c >> 16) + (c & 0xffff);
240
241 return (c & 0xffff);
242}
243#endif
244
245static void
246tcp_lro_rx_done(struct lro_ctrl *lc)
247{
248 struct lro_entry *le;
249
250 while ((le = LIST_FIRST(&lc->lro_active)) != NULL) {
251 tcp_lro_active_remove(le);
252 tcp_lro_flush(lc, le);
253 }
254}
255
256void
257tcp_lro_flush_inactive(struct lro_ctrl *lc, const struct timeval *timeout)
258{
259 struct lro_entry *le, *le_tmp;
260 struct timeval tv;
261
262 if (LIST_EMPTY(&lc->lro_active))
263 return;
264
265 getmicrotime(&tv);
266 timevalsub(&tv, timeout);
267 LIST_FOREACH_SAFE(le, &lc->lro_active, next, le_tmp) {
268 if (timevalcmp(&tv, &le->mtime, >=)) {
269 tcp_lro_active_remove(le);
270 tcp_lro_flush(lc, le);
271 }
272 }
273}
274
275void
276tcp_lro_flush(struct lro_ctrl *lc, struct lro_entry *le)
277{
278
279 if (le->append_cnt > 0) {
280 struct tcphdr *th;
281 uint16_t p_len;
282
283 p_len = htons(le->p_len);
284 switch (le->eh_type) {
285#ifdef INET6
286 case ETHERTYPE_IPV6:
287 {
288 struct ip6_hdr *ip6;
289
290 ip6 = le->le_ip6;
291 ip6->ip6_plen = p_len;
292 th = (struct tcphdr *)(ip6 + 1);
293 le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID |
294 CSUM_PSEUDO_HDR;
295 le->p_len += ETHER_HDR_LEN + sizeof(*ip6);
296 break;
297 }
298#endif
299#ifdef INET
300 case ETHERTYPE_IP:
301 {
302 struct ip *ip4;
303#ifdef TCP_LRO_UPDATE_CSUM
304 uint32_t cl;
305 uint16_t c;
306#endif
307
308 ip4 = le->le_ip4;
309#ifdef TCP_LRO_UPDATE_CSUM
310 /* Fix IP header checksum for new length. */
311 c = ~ip4->ip_sum;
312 cl = c;
313 c = ~ip4->ip_len;
314 cl += c + p_len;
315 while (cl > 0xffff)
316 cl = (cl >> 16) + (cl & 0xffff);
317 c = cl;
318 ip4->ip_sum = ~c;
319#else
320 ip4->ip_sum = TCP_LRO_INVALID_CSUM;
321#endif
322 ip4->ip_len = p_len;
323 th = (struct tcphdr *)(ip4 + 1);
324 le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID |
325 CSUM_PSEUDO_HDR | CSUM_IP_CHECKED | CSUM_IP_VALID;
326 le->p_len += ETHER_HDR_LEN;
327 break;
328 }
329#endif
330 default:
331 th = NULL; /* Keep compiler happy. */
332 }
333 le->m_head->m_pkthdr.csum_data = 0xffff;
334 le->m_head->m_pkthdr.len = le->p_len;
335
336 /* Incorporate the latest ACK into the TCP header. */
337 th->th_ack = le->ack_seq;
338 th->th_win = le->window;
339 /* Incorporate latest timestamp into the TCP header. */
340 if (le->timestamp != 0) {
341 uint32_t *ts_ptr;
342
343 ts_ptr = (uint32_t *)(th + 1);
344 ts_ptr[1] = htonl(le->tsval);
345 ts_ptr[2] = le->tsecr;
346 }
347#ifdef TCP_LRO_UPDATE_CSUM
348 /* Update the TCP header checksum. */
349 le->ulp_csum += p_len;
350 le->ulp_csum += tcp_lro_csum_th(th);
351 while (le->ulp_csum > 0xffff)
352 le->ulp_csum = (le->ulp_csum >> 16) +
353 (le->ulp_csum & 0xffff);
354 th->th_sum = (le->ulp_csum & 0xffff);
355 th->th_sum = ~th->th_sum;
356#else
357 th->th_sum = TCP_LRO_INVALID_CSUM;
358#endif
359 }
360
361 (*lc->ifp->if_input)(lc->ifp, le->m_head);
362 lc->lro_queued += le->append_cnt + 1;
363 lc->lro_flushed++;
364 bzero(le, sizeof(*le));
365 LIST_INSERT_HEAD(&lc->lro_free, le, next);
366}
367
368#ifdef HAVE_INLINE_FLSLL
369#define tcp_lro_msb_64(x) (1ULL << (flsll(x) - 1))
370#else
371static inline uint64_t
372tcp_lro_msb_64(uint64_t x)
373{
374 x |= (x >> 1);
375 x |= (x >> 2);
376 x |= (x >> 4);
377 x |= (x >> 8);
378 x |= (x >> 16);
379 x |= (x >> 32);
380 return (x & ~(x >> 1));
381}
382#endif
383
384/*
385 * The tcp_lro_sort() routine is comparable to qsort(), except it has
386 * a worst case complexity limit of O(MIN(N,64)*N), where N is the
387 * number of elements to sort and 64 is the number of sequence bits
388 * available. The algorithm is bit-slicing the 64-bit sequence number,
389 * sorting one bit at a time from the most significant bit until the
| 35
36#include "opt_inet.h"
37#include "opt_inet6.h"
38
39#include <sys/param.h>
40#include <sys/systm.h>
41#include <sys/kernel.h>
42#include <sys/malloc.h>
43#include <sys/mbuf.h>
44#include <sys/socket.h>
45
46#include <net/if.h>
47#include <net/if_var.h>
48#include <net/ethernet.h>
49#include <net/vnet.h>
50
51#include <netinet/in_systm.h>
52#include <netinet/in.h>
53#include <netinet/ip6.h>
54#include <netinet/ip.h>
55#include <netinet/ip_var.h>
56#include <netinet/tcp.h>
57#include <netinet/tcp_lro.h>
58
59#include <netinet6/ip6_var.h>
60
61#include <machine/in_cksum.h>
62
63static MALLOC_DEFINE(M_LRO, "LRO", "LRO control structures");
64
65#define TCP_LRO_UPDATE_CSUM 1
66#ifndef TCP_LRO_UPDATE_CSUM
67#define TCP_LRO_INVALID_CSUM 0x0000
68#endif
69
70static void tcp_lro_rx_done(struct lro_ctrl *lc);
71
72static __inline void
73tcp_lro_active_insert(struct lro_ctrl *lc, struct lro_entry *le)
74{
75
76 LIST_INSERT_HEAD(&lc->lro_active, le, next);
77}
78
79static __inline void
80tcp_lro_active_remove(struct lro_entry *le)
81{
82
83 LIST_REMOVE(le, next);
84}
85
86int
87tcp_lro_init(struct lro_ctrl *lc)
88{
89 return (tcp_lro_init_args(lc, NULL, TCP_LRO_ENTRIES, 0));
90}
91
92int
93tcp_lro_init_args(struct lro_ctrl *lc, struct ifnet *ifp,
94 unsigned lro_entries, unsigned lro_mbufs)
95{
96 struct lro_entry *le;
97 size_t size;
98 unsigned i;
99
100 lc->lro_bad_csum = 0;
101 lc->lro_queued = 0;
102 lc->lro_flushed = 0;
103 lc->lro_cnt = 0;
104 lc->lro_mbuf_count = 0;
105 lc->lro_mbuf_max = lro_mbufs;
106 lc->lro_cnt = lro_entries;
107 lc->lro_ackcnt_lim = TCP_LRO_ACKCNT_MAX;
108 lc->lro_length_lim = TCP_LRO_LENGTH_MAX;
109 lc->ifp = ifp;
110 LIST_INIT(&lc->lro_free);
111 LIST_INIT(&lc->lro_active);
112
113 /* compute size to allocate */
114 size = (lro_mbufs * sizeof(struct lro_mbuf_sort)) +
115 (lro_entries * sizeof(*le));
116 lc->lro_mbuf_data = (struct lro_mbuf_sort *)
117 malloc(size, M_LRO, M_NOWAIT | M_ZERO);
118
119 /* check for out of memory */
120 if (lc->lro_mbuf_data == NULL) {
121 memset(lc, 0, sizeof(*lc));
122 return (ENOMEM);
123 }
124 /* compute offset for LRO entries */
125 le = (struct lro_entry *)
126 (lc->lro_mbuf_data + lro_mbufs);
127
128 /* setup linked list */
129 for (i = 0; i != lro_entries; i++)
130 LIST_INSERT_HEAD(&lc->lro_free, le + i, next);
131
132 return (0);
133}
134
135void
136tcp_lro_free(struct lro_ctrl *lc)
137{
138 struct lro_entry *le;
139 unsigned x;
140
141 /* reset LRO free list */
142 LIST_INIT(&lc->lro_free);
143
144 /* free active mbufs, if any */
145 while ((le = LIST_FIRST(&lc->lro_active)) != NULL) {
146 tcp_lro_active_remove(le);
147 m_freem(le->m_head);
148 }
149
150 /* free mbuf array, if any */
151 for (x = 0; x != lc->lro_mbuf_count; x++)
152 m_freem(lc->lro_mbuf_data[x].mb);
153 lc->lro_mbuf_count = 0;
154
155 /* free allocated memory, if any */
156 free(lc->lro_mbuf_data, M_LRO);
157 lc->lro_mbuf_data = NULL;
158}
159
160#ifdef TCP_LRO_UPDATE_CSUM
161static uint16_t
162tcp_lro_csum_th(struct tcphdr *th)
163{
164 uint32_t ch;
165 uint16_t *p, l;
166
167 ch = th->th_sum = 0x0000;
168 l = th->th_off;
169 p = (uint16_t *)th;
170 while (l > 0) {
171 ch += *p;
172 p++;
173 ch += *p;
174 p++;
175 l--;
176 }
177 while (ch > 0xffff)
178 ch = (ch >> 16) + (ch & 0xffff);
179
180 return (ch & 0xffff);
181}
182
183static uint16_t
184tcp_lro_rx_csum_fixup(struct lro_entry *le, void *l3hdr, struct tcphdr *th,
185 uint16_t tcp_data_len, uint16_t csum)
186{
187 uint32_t c;
188 uint16_t cs;
189
190 c = csum;
191
192 /* Remove length from checksum. */
193 switch (le->eh_type) {
194#ifdef INET6
195 case ETHERTYPE_IPV6:
196 {
197 struct ip6_hdr *ip6;
198
199 ip6 = (struct ip6_hdr *)l3hdr;
200 if (le->append_cnt == 0)
201 cs = ip6->ip6_plen;
202 else {
203 uint32_t cx;
204
205 cx = ntohs(ip6->ip6_plen);
206 cs = in6_cksum_pseudo(ip6, cx, ip6->ip6_nxt, 0);
207 }
208 break;
209 }
210#endif
211#ifdef INET
212 case ETHERTYPE_IP:
213 {
214 struct ip *ip4;
215
216 ip4 = (struct ip *)l3hdr;
217 if (le->append_cnt == 0)
218 cs = ip4->ip_len;
219 else {
220 cs = in_addword(ntohs(ip4->ip_len) - sizeof(*ip4),
221 IPPROTO_TCP);
222 cs = in_pseudo(ip4->ip_src.s_addr, ip4->ip_dst.s_addr,
223 htons(cs));
224 }
225 break;
226 }
227#endif
228 default:
229 cs = 0; /* Keep compiler happy. */
230 }
231
232 cs = ~cs;
233 c += cs;
234
235 /* Remove TCP header csum. */
236 cs = ~tcp_lro_csum_th(th);
237 c += cs;
238 while (c > 0xffff)
239 c = (c >> 16) + (c & 0xffff);
240
241 return (c & 0xffff);
242}
243#endif
244
245static void
246tcp_lro_rx_done(struct lro_ctrl *lc)
247{
248 struct lro_entry *le;
249
250 while ((le = LIST_FIRST(&lc->lro_active)) != NULL) {
251 tcp_lro_active_remove(le);
252 tcp_lro_flush(lc, le);
253 }
254}
255
256void
257tcp_lro_flush_inactive(struct lro_ctrl *lc, const struct timeval *timeout)
258{
259 struct lro_entry *le, *le_tmp;
260 struct timeval tv;
261
262 if (LIST_EMPTY(&lc->lro_active))
263 return;
264
265 getmicrotime(&tv);
266 timevalsub(&tv, timeout);
267 LIST_FOREACH_SAFE(le, &lc->lro_active, next, le_tmp) {
268 if (timevalcmp(&tv, &le->mtime, >=)) {
269 tcp_lro_active_remove(le);
270 tcp_lro_flush(lc, le);
271 }
272 }
273}
274
275void
276tcp_lro_flush(struct lro_ctrl *lc, struct lro_entry *le)
277{
278
279 if (le->append_cnt > 0) {
280 struct tcphdr *th;
281 uint16_t p_len;
282
283 p_len = htons(le->p_len);
284 switch (le->eh_type) {
285#ifdef INET6
286 case ETHERTYPE_IPV6:
287 {
288 struct ip6_hdr *ip6;
289
290 ip6 = le->le_ip6;
291 ip6->ip6_plen = p_len;
292 th = (struct tcphdr *)(ip6 + 1);
293 le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID |
294 CSUM_PSEUDO_HDR;
295 le->p_len += ETHER_HDR_LEN + sizeof(*ip6);
296 break;
297 }
298#endif
299#ifdef INET
300 case ETHERTYPE_IP:
301 {
302 struct ip *ip4;
303#ifdef TCP_LRO_UPDATE_CSUM
304 uint32_t cl;
305 uint16_t c;
306#endif
307
308 ip4 = le->le_ip4;
309#ifdef TCP_LRO_UPDATE_CSUM
310 /* Fix IP header checksum for new length. */
311 c = ~ip4->ip_sum;
312 cl = c;
313 c = ~ip4->ip_len;
314 cl += c + p_len;
315 while (cl > 0xffff)
316 cl = (cl >> 16) + (cl & 0xffff);
317 c = cl;
318 ip4->ip_sum = ~c;
319#else
320 ip4->ip_sum = TCP_LRO_INVALID_CSUM;
321#endif
322 ip4->ip_len = p_len;
323 th = (struct tcphdr *)(ip4 + 1);
324 le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID |
325 CSUM_PSEUDO_HDR | CSUM_IP_CHECKED | CSUM_IP_VALID;
326 le->p_len += ETHER_HDR_LEN;
327 break;
328 }
329#endif
330 default:
331 th = NULL; /* Keep compiler happy. */
332 }
333 le->m_head->m_pkthdr.csum_data = 0xffff;
334 le->m_head->m_pkthdr.len = le->p_len;
335
336 /* Incorporate the latest ACK into the TCP header. */
337 th->th_ack = le->ack_seq;
338 th->th_win = le->window;
339 /* Incorporate latest timestamp into the TCP header. */
340 if (le->timestamp != 0) {
341 uint32_t *ts_ptr;
342
343 ts_ptr = (uint32_t *)(th + 1);
344 ts_ptr[1] = htonl(le->tsval);
345 ts_ptr[2] = le->tsecr;
346 }
347#ifdef TCP_LRO_UPDATE_CSUM
348 /* Update the TCP header checksum. */
349 le->ulp_csum += p_len;
350 le->ulp_csum += tcp_lro_csum_th(th);
351 while (le->ulp_csum > 0xffff)
352 le->ulp_csum = (le->ulp_csum >> 16) +
353 (le->ulp_csum & 0xffff);
354 th->th_sum = (le->ulp_csum & 0xffff);
355 th->th_sum = ~th->th_sum;
356#else
357 th->th_sum = TCP_LRO_INVALID_CSUM;
358#endif
359 }
360
361 (*lc->ifp->if_input)(lc->ifp, le->m_head);
362 lc->lro_queued += le->append_cnt + 1;
363 lc->lro_flushed++;
364 bzero(le, sizeof(*le));
365 LIST_INSERT_HEAD(&lc->lro_free, le, next);
366}
367
368#ifdef HAVE_INLINE_FLSLL
369#define tcp_lro_msb_64(x) (1ULL << (flsll(x) - 1))
370#else
371static inline uint64_t
372tcp_lro_msb_64(uint64_t x)
373{
374 x |= (x >> 1);
375 x |= (x >> 2);
376 x |= (x >> 4);
377 x |= (x >> 8);
378 x |= (x >> 16);
379 x |= (x >> 32);
380 return (x & ~(x >> 1));
381}
382#endif
383
384/*
385 * The tcp_lro_sort() routine is comparable to qsort(), except it has
386 * a worst case complexity limit of O(MIN(N,64)*N), where N is the
387 * number of elements to sort and 64 is the number of sequence bits
388 * available. The algorithm is bit-slicing the 64-bit sequence number,
389 * sorting one bit at a time from the most significant bit until the
|
390 * least significant one, skipping the constant bits.
| 390 * least significant one, skipping the constant bits. This is
391 * typically called a radix sort.
|
391 */
392static void
393tcp_lro_sort(struct lro_mbuf_sort *parray, uint32_t size)
394{
395 struct lro_mbuf_sort temp;
396 uint64_t ones;
397 uint64_t zeros;
398 uint32_t x;
399 uint32_t y;
400
401repeat:
| 392 */
393static void
394tcp_lro_sort(struct lro_mbuf_sort *parray, uint32_t size)
395{
396 struct lro_mbuf_sort temp;
397 uint64_t ones;
398 uint64_t zeros;
399 uint32_t x;
400 uint32_t y;
401
402repeat:
|
402 /* for small arrays bubble sort is faster */
| 403 /* for small arrays insertion sort is faster */
|
403 if (size <= 12) {
| 404 if (size <= 12) {
|
404 for (x = 0; x != size; x++) {
405 for (y = x + 1; y != size; y++) {
406 if (parray[x].seq > parray[y].seq) {
407 /* swap entries */
408 temp = parray[x];
409 parray[x] = parray[y];
410 parray[y] = temp;
411 }
412 }
| 405 for (x = 1; x < size; x++) {
406 temp = parray[x];
407 for (y = x; y > 0 && temp.seq < parray[y - 1].seq; y--)
408 parray[y] = parray[y - 1];
409 parray[y] = temp;
|
413 }
414 return;
415 }
416
417 /* compute sequence bits which are constant */
418 ones = 0;
419 zeros = 0;
420 for (x = 0; x != size; x++) {
421 ones |= parray[x].seq;
422 zeros |= ~parray[x].seq;
423 }
424
425 /* compute bits which are not constant into "ones" */
426 ones &= zeros;
427 if (ones == 0)
428 return;
429
430 /* pick the most significant bit which is not constant */
431 ones = tcp_lro_msb_64(ones);
432
433 /*
434 * Move entries having cleared sequence bits to the beginning
435 * of the array:
436 */
437 for (x = y = 0; y != size; y++) {
438 /* skip set bits */
439 if (parray[y].seq & ones)
440 continue;
441 /* swap entries */
442 temp = parray[x];
443 parray[x] = parray[y];
444 parray[y] = temp;
445 x++;
446 }
447
448 KASSERT(x != 0 && x != size, ("Memory is corrupted\n"));
449
450 /* sort zeros */
451 tcp_lro_sort(parray, x);
452
453 /* sort ones */
454 parray += x;
455 size -= x;
456 goto repeat;
457}
458
459void
460tcp_lro_flush_all(struct lro_ctrl *lc)
461{
462 uint64_t seq;
463 uint64_t nseq;
464 unsigned x;
465
466 /* check if no mbufs to flush */
467 if (lc->lro_mbuf_count == 0)
468 goto done;
469
470 /* sort all mbufs according to stream */
471 tcp_lro_sort(lc->lro_mbuf_data, lc->lro_mbuf_count);
472
473 /* input data into LRO engine, stream by stream */
474 seq = 0;
475 for (x = 0; x != lc->lro_mbuf_count; x++) {
476 struct mbuf *mb;
477
478 /* get mbuf */
479 mb = lc->lro_mbuf_data[x].mb;
480
481 /* get sequence number, masking away the packet index */
482 nseq = lc->lro_mbuf_data[x].seq & (-1ULL << 24);
483
484 /* check for new stream */
485 if (seq != nseq) {
486 seq = nseq;
487
488 /* flush active streams */
489 tcp_lro_rx_done(lc);
490 }
491
492 /* add packet to LRO engine */
493 if (tcp_lro_rx(lc, mb, 0) != 0) {
494 /* input packet to network layer */
495 (*lc->ifp->if_input)(lc->ifp, mb);
496 lc->lro_queued++;
497 lc->lro_flushed++;
498 }
499 }
500done:
501 /* flush active streams */
502 tcp_lro_rx_done(lc);
503
504 lc->lro_mbuf_count = 0;
505}
506
507#ifdef INET6
508static int
509tcp_lro_rx_ipv6(struct lro_ctrl *lc, struct mbuf *m, struct ip6_hdr *ip6,
510 struct tcphdr **th)
511{
512
513 /* XXX-BZ we should check the flow-label. */
514
515 /* XXX-BZ We do not yet support ext. hdrs. */
516 if (ip6->ip6_nxt != IPPROTO_TCP)
517 return (TCP_LRO_NOT_SUPPORTED);
518
519 /* Find the TCP header. */
520 *th = (struct tcphdr *)(ip6 + 1);
521
522 return (0);
523}
524#endif
525
526#ifdef INET
527static int
528tcp_lro_rx_ipv4(struct lro_ctrl *lc, struct mbuf *m, struct ip *ip4,
529 struct tcphdr **th)
530{
531 int csum_flags;
532 uint16_t csum;
533
534 if (ip4->ip_p != IPPROTO_TCP)
535 return (TCP_LRO_NOT_SUPPORTED);
536
537 /* Ensure there are no options. */
538 if ((ip4->ip_hl << 2) != sizeof (*ip4))
539 return (TCP_LRO_CANNOT);
540
541 /* .. and the packet is not fragmented. */
542 if (ip4->ip_off & htons(IP_MF|IP_OFFMASK))
543 return (TCP_LRO_CANNOT);
544
545 /* Legacy IP has a header checksum that needs to be correct. */
546 csum_flags = m->m_pkthdr.csum_flags;
547 if (csum_flags & CSUM_IP_CHECKED) {
548 if (__predict_false((csum_flags & CSUM_IP_VALID) == 0)) {
549 lc->lro_bad_csum++;
550 return (TCP_LRO_CANNOT);
551 }
552 } else {
553 csum = in_cksum_hdr(ip4);
554 if (__predict_false((csum) != 0)) {
555 lc->lro_bad_csum++;
556 return (TCP_LRO_CANNOT);
557 }
558 }
559
560 /* Find the TCP header (we assured there are no IP options). */
561 *th = (struct tcphdr *)(ip4 + 1);
562
563 return (0);
564}
565#endif
566
567int
568tcp_lro_rx(struct lro_ctrl *lc, struct mbuf *m, uint32_t csum)
569{
570 struct lro_entry *le;
571 struct ether_header *eh;
572#ifdef INET6
573 struct ip6_hdr *ip6 = NULL; /* Keep compiler happy. */
574#endif
575#ifdef INET
576 struct ip *ip4 = NULL; /* Keep compiler happy. */
577#endif
578 struct tcphdr *th;
579 void *l3hdr = NULL; /* Keep compiler happy. */
580 uint32_t *ts_ptr;
581 tcp_seq seq;
582 int error, ip_len, l;
583 uint16_t eh_type, tcp_data_len;
584
585 /* We expect a contiguous header [eh, ip, tcp]. */
586
587 eh = mtod(m, struct ether_header *);
588 eh_type = ntohs(eh->ether_type);
589 switch (eh_type) {
590#ifdef INET6
591 case ETHERTYPE_IPV6:
592 {
593 CURVNET_SET(lc->ifp->if_vnet);
594 if (V_ip6_forwarding != 0) {
595 /* XXX-BZ stats but changing lro_ctrl is a problem. */
596 CURVNET_RESTORE();
597 return (TCP_LRO_CANNOT);
598 }
599 CURVNET_RESTORE();
600 l3hdr = ip6 = (struct ip6_hdr *)(eh + 1);
601 error = tcp_lro_rx_ipv6(lc, m, ip6, &th);
602 if (error != 0)
603 return (error);
604 tcp_data_len = ntohs(ip6->ip6_plen);
605 ip_len = sizeof(*ip6) + tcp_data_len;
606 break;
607 }
608#endif
609#ifdef INET
610 case ETHERTYPE_IP:
611 {
612 CURVNET_SET(lc->ifp->if_vnet);
613 if (V_ipforwarding != 0) {
614 /* XXX-BZ stats but changing lro_ctrl is a problem. */
615 CURVNET_RESTORE();
616 return (TCP_LRO_CANNOT);
617 }
618 CURVNET_RESTORE();
619 l3hdr = ip4 = (struct ip *)(eh + 1);
620 error = tcp_lro_rx_ipv4(lc, m, ip4, &th);
621 if (error != 0)
622 return (error);
623 ip_len = ntohs(ip4->ip_len);
624 tcp_data_len = ip_len - sizeof(*ip4);
625 break;
626 }
627#endif
628 /* XXX-BZ what happens in case of VLAN(s)? */
629 default:
630 return (TCP_LRO_NOT_SUPPORTED);
631 }
632
633 /*
634 * If the frame is padded beyond the end of the IP packet, then we must
635 * trim the extra bytes off.
636 */
637 l = m->m_pkthdr.len - (ETHER_HDR_LEN + ip_len);
638 if (l != 0) {
639 if (l < 0)
640 /* Truncated packet. */
641 return (TCP_LRO_CANNOT);
642
643 m_adj(m, -l);
644 }
645
646 /*
647 * Check TCP header constraints.
648 */
649 /* Ensure no bits set besides ACK or PSH. */
650 if ((th->th_flags & ~(TH_ACK | TH_PUSH)) != 0)
651 return (TCP_LRO_CANNOT);
652
653 /* XXX-BZ We lose a ACK|PUSH flag concatenating multiple segments. */
654 /* XXX-BZ Ideally we'd flush on PUSH? */
655
656 /*
657 * Check for timestamps.
658 * Since the only option we handle are timestamps, we only have to
659 * handle the simple case of aligned timestamps.
660 */
661 l = (th->th_off << 2);
662 tcp_data_len -= l;
663 l -= sizeof(*th);
664 ts_ptr = (uint32_t *)(th + 1);
665 if (l != 0 && (__predict_false(l != TCPOLEN_TSTAMP_APPA) ||
666 (*ts_ptr != ntohl(TCPOPT_NOP<<24|TCPOPT_NOP<<16|
667 TCPOPT_TIMESTAMP<<8|TCPOLEN_TIMESTAMP))))
668 return (TCP_LRO_CANNOT);
669
670 /* If the driver did not pass in the checksum, set it now. */
671 if (csum == 0x0000)
672 csum = th->th_sum;
673
674 seq = ntohl(th->th_seq);
675
676 /* Try to find a matching previous segment. */
677 LIST_FOREACH(le, &lc->lro_active, next) {
678 if (le->eh_type != eh_type)
679 continue;
680 if (le->source_port != th->th_sport ||
681 le->dest_port != th->th_dport)
682 continue;
683 switch (eh_type) {
684#ifdef INET6
685 case ETHERTYPE_IPV6:
686 if (bcmp(&le->source_ip6, &ip6->ip6_src,
687 sizeof(struct in6_addr)) != 0 ||
688 bcmp(&le->dest_ip6, &ip6->ip6_dst,
689 sizeof(struct in6_addr)) != 0)
690 continue;
691 break;
692#endif
693#ifdef INET
694 case ETHERTYPE_IP:
695 if (le->source_ip4 != ip4->ip_src.s_addr ||
696 le->dest_ip4 != ip4->ip_dst.s_addr)
697 continue;
698 break;
699#endif
700 }
701
702 /* Flush now if appending will result in overflow. */
703 if (le->p_len > (lc->lro_length_lim - tcp_data_len)) {
704 tcp_lro_active_remove(le);
705 tcp_lro_flush(lc, le);
706 break;
707 }
708
709 /* Try to append the new segment. */
710 if (__predict_false(seq != le->next_seq ||
711 (tcp_data_len == 0 && le->ack_seq == th->th_ack))) {
712 /* Out of order packet or duplicate ACK. */
713 tcp_lro_active_remove(le);
714 tcp_lro_flush(lc, le);
715 return (TCP_LRO_CANNOT);
716 }
717
718 if (l != 0) {
719 uint32_t tsval = ntohl(*(ts_ptr + 1));
720 /* Make sure timestamp values are increasing. */
721 /* XXX-BZ flip and use TSTMP_GEQ macro for this? */
722 if (__predict_false(le->tsval > tsval ||
723 *(ts_ptr + 2) == 0))
724 return (TCP_LRO_CANNOT);
725 le->tsval = tsval;
726 le->tsecr = *(ts_ptr + 2);
727 }
728
729 le->next_seq += tcp_data_len;
730 le->ack_seq = th->th_ack;
731 le->window = th->th_win;
732 le->append_cnt++;
733
734#ifdef TCP_LRO_UPDATE_CSUM
735 le->ulp_csum += tcp_lro_rx_csum_fixup(le, l3hdr, th,
736 tcp_data_len, ~csum);
737#endif
738
739 if (tcp_data_len == 0) {
740 m_freem(m);
741 /*
742 * Flush this LRO entry, if this ACK should not
743 * be further delayed.
744 */
745 if (le->append_cnt >= lc->lro_ackcnt_lim) {
746 tcp_lro_active_remove(le);
747 tcp_lro_flush(lc, le);
748 }
749 return (0);
750 }
751
752 le->p_len += tcp_data_len;
753
754 /*
755 * Adjust the mbuf so that m_data points to the first byte of
756 * the ULP payload. Adjust the mbuf to avoid complications and
757 * append new segment to existing mbuf chain.
758 */
759 m_adj(m, m->m_pkthdr.len - tcp_data_len);
760 m_demote_pkthdr(m);
761
762 le->m_tail->m_next = m;
763 le->m_tail = m_last(m);
764
765 /*
766 * If a possible next full length packet would cause an
767 * overflow, pro-actively flush now.
768 */
769 if (le->p_len > (lc->lro_length_lim - lc->ifp->if_mtu)) {
770 tcp_lro_active_remove(le);
771 tcp_lro_flush(lc, le);
772 } else
773 getmicrotime(&le->mtime);
774
775 return (0);
776 }
777
778 /* Try to find an empty slot. */
779 if (LIST_EMPTY(&lc->lro_free))
780 return (TCP_LRO_NO_ENTRIES);
781
782 /* Start a new segment chain. */
783 le = LIST_FIRST(&lc->lro_free);
784 LIST_REMOVE(le, next);
785 tcp_lro_active_insert(lc, le);
786 getmicrotime(&le->mtime);
787
788 /* Start filling in details. */
789 switch (eh_type) {
790#ifdef INET6
791 case ETHERTYPE_IPV6:
792 le->le_ip6 = ip6;
793 le->source_ip6 = ip6->ip6_src;
794 le->dest_ip6 = ip6->ip6_dst;
795 le->eh_type = eh_type;
796 le->p_len = m->m_pkthdr.len - ETHER_HDR_LEN - sizeof(*ip6);
797 break;
798#endif
799#ifdef INET
800 case ETHERTYPE_IP:
801 le->le_ip4 = ip4;
802 le->source_ip4 = ip4->ip_src.s_addr;
803 le->dest_ip4 = ip4->ip_dst.s_addr;
804 le->eh_type = eh_type;
805 le->p_len = m->m_pkthdr.len - ETHER_HDR_LEN;
806 break;
807#endif
808 }
809 le->source_port = th->th_sport;
810 le->dest_port = th->th_dport;
811
812 le->next_seq = seq + tcp_data_len;
813 le->ack_seq = th->th_ack;
814 le->window = th->th_win;
815 if (l != 0) {
816 le->timestamp = 1;
817 le->tsval = ntohl(*(ts_ptr + 1));
818 le->tsecr = *(ts_ptr + 2);
819 }
820
821#ifdef TCP_LRO_UPDATE_CSUM
822 /*
823 * Do not touch the csum of the first packet. However save the
824 * "adjusted" checksum of just the source and destination addresses,
825 * the next header and the TCP payload. The length and TCP header
826 * parts may change, so we remove those from the saved checksum and
827 * re-add with final values on tcp_lro_flush() if needed.
828 */
829 KASSERT(le->ulp_csum == 0, ("%s: le=%p le->ulp_csum=0x%04x\n",
830 __func__, le, le->ulp_csum));
831
832 le->ulp_csum = tcp_lro_rx_csum_fixup(le, l3hdr, th, tcp_data_len,
833 ~csum);
834 th->th_sum = csum; /* Restore checksum on first packet. */
835#endif
836
837 le->m_head = m;
838 le->m_tail = m_last(m);
839
840 return (0);
841}
842
843void
844tcp_lro_queue_mbuf(struct lro_ctrl *lc, struct mbuf *mb)
845{
846 /* sanity checks */
847 if (__predict_false(lc->ifp == NULL || lc->lro_mbuf_data == NULL ||
848 lc->lro_mbuf_max == 0)) {
849 /* packet drop */
850 m_freem(mb);
851 return;
852 }
853
854 /* check if packet is not LRO capable */
855 if (__predict_false(mb->m_pkthdr.csum_flags == 0 ||
856 (lc->ifp->if_capenable & IFCAP_LRO) == 0)) {
857 lc->lro_flushed++;
858 lc->lro_queued++;
859
860 /* input packet to network layer */
861 (*lc->ifp->if_input) (lc->ifp, mb);
862 return;
863 }
864
865 /* check if array is full */
866 if (__predict_false(lc->lro_mbuf_count == lc->lro_mbuf_max))
867 tcp_lro_flush_all(lc);
868
869 /* create sequence number */
870 lc->lro_mbuf_data[lc->lro_mbuf_count].seq =
871 (((uint64_t)M_HASHTYPE_GET(mb)) << 56) |
872 (((uint64_t)mb->m_pkthdr.flowid) << 24) |
873 ((uint64_t)lc->lro_mbuf_count);
874
875 /* enter mbuf */
876 lc->lro_mbuf_data[lc->lro_mbuf_count++].mb = mb;
877}
878
879/* end */
| 410 }
411 return;
412 }
413
414 /* compute sequence bits which are constant */
415 ones = 0;
416 zeros = 0;
417 for (x = 0; x != size; x++) {
418 ones |= parray[x].seq;
419 zeros |= ~parray[x].seq;
420 }
421
422 /* compute bits which are not constant into "ones" */
423 ones &= zeros;
424 if (ones == 0)
425 return;
426
427 /* pick the most significant bit which is not constant */
428 ones = tcp_lro_msb_64(ones);
429
430 /*
431 * Move entries having cleared sequence bits to the beginning
432 * of the array:
433 */
434 for (x = y = 0; y != size; y++) {
435 /* skip set bits */
436 if (parray[y].seq & ones)
437 continue;
438 /* swap entries */
439 temp = parray[x];
440 parray[x] = parray[y];
441 parray[y] = temp;
442 x++;
443 }
444
445 KASSERT(x != 0 && x != size, ("Memory is corrupted\n"));
446
447 /* sort zeros */
448 tcp_lro_sort(parray, x);
449
450 /* sort ones */
451 parray += x;
452 size -= x;
453 goto repeat;
454}
455
456void
457tcp_lro_flush_all(struct lro_ctrl *lc)
458{
459 uint64_t seq;
460 uint64_t nseq;
461 unsigned x;
462
463 /* check if no mbufs to flush */
464 if (lc->lro_mbuf_count == 0)
465 goto done;
466
467 /* sort all mbufs according to stream */
468 tcp_lro_sort(lc->lro_mbuf_data, lc->lro_mbuf_count);
469
470 /* input data into LRO engine, stream by stream */
471 seq = 0;
472 for (x = 0; x != lc->lro_mbuf_count; x++) {
473 struct mbuf *mb;
474
475 /* get mbuf */
476 mb = lc->lro_mbuf_data[x].mb;
477
478 /* get sequence number, masking away the packet index */
479 nseq = lc->lro_mbuf_data[x].seq & (-1ULL << 24);
480
481 /* check for new stream */
482 if (seq != nseq) {
483 seq = nseq;
484
485 /* flush active streams */
486 tcp_lro_rx_done(lc);
487 }
488
489 /* add packet to LRO engine */
490 if (tcp_lro_rx(lc, mb, 0) != 0) {
491 /* input packet to network layer */
492 (*lc->ifp->if_input)(lc->ifp, mb);
493 lc->lro_queued++;
494 lc->lro_flushed++;
495 }
496 }
497done:
498 /* flush active streams */
499 tcp_lro_rx_done(lc);
500
501 lc->lro_mbuf_count = 0;
502}
503
504#ifdef INET6
505static int
506tcp_lro_rx_ipv6(struct lro_ctrl *lc, struct mbuf *m, struct ip6_hdr *ip6,
507 struct tcphdr **th)
508{
509
510 /* XXX-BZ we should check the flow-label. */
511
512 /* XXX-BZ We do not yet support ext. hdrs. */
513 if (ip6->ip6_nxt != IPPROTO_TCP)
514 return (TCP_LRO_NOT_SUPPORTED);
515
516 /* Find the TCP header. */
517 *th = (struct tcphdr *)(ip6 + 1);
518
519 return (0);
520}
521#endif
522
523#ifdef INET
524static int
525tcp_lro_rx_ipv4(struct lro_ctrl *lc, struct mbuf *m, struct ip *ip4,
526 struct tcphdr **th)
527{
528 int csum_flags;
529 uint16_t csum;
530
531 if (ip4->ip_p != IPPROTO_TCP)
532 return (TCP_LRO_NOT_SUPPORTED);
533
534 /* Ensure there are no options. */
535 if ((ip4->ip_hl << 2) != sizeof (*ip4))
536 return (TCP_LRO_CANNOT);
537
538 /* .. and the packet is not fragmented. */
539 if (ip4->ip_off & htons(IP_MF|IP_OFFMASK))
540 return (TCP_LRO_CANNOT);
541
542 /* Legacy IP has a header checksum that needs to be correct. */
543 csum_flags = m->m_pkthdr.csum_flags;
544 if (csum_flags & CSUM_IP_CHECKED) {
545 if (__predict_false((csum_flags & CSUM_IP_VALID) == 0)) {
546 lc->lro_bad_csum++;
547 return (TCP_LRO_CANNOT);
548 }
549 } else {
550 csum = in_cksum_hdr(ip4);
551 if (__predict_false((csum) != 0)) {
552 lc->lro_bad_csum++;
553 return (TCP_LRO_CANNOT);
554 }
555 }
556
557 /* Find the TCP header (we assured there are no IP options). */
558 *th = (struct tcphdr *)(ip4 + 1);
559
560 return (0);
561}
562#endif
563
564int
565tcp_lro_rx(struct lro_ctrl *lc, struct mbuf *m, uint32_t csum)
566{
567 struct lro_entry *le;
568 struct ether_header *eh;
569#ifdef INET6
570 struct ip6_hdr *ip6 = NULL; /* Keep compiler happy. */
571#endif
572#ifdef INET
573 struct ip *ip4 = NULL; /* Keep compiler happy. */
574#endif
575 struct tcphdr *th;
576 void *l3hdr = NULL; /* Keep compiler happy. */
577 uint32_t *ts_ptr;
578 tcp_seq seq;
579 int error, ip_len, l;
580 uint16_t eh_type, tcp_data_len;
581
582 /* We expect a contiguous header [eh, ip, tcp]. */
583
584 eh = mtod(m, struct ether_header *);
585 eh_type = ntohs(eh->ether_type);
586 switch (eh_type) {
587#ifdef INET6
588 case ETHERTYPE_IPV6:
589 {
590 CURVNET_SET(lc->ifp->if_vnet);
591 if (V_ip6_forwarding != 0) {
592 /* XXX-BZ stats but changing lro_ctrl is a problem. */
593 CURVNET_RESTORE();
594 return (TCP_LRO_CANNOT);
595 }
596 CURVNET_RESTORE();
597 l3hdr = ip6 = (struct ip6_hdr *)(eh + 1);
598 error = tcp_lro_rx_ipv6(lc, m, ip6, &th);
599 if (error != 0)
600 return (error);
601 tcp_data_len = ntohs(ip6->ip6_plen);
602 ip_len = sizeof(*ip6) + tcp_data_len;
603 break;
604 }
605#endif
606#ifdef INET
607 case ETHERTYPE_IP:
608 {
609 CURVNET_SET(lc->ifp->if_vnet);
610 if (V_ipforwarding != 0) {
611 /* XXX-BZ stats but changing lro_ctrl is a problem. */
612 CURVNET_RESTORE();
613 return (TCP_LRO_CANNOT);
614 }
615 CURVNET_RESTORE();
616 l3hdr = ip4 = (struct ip *)(eh + 1);
617 error = tcp_lro_rx_ipv4(lc, m, ip4, &th);
618 if (error != 0)
619 return (error);
620 ip_len = ntohs(ip4->ip_len);
621 tcp_data_len = ip_len - sizeof(*ip4);
622 break;
623 }
624#endif
625 /* XXX-BZ what happens in case of VLAN(s)? */
626 default:
627 return (TCP_LRO_NOT_SUPPORTED);
628 }
629
630 /*
631 * If the frame is padded beyond the end of the IP packet, then we must
632 * trim the extra bytes off.
633 */
634 l = m->m_pkthdr.len - (ETHER_HDR_LEN + ip_len);
635 if (l != 0) {
636 if (l < 0)
637 /* Truncated packet. */
638 return (TCP_LRO_CANNOT);
639
640 m_adj(m, -l);
641 }
642
643 /*
644 * Check TCP header constraints.
645 */
646 /* Ensure no bits set besides ACK or PSH. */
647 if ((th->th_flags & ~(TH_ACK | TH_PUSH)) != 0)
648 return (TCP_LRO_CANNOT);
649
650 /* XXX-BZ We lose a ACK|PUSH flag concatenating multiple segments. */
651 /* XXX-BZ Ideally we'd flush on PUSH? */
652
653 /*
654 * Check for timestamps.
655 * Since the only option we handle are timestamps, we only have to
656 * handle the simple case of aligned timestamps.
657 */
658 l = (th->th_off << 2);
659 tcp_data_len -= l;
660 l -= sizeof(*th);
661 ts_ptr = (uint32_t *)(th + 1);
662 if (l != 0 && (__predict_false(l != TCPOLEN_TSTAMP_APPA) ||
663 (*ts_ptr != ntohl(TCPOPT_NOP<<24|TCPOPT_NOP<<16|
664 TCPOPT_TIMESTAMP<<8|TCPOLEN_TIMESTAMP))))
665 return (TCP_LRO_CANNOT);
666
667 /* If the driver did not pass in the checksum, set it now. */
668 if (csum == 0x0000)
669 csum = th->th_sum;
670
671 seq = ntohl(th->th_seq);
672
673 /* Try to find a matching previous segment. */
674 LIST_FOREACH(le, &lc->lro_active, next) {
675 if (le->eh_type != eh_type)
676 continue;
677 if (le->source_port != th->th_sport ||
678 le->dest_port != th->th_dport)
679 continue;
680 switch (eh_type) {
681#ifdef INET6
682 case ETHERTYPE_IPV6:
683 if (bcmp(&le->source_ip6, &ip6->ip6_src,
684 sizeof(struct in6_addr)) != 0 ||
685 bcmp(&le->dest_ip6, &ip6->ip6_dst,
686 sizeof(struct in6_addr)) != 0)
687 continue;
688 break;
689#endif
690#ifdef INET
691 case ETHERTYPE_IP:
692 if (le->source_ip4 != ip4->ip_src.s_addr ||
693 le->dest_ip4 != ip4->ip_dst.s_addr)
694 continue;
695 break;
696#endif
697 }
698
699 /* Flush now if appending will result in overflow. */
700 if (le->p_len > (lc->lro_length_lim - tcp_data_len)) {
701 tcp_lro_active_remove(le);
702 tcp_lro_flush(lc, le);
703 break;
704 }
705
706 /* Try to append the new segment. */
707 if (__predict_false(seq != le->next_seq ||
708 (tcp_data_len == 0 && le->ack_seq == th->th_ack))) {
709 /* Out of order packet or duplicate ACK. */
710 tcp_lro_active_remove(le);
711 tcp_lro_flush(lc, le);
712 return (TCP_LRO_CANNOT);
713 }
714
715 if (l != 0) {
716 uint32_t tsval = ntohl(*(ts_ptr + 1));
717 /* Make sure timestamp values are increasing. */
718 /* XXX-BZ flip and use TSTMP_GEQ macro for this? */
719 if (__predict_false(le->tsval > tsval ||
720 *(ts_ptr + 2) == 0))
721 return (TCP_LRO_CANNOT);
722 le->tsval = tsval;
723 le->tsecr = *(ts_ptr + 2);
724 }
725
726 le->next_seq += tcp_data_len;
727 le->ack_seq = th->th_ack;
728 le->window = th->th_win;
729 le->append_cnt++;
730
731#ifdef TCP_LRO_UPDATE_CSUM
732 le->ulp_csum += tcp_lro_rx_csum_fixup(le, l3hdr, th,
733 tcp_data_len, ~csum);
734#endif
735
736 if (tcp_data_len == 0) {
737 m_freem(m);
738 /*
739 * Flush this LRO entry, if this ACK should not
740 * be further delayed.
741 */
742 if (le->append_cnt >= lc->lro_ackcnt_lim) {
743 tcp_lro_active_remove(le);
744 tcp_lro_flush(lc, le);
745 }
746 return (0);
747 }
748
749 le->p_len += tcp_data_len;
750
751 /*
752 * Adjust the mbuf so that m_data points to the first byte of
753 * the ULP payload. Adjust the mbuf to avoid complications and
754 * append new segment to existing mbuf chain.
755 */
756 m_adj(m, m->m_pkthdr.len - tcp_data_len);
757 m_demote_pkthdr(m);
758
759 le->m_tail->m_next = m;
760 le->m_tail = m_last(m);
761
762 /*
763 * If a possible next full length packet would cause an
764 * overflow, pro-actively flush now.
765 */
766 if (le->p_len > (lc->lro_length_lim - lc->ifp->if_mtu)) {
767 tcp_lro_active_remove(le);
768 tcp_lro_flush(lc, le);
769 } else
770 getmicrotime(&le->mtime);
771
772 return (0);
773 }
774
775 /* Try to find an empty slot. */
776 if (LIST_EMPTY(&lc->lro_free))
777 return (TCP_LRO_NO_ENTRIES);
778
779 /* Start a new segment chain. */
780 le = LIST_FIRST(&lc->lro_free);
781 LIST_REMOVE(le, next);
782 tcp_lro_active_insert(lc, le);
783 getmicrotime(&le->mtime);
784
785 /* Start filling in details. */
786 switch (eh_type) {
787#ifdef INET6
788 case ETHERTYPE_IPV6:
789 le->le_ip6 = ip6;
790 le->source_ip6 = ip6->ip6_src;
791 le->dest_ip6 = ip6->ip6_dst;
792 le->eh_type = eh_type;
793 le->p_len = m->m_pkthdr.len - ETHER_HDR_LEN - sizeof(*ip6);
794 break;
795#endif
796#ifdef INET
797 case ETHERTYPE_IP:
798 le->le_ip4 = ip4;
799 le->source_ip4 = ip4->ip_src.s_addr;
800 le->dest_ip4 = ip4->ip_dst.s_addr;
801 le->eh_type = eh_type;
802 le->p_len = m->m_pkthdr.len - ETHER_HDR_LEN;
803 break;
804#endif
805 }
806 le->source_port = th->th_sport;
807 le->dest_port = th->th_dport;
808
809 le->next_seq = seq + tcp_data_len;
810 le->ack_seq = th->th_ack;
811 le->window = th->th_win;
812 if (l != 0) {
813 le->timestamp = 1;
814 le->tsval = ntohl(*(ts_ptr + 1));
815 le->tsecr = *(ts_ptr + 2);
816 }
817
818#ifdef TCP_LRO_UPDATE_CSUM
819 /*
820 * Do not touch the csum of the first packet. However save the
821 * "adjusted" checksum of just the source and destination addresses,
822 * the next header and the TCP payload. The length and TCP header
823 * parts may change, so we remove those from the saved checksum and
824 * re-add with final values on tcp_lro_flush() if needed.
825 */
826 KASSERT(le->ulp_csum == 0, ("%s: le=%p le->ulp_csum=0x%04x\n",
827 __func__, le, le->ulp_csum));
828
829 le->ulp_csum = tcp_lro_rx_csum_fixup(le, l3hdr, th, tcp_data_len,
830 ~csum);
831 th->th_sum = csum; /* Restore checksum on first packet. */
832#endif
833
834 le->m_head = m;
835 le->m_tail = m_last(m);
836
837 return (0);
838}
839
840void
841tcp_lro_queue_mbuf(struct lro_ctrl *lc, struct mbuf *mb)
842{
843 /* sanity checks */
844 if (__predict_false(lc->ifp == NULL || lc->lro_mbuf_data == NULL ||
845 lc->lro_mbuf_max == 0)) {
846 /* packet drop */
847 m_freem(mb);
848 return;
849 }
850
851 /* check if packet is not LRO capable */
852 if (__predict_false(mb->m_pkthdr.csum_flags == 0 ||
853 (lc->ifp->if_capenable & IFCAP_LRO) == 0)) {
854 lc->lro_flushed++;
855 lc->lro_queued++;
856
857 /* input packet to network layer */
858 (*lc->ifp->if_input) (lc->ifp, mb);
859 return;
860 }
861
862 /* check if array is full */
863 if (__predict_false(lc->lro_mbuf_count == lc->lro_mbuf_max))
864 tcp_lro_flush_all(lc);
865
866 /* create sequence number */
867 lc->lro_mbuf_data[lc->lro_mbuf_count].seq =
868 (((uint64_t)M_HASHTYPE_GET(mb)) << 56) |
869 (((uint64_t)mb->m_pkthdr.flowid) << 24) |
870 ((uint64_t)lc->lro_mbuf_count);
871
872 /* enter mbuf */
873 lc->lro_mbuf_data[lc->lro_mbuf_count++].mb = mb;
874}
875
876/* end */
|