1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 *	IPV6 GSO/GRO offload support
4 *	Linux INET6 implementation
5 */
6
7#include <linux/kernel.h>
8#include <linux/socket.h>
9#include <linux/netdevice.h>
10#include <linux/skbuff.h>
11#include <linux/printk.h>
12
13#include <net/protocol.h>
14#include <net/ipv6.h>
15#include <net/inet_common.h>
16#include <net/tcp.h>
17#include <net/udp.h>
18#include <net/gro.h>
19#include <net/gso.h>
20
21#include "ip6_offload.h"
22
23/* All GRO functions are always builtin, except UDP over ipv6, which lays in
24 * ipv6 module, as it depends on UDPv6 lookup function, so we need special care
25 * when ipv6 is built as a module
26 */
27#if IS_BUILTIN(CONFIG_IPV6)
28#define INDIRECT_CALL_L4(f, f2, f1, ...) INDIRECT_CALL_2(f, f2, f1, __VA_ARGS__)
29#else
30#define INDIRECT_CALL_L4(f, f2, f1, ...) INDIRECT_CALL_1(f, f2, __VA_ARGS__)
31#endif
32
33#define indirect_call_gro_receive_l4(f2, f1, cb, head, skb)	\
34({								\
35	unlikely(gro_recursion_inc_test(skb)) ?			\
36		NAPI_GRO_CB(skb)->flush |= 1, NULL :		\
37		INDIRECT_CALL_L4(cb, f2, f1, head, skb);	\
38})
39
40static int ipv6_gro_pull_exthdrs(struct sk_buff *skb, int off, int proto)
41{
42	const struct net_offload *ops = NULL;
43	struct ipv6_opt_hdr *opth;
44
45	for (;;) {
46		int len;
47
48		ops = rcu_dereference(inet6_offloads[proto]);
49
50		if (unlikely(!ops))
51			break;
52
53		if (!(ops->flags & INET6_PROTO_GSO_EXTHDR))
54			break;
55
56		opth = skb_gro_header(skb, off + sizeof(*opth), off);
57		if (unlikely(!opth))
58			break;
59
60		len = ipv6_optlen(opth);
61
62		opth = skb_gro_header(skb, off + len, off);
63		if (unlikely(!opth))
64			break;
65		proto = opth->nexthdr;
66
67		off += len;
68	}
69
70	skb_gro_pull(skb, off - skb_network_offset(skb));
71	return proto;
72}
73
74static int ipv6_gso_pull_exthdrs(struct sk_buff *skb, int proto)
75{
76	const struct net_offload *ops = NULL;
77
78	for (;;) {
79		struct ipv6_opt_hdr *opth;
80		int len;
81
82		ops = rcu_dereference(inet6_offloads[proto]);
83
84		if (unlikely(!ops))
85			break;
86
87		if (!(ops->flags & INET6_PROTO_GSO_EXTHDR))
88			break;
89
90		if (unlikely(!pskb_may_pull(skb, 8)))
91			break;
92
93		opth = (void *)skb->data;
94		len = ipv6_optlen(opth);
95
96		if (unlikely(!pskb_may_pull(skb, len)))
97			break;
98
99		opth = (void *)skb->data;
100		proto = opth->nexthdr;
101		__skb_pull(skb, len);
102	}
103
104	return proto;
105}
106
107static struct sk_buff *ipv6_gso_segment(struct sk_buff *skb,
108	netdev_features_t features)
109{
110	struct sk_buff *segs = ERR_PTR(-EINVAL);
111	struct ipv6hdr *ipv6h;
112	const struct net_offload *ops;
113	int proto, err;
114	struct frag_hdr *fptr;
115	unsigned int payload_len;
116	u8 *prevhdr;
117	int offset = 0;
118	bool encap, udpfrag;
119	int nhoff;
120	bool gso_partial;
121
122	skb_reset_network_header(skb);
123	err = ipv6_hopopt_jumbo_remove(skb);
124	if (err)
125		return ERR_PTR(err);
126	nhoff = skb_network_header(skb) - skb_mac_header(skb);
127	if (unlikely(!pskb_may_pull(skb, sizeof(*ipv6h))))
128		goto out;
129
130	encap = SKB_GSO_CB(skb)->encap_level > 0;
131	if (encap)
132		features &= skb->dev->hw_enc_features;
133	SKB_GSO_CB(skb)->encap_level += sizeof(*ipv6h);
134
135	ipv6h = ipv6_hdr(skb);
136	__skb_pull(skb, sizeof(*ipv6h));
137	segs = ERR_PTR(-EPROTONOSUPPORT);
138
139	proto = ipv6_gso_pull_exthdrs(skb, ipv6h->nexthdr);
140
141	if (skb->encapsulation &&
142	    skb_shinfo(skb)->gso_type & (SKB_GSO_IPXIP4 | SKB_GSO_IPXIP6))
143		udpfrag = proto == IPPROTO_UDP && encap &&
144			  (skb_shinfo(skb)->gso_type & SKB_GSO_UDP);
145	else
146		udpfrag = proto == IPPROTO_UDP && !skb->encapsulation &&
147			  (skb_shinfo(skb)->gso_type & SKB_GSO_UDP);
148
149	ops = rcu_dereference(inet6_offloads[proto]);
150	if (likely(ops && ops->callbacks.gso_segment)) {
151		skb_reset_transport_header(skb);
152		segs = ops->callbacks.gso_segment(skb, features);
153		if (!segs)
154			skb->network_header = skb_mac_header(skb) + nhoff - skb->head;
155	}
156
157	if (IS_ERR_OR_NULL(segs))
158		goto out;
159
160	gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL);
161
162	for (skb = segs; skb; skb = skb->next) {
163		ipv6h = (struct ipv6hdr *)(skb_mac_header(skb) + nhoff);
164		if (gso_partial && skb_is_gso(skb))
165			payload_len = skb_shinfo(skb)->gso_size +
166				      SKB_GSO_CB(skb)->data_offset +
167				      skb->head - (unsigned char *)(ipv6h + 1);
168		else
169			payload_len = skb->len - nhoff - sizeof(*ipv6h);
170		ipv6h->payload_len = htons(payload_len);
171		skb->network_header = (u8 *)ipv6h - skb->head;
172		skb_reset_mac_len(skb);
173
174		if (udpfrag) {
175			int err = ip6_find_1stfragopt(skb, &prevhdr);
176			if (err < 0) {
177				kfree_skb_list(segs);
178				return ERR_PTR(err);
179			}
180			fptr = (struct frag_hdr *)((u8 *)ipv6h + err);
181			fptr->frag_off = htons(offset);
182			if (skb->next)
183				fptr->frag_off |= htons(IP6_MF);
184			offset += (ntohs(ipv6h->payload_len) -
185				   sizeof(struct frag_hdr));
186		}
187		if (encap)
188			skb_reset_inner_headers(skb);
189	}
190
191out:
192	return segs;
193}
194
195/* Return the total length of all the extension hdrs, following the same
196 * logic in ipv6_gso_pull_exthdrs() when parsing ext-hdrs.
197 */
198static int ipv6_exthdrs_len(struct ipv6hdr *iph,
199			    const struct net_offload **opps)
200{
201	struct ipv6_opt_hdr *opth = (void *)iph;
202	int len = 0, proto, optlen = sizeof(*iph);
203
204	proto = iph->nexthdr;
205	for (;;) {
206		*opps = rcu_dereference(inet6_offloads[proto]);
207		if (unlikely(!(*opps)))
208			break;
209		if (!((*opps)->flags & INET6_PROTO_GSO_EXTHDR))
210			break;
211
212		opth = (void *)opth + optlen;
213		optlen = ipv6_optlen(opth);
214		len += optlen;
215		proto = opth->nexthdr;
216	}
217	return len;
218}
219
220INDIRECT_CALLABLE_SCOPE struct sk_buff *ipv6_gro_receive(struct list_head *head,
221							 struct sk_buff *skb)
222{
223	const struct net_offload *ops;
224	struct sk_buff *pp = NULL;
225	struct sk_buff *p;
226	struct ipv6hdr *iph;
227	unsigned int nlen;
228	unsigned int hlen;
229	unsigned int off;
230	u16 flush = 1;
231	int proto;
232
233	off = skb_gro_offset(skb);
234	hlen = off + sizeof(*iph);
235	iph = skb_gro_header(skb, hlen, off);
236	if (unlikely(!iph))
237		goto out;
238
239	skb_set_network_header(skb, off);
240	NAPI_GRO_CB(skb)->inner_network_offset = off;
241
242	flush += ntohs(iph->payload_len) != skb->len - hlen;
243
244	proto = iph->nexthdr;
245	ops = rcu_dereference(inet6_offloads[proto]);
246	if (!ops || !ops->callbacks.gro_receive) {
247		proto = ipv6_gro_pull_exthdrs(skb, hlen, proto);
248
249		ops = rcu_dereference(inet6_offloads[proto]);
250		if (!ops || !ops->callbacks.gro_receive)
251			goto out;
252
253		iph = skb_gro_network_header(skb);
254	} else {
255		skb_gro_pull(skb, sizeof(*iph));
256	}
257
258	skb_set_transport_header(skb, skb_gro_offset(skb));
259
260	NAPI_GRO_CB(skb)->proto = proto;
261
262	flush--;
263	nlen = skb_network_header_len(skb);
264
265	list_for_each_entry(p, head, list) {
266		const struct ipv6hdr *iph2;
267		__be32 first_word; /* <Version:4><Traffic_Class:8><Flow_Label:20> */
268
269		if (!NAPI_GRO_CB(p)->same_flow)
270			continue;
271
272		iph2 = (struct ipv6hdr *)(p->data + off);
273		first_word = *(__be32 *)iph ^ *(__be32 *)iph2;
274
275		/* All fields must match except length and Traffic Class.
276		 * XXX skbs on the gro_list have all been parsed and pulled
277		 * already so we don't need to compare nlen
278		 * (nlen != (sizeof(*iph2) + ipv6_exthdrs_len(iph2, &ops)))
279		 * memcmp() alone below is sufficient, right?
280		 */
281		 if ((first_word & htonl(0xF00FFFFF)) ||
282		     !ipv6_addr_equal(&iph->saddr, &iph2->saddr) ||
283		     !ipv6_addr_equal(&iph->daddr, &iph2->daddr) ||
284		     iph->nexthdr != iph2->nexthdr) {
285not_same_flow:
286			NAPI_GRO_CB(p)->same_flow = 0;
287			continue;
288		}
289		if (unlikely(nlen > sizeof(struct ipv6hdr))) {
290			if (memcmp(iph + 1, iph2 + 1,
291				   nlen - sizeof(struct ipv6hdr)))
292				goto not_same_flow;
293		}
294		/* flush if Traffic Class fields are different */
295		NAPI_GRO_CB(p)->flush |= !!((first_word & htonl(0x0FF00000)) |
296			(__force __be32)(iph->hop_limit ^ iph2->hop_limit));
297		NAPI_GRO_CB(p)->flush |= flush;
298
299		/* If the previous IP ID value was based on an atomic
300		 * datagram we can overwrite the value and ignore it.
301		 */
302		if (NAPI_GRO_CB(skb)->is_atomic)
303			NAPI_GRO_CB(p)->flush_id = 0;
304	}
305
306	NAPI_GRO_CB(skb)->is_atomic = true;
307	NAPI_GRO_CB(skb)->flush |= flush;
308
309	skb_gro_postpull_rcsum(skb, iph, nlen);
310
311	pp = indirect_call_gro_receive_l4(tcp6_gro_receive, udp6_gro_receive,
312					 ops->callbacks.gro_receive, head, skb);
313
314out:
315	skb_gro_flush_final(skb, pp, flush);
316
317	return pp;
318}
319
320static struct sk_buff *sit_ip6ip6_gro_receive(struct list_head *head,
321					      struct sk_buff *skb)
322{
323	/* Common GRO receive for SIT and IP6IP6 */
324
325	if (NAPI_GRO_CB(skb)->encap_mark) {
326		NAPI_GRO_CB(skb)->flush = 1;
327		return NULL;
328	}
329
330	NAPI_GRO_CB(skb)->encap_mark = 1;
331
332	return ipv6_gro_receive(head, skb);
333}
334
335static struct sk_buff *ip4ip6_gro_receive(struct list_head *head,
336					  struct sk_buff *skb)
337{
338	/* Common GRO receive for SIT and IP6IP6 */
339
340	if (NAPI_GRO_CB(skb)->encap_mark) {
341		NAPI_GRO_CB(skb)->flush = 1;
342		return NULL;
343	}
344
345	NAPI_GRO_CB(skb)->encap_mark = 1;
346
347	return inet_gro_receive(head, skb);
348}
349
350INDIRECT_CALLABLE_SCOPE int ipv6_gro_complete(struct sk_buff *skb, int nhoff)
351{
352	const struct net_offload *ops;
353	struct ipv6hdr *iph;
354	int err = -ENOSYS;
355	u32 payload_len;
356
357	if (skb->encapsulation) {
358		skb_set_inner_protocol(skb, cpu_to_be16(ETH_P_IPV6));
359		skb_set_inner_network_header(skb, nhoff);
360	}
361
362	payload_len = skb->len - nhoff - sizeof(*iph);
363	if (unlikely(payload_len > IPV6_MAXPLEN)) {
364		struct hop_jumbo_hdr *hop_jumbo;
365		int hoplen = sizeof(*hop_jumbo);
366
367		/* Move network header left */
368		memmove(skb_mac_header(skb) - hoplen, skb_mac_header(skb),
369			skb->transport_header - skb->mac_header);
370		skb->data -= hoplen;
371		skb->len += hoplen;
372		skb->mac_header -= hoplen;
373		skb->network_header -= hoplen;
374		iph = (struct ipv6hdr *)(skb->data + nhoff);
375		hop_jumbo = (struct hop_jumbo_hdr *)(iph + 1);
376
377		/* Build hop-by-hop options */
378		hop_jumbo->nexthdr = iph->nexthdr;
379		hop_jumbo->hdrlen = 0;
380		hop_jumbo->tlv_type = IPV6_TLV_JUMBO;
381		hop_jumbo->tlv_len = 4;
382		hop_jumbo->jumbo_payload_len = htonl(payload_len + hoplen);
383
384		iph->nexthdr = NEXTHDR_HOP;
385		iph->payload_len = 0;
386	} else {
387		iph = (struct ipv6hdr *)(skb->data + nhoff);
388		iph->payload_len = htons(payload_len);
389	}
390
391	nhoff += sizeof(*iph) + ipv6_exthdrs_len(iph, &ops);
392	if (WARN_ON(!ops || !ops->callbacks.gro_complete))
393		goto out;
394
395	err = INDIRECT_CALL_L4(ops->callbacks.gro_complete, tcp6_gro_complete,
396			       udp6_gro_complete, skb, nhoff);
397
398out:
399	return err;
400}
401
402static int sit_gro_complete(struct sk_buff *skb, int nhoff)
403{
404	skb->encapsulation = 1;
405	skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP4;
406	return ipv6_gro_complete(skb, nhoff);
407}
408
409static int ip6ip6_gro_complete(struct sk_buff *skb, int nhoff)
410{
411	skb->encapsulation = 1;
412	skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;
413	return ipv6_gro_complete(skb, nhoff);
414}
415
416static int ip4ip6_gro_complete(struct sk_buff *skb, int nhoff)
417{
418	skb->encapsulation = 1;
419	skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;
420	return inet_gro_complete(skb, nhoff);
421}
422
423
424static struct sk_buff *sit_gso_segment(struct sk_buff *skb,
425				       netdev_features_t features)
426{
427	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP4))
428		return ERR_PTR(-EINVAL);
429
430	return ipv6_gso_segment(skb, features);
431}
432
433static struct sk_buff *ip4ip6_gso_segment(struct sk_buff *skb,
434					  netdev_features_t features)
435{
436	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6))
437		return ERR_PTR(-EINVAL);
438
439	return inet_gso_segment(skb, features);
440}
441
442static struct sk_buff *ip6ip6_gso_segment(struct sk_buff *skb,
443					  netdev_features_t features)
444{
445	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6))
446		return ERR_PTR(-EINVAL);
447
448	return ipv6_gso_segment(skb, features);
449}
450
451static const struct net_offload sit_offload = {
452	.callbacks = {
453		.gso_segment	= sit_gso_segment,
454		.gro_receive    = sit_ip6ip6_gro_receive,
455		.gro_complete   = sit_gro_complete,
456	},
457};
458
459static const struct net_offload ip4ip6_offload = {
460	.callbacks = {
461		.gso_segment	= ip4ip6_gso_segment,
462		.gro_receive    = ip4ip6_gro_receive,
463		.gro_complete   = ip4ip6_gro_complete,
464	},
465};
466
467static const struct net_offload ip6ip6_offload = {
468	.callbacks = {
469		.gso_segment	= ip6ip6_gso_segment,
470		.gro_receive    = sit_ip6ip6_gro_receive,
471		.gro_complete   = ip6ip6_gro_complete,
472	},
473};
474static int __init ipv6_offload_init(void)
475{
476
477	if (tcpv6_offload_init() < 0)
478		pr_crit("%s: Cannot add TCP protocol offload\n", __func__);
479	if (ipv6_exthdrs_offload_init() < 0)
480		pr_crit("%s: Cannot add EXTHDRS protocol offload\n", __func__);
481
482	net_hotdata.ipv6_packet_offload = (struct packet_offload) {
483		.type = cpu_to_be16(ETH_P_IPV6),
484		.callbacks = {
485			.gso_segment = ipv6_gso_segment,
486			.gro_receive = ipv6_gro_receive,
487			.gro_complete = ipv6_gro_complete,
488		},
489	};
490	dev_add_offload(&net_hotdata.ipv6_packet_offload);
491
492	inet_add_offload(&sit_offload, IPPROTO_IPV6);
493	inet6_add_offload(&ip6ip6_offload, IPPROTO_IPV6);
494	inet6_add_offload(&ip4ip6_offload, IPPROTO_IPIP);
495
496	return 0;
497}
498
499fs_initcall(ipv6_offload_init);
500