13229Spst// SPDX-License-Identifier: GPL-2.0-only 23229Spst/* 33229Spst * IPV6 GSO/GRO offload support 43229Spst * Linux INET implementation 53229Spst * 63229Spst * Copyright (C) 2016 secunet Security Networks AG 73229Spst * Author: Steffen Klassert <steffen.klassert@secunet.com> 83229Spst * 93229Spst * ESP GRO support 103229Spst */ 113229Spst 123229Spst#include <linux/skbuff.h> 133229Spst#include <linux/init.h> 143229Spst#include <net/protocol.h> 153229Spst#include <crypto/aead.h> 163229Spst#include <crypto/authenc.h> 173229Spst#include <linux/err.h> 183229Spst#include <linux/module.h> 193229Spst#include <net/gro.h> 203229Spst#include <net/gso.h> 2118471Swosch#include <net/ip.h> 2250476Speter#include <net/xfrm.h> 2318471Swosch#include <net/esp.h> 243229Spst#include <linux/scatterlist.h> 253229Spst#include <linux/kernel.h> 263229Spst#include <linux/slab.h> 273229Spst#include <linux/spinlock.h> 283229Spst#include <net/ip6_route.h> 293229Spst#include <net/ipv6.h> 303229Spst#include <linux/icmpv6.h> 313229Spst 323229Spststatic __u16 esp6_nexthdr_esp_offset(struct ipv6hdr *ipv6_hdr, int nhlen) 333229Spst{ 343229Spst int off = sizeof(struct ipv6hdr); 353229Spst struct ipv6_opt_hdr *exthdr; 363229Spst 373229Spst /* ESP or ESPINUDP */ 383229Spst if (likely(ipv6_hdr->nexthdr == NEXTHDR_ESP || 393229Spst ipv6_hdr->nexthdr == NEXTHDR_UDP)) 403229Spst return offsetof(struct ipv6hdr, nexthdr); 413229Spst 423229Spst while (off < nhlen) { 433229Spst exthdr = (void *)ipv6_hdr + off; 443229Spst if (exthdr->nexthdr == NEXTHDR_ESP) 453229Spst return off; 463229Spst 473229Spst off += ipv6_optlen(exthdr); 483229Spst } 493229Spst 503229Spst return 0; 513229Spst} 523229Spst 533229Spststatic struct sk_buff *esp6_gro_receive(struct list_head *head, 543229Spst struct sk_buff *skb) 553229Spst{ 563229Spst int offset = skb_gro_offset(skb); 573229Spst struct xfrm_offload *xo; 583229Spst struct xfrm_state *x; 593229Spst int encap_type = 0; 603229Spst __be32 seq; 613229Spst __be32 spi; 623229Spst int nhoff; 633229Spst 643229Spst if (NAPI_GRO_CB(skb)->proto == IPPROTO_UDP) 653229Spst encap_type = UDP_ENCAP_ESPINUDP; 663229Spst 673229Spst if (!pskb_pull(skb, offset)) 683229Spst return NULL; 693229Spst 703229Spst if (xfrm_parse_spi(skb, IPPROTO_ESP, &spi, &seq) != 0) 713229Spst goto out; 723229Spst 733229Spst xo = xfrm_offload(skb); 743229Spst if (!xo || !(xo->flags & CRYPTO_DONE)) { 753229Spst struct sec_path *sp = secpath_set(skb); 763229Spst 773229Spst if (!sp) 783229Spst goto out; 793229Spst 803229Spst if (sp->len == XFRM_MAX_DEPTH) 813229Spst goto out_reset; 823229Spst 833229Spst x = xfrm_state_lookup(dev_net(skb->dev), skb->mark, 843229Spst (xfrm_address_t *)&ipv6_hdr(skb)->daddr, 853229Spst spi, IPPROTO_ESP, AF_INET6); 863229Spst if (!x) 873229Spst goto out_reset; 8897419Salfred 8997419Salfred skb->mark = xfrm_smark_get(skb->mark, x); 903229Spst 913229Spst sp->xvec[sp->len++] = x; 923229Spst sp->olen++; 933229Spst 943229Spst xo = xfrm_offload(skb); 953229Spst if (!xo) 963229Spst goto out_reset; 973229Spst } 983229Spst 993229Spst xo->flags |= XFRM_GRO; 1003229Spst 1013229Spst nhoff = esp6_nexthdr_esp_offset(ipv6_hdr(skb), offset); 1023229Spst if (!nhoff) 1033229Spst goto out; 1043229Spst 1053229Spst IP6CB(skb)->nhoff = nhoff; 1063229Spst XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6 = NULL; 1073229Spst XFRM_SPI_SKB_CB(skb)->family = AF_INET6; 1083229Spst XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct ipv6hdr, daddr); 1093229Spst XFRM_SPI_SKB_CB(skb)->seq = seq; 1103229Spst 1113229Spst /* We don't need to handle errors from xfrm_input, it does all 1123229Spst * the error handling and frees the resources on error. */ 1133229Spst xfrm_input(skb, IPPROTO_ESP, spi, encap_type); 1143229Spst 1153229Spst return ERR_PTR(-EINPROGRESS); 1163229Spstout_reset: 1173229Spst secpath_reset(skb); 1183229Spstout: 1193229Spst skb_push(skb, offset); 1203229Spst NAPI_GRO_CB(skb)->same_flow = 0; 1213229Spst NAPI_GRO_CB(skb)->flush = 1; 1223229Spst 1233229Spst return NULL; 1243229Spst} 1253229Spst 12646078Simpstatic void esp6_gso_encap(struct xfrm_state *x, struct sk_buff *skb) 1273229Spst{ 1283229Spst struct ip_esp_hdr *esph; 1293229Spst struct ipv6hdr *iph = ipv6_hdr(skb); 1303229Spst struct xfrm_offload *xo = xfrm_offload(skb); 1313229Spst u8 proto = iph->nexthdr; 1323229Spst 1333229Spst skb_push(skb, -skb_network_offset(skb)); 1343229Spst 1353229Spst if (x->outer_mode.encap == XFRM_MODE_TRANSPORT) { 1363229Spst __be16 frag; 1373229Spst 1383229Spst ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &proto, &frag); 1393229Spst } 1403229Spst 1413229Spst esph = ip_esp_hdr(skb); 1423229Spst *skb_mac_header(skb) = IPPROTO_ESP; 1433229Spst 1443229Spst esph->spi = x->id.spi; 1453229Spst esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output.low); 1463229Spst 1473229Spst xo->proto = proto; 1483229Spst} 1493229Spst 1503229Spststatic struct sk_buff *xfrm6_tunnel_gso_segment(struct xfrm_state *x, 1513229Spst struct sk_buff *skb, 1523229Spst netdev_features_t features) 1533229Spst{ 1543229Spst __be16 type = x->inner_mode.family == AF_INET ? htons(ETH_P_IP) 1553229Spst : htons(ETH_P_IPV6); 1563229Spst 1573229Spst return skb_eth_gso_segment(skb, features, type); 1583229Spst} 1593229Spst 1603229Spststatic struct sk_buff *xfrm6_transport_gso_segment(struct xfrm_state *x, 1613229Spst struct sk_buff *skb, 1623229Spst netdev_features_t features) 1633229Spst{ 1643229Spst const struct net_offload *ops; 1653229Spst struct sk_buff *segs = ERR_PTR(-EINVAL); 1663229Spst struct xfrm_offload *xo = xfrm_offload(skb); 1673229Spst 1683229Spst skb->transport_header += x->props.header_len; 1693229Spst ops = rcu_dereference(inet6_offloads[xo->proto]); 1703229Spst if (likely(ops && ops->callbacks.gso_segment)) 1713229Spst segs = ops->callbacks.gso_segment(skb, features); 1723229Spst 1733229Spst return segs; 1743229Spst} 1753229Spst 1763229Spststatic struct sk_buff *xfrm6_beet_gso_segment(struct xfrm_state *x, 1773229Spst struct sk_buff *skb, 1783229Spst netdev_features_t features) 1793229Spst{ 1803229Spst struct xfrm_offload *xo = xfrm_offload(skb); 1813229Spst struct sk_buff *segs = ERR_PTR(-EINVAL); 1823229Spst const struct net_offload *ops; 1833229Spst u8 proto = xo->proto; 1843229Spst 1853229Spst skb->transport_header += x->props.header_len; 1863229Spst 1873229Spst if (x->sel.family != AF_INET6) { 1883229Spst skb->transport_header -= 1893229Spst (sizeof(struct ipv6hdr) - sizeof(struct iphdr)); 1903229Spst 1913229Spst if (proto == IPPROTO_BEETPH) { 1923229Spst struct ip_beet_phdr *ph = 1933229Spst (struct ip_beet_phdr *)skb->data; 1943229Spst 1953229Spst skb->transport_header += ph->hdrlen * 8; 1963229Spst proto = ph->nexthdr; 1973229Spst } else { 1983229Spst skb->transport_header -= IPV4_BEET_PHMAXLEN; 1993229Spst } 2003229Spst 2013229Spst if (proto == IPPROTO_TCP) 2023229Spst skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV6; 2033229Spst } else { 2043229Spst __be16 frag; 2053229Spst 2063229Spst skb->transport_header += 2073229Spst ipv6_skip_exthdr(skb, 0, &proto, &frag); 2083229Spst } 2093229Spst 2103229Spst if (proto == IPPROTO_IPIP) 2113229Spst skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6; 2123229Spst 2133229Spst __skb_pull(skb, skb_transport_offset(skb)); 2143229Spst ops = rcu_dereference(inet6_offloads[proto]); 2153229Spst if (likely(ops && ops->callbacks.gso_segment)) 2163229Spst segs = ops->callbacks.gso_segment(skb, features); 2173229Spst 2183229Spst return segs; 2193229Spst} 2203229Spst 2213229Spststatic struct sk_buff *xfrm6_outer_mode_gso_segment(struct xfrm_state *x, 2223229Spst struct sk_buff *skb, 2233229Spst netdev_features_t features) 2243229Spst{ 2253229Spst switch (x->outer_mode.encap) { 2263229Spst case XFRM_MODE_TUNNEL: 2273229Spst return xfrm6_tunnel_gso_segment(x, skb, features); 2283229Spst case XFRM_MODE_TRANSPORT: 2293229Spst return xfrm6_transport_gso_segment(x, skb, features); 2303229Spst case XFRM_MODE_BEET: 2313229Spst return xfrm6_beet_gso_segment(x, skb, features); 2323229Spst } 2333229Spst 2343229Spst return ERR_PTR(-EOPNOTSUPP); 2353229Spst} 2363229Spst 2373229Spststatic struct sk_buff *esp6_gso_segment(struct sk_buff *skb, 2383229Spst netdev_features_t features) 2393229Spst{ 2403229Spst struct xfrm_state *x; 2413229Spst struct ip_esp_hdr *esph; 2423229Spst struct crypto_aead *aead; 2433229Spst netdev_features_t esp_features = features; 2443229Spst struct xfrm_offload *xo = xfrm_offload(skb); 2453229Spst struct sec_path *sp; 2463229Spst 2473229Spst if (!xo) 2483229Spst return ERR_PTR(-EINVAL); 2493229Spst 2503229Spst if (!(skb_shinfo(skb)->gso_type & SKB_GSO_ESP)) 2513229Spst return ERR_PTR(-EINVAL); 2523229Spst 2533229Spst sp = skb_sec_path(skb); 2543229Spst x = sp->xvec[sp->len - 1]; 2553229Spst aead = x->data; 2563229Spst esph = ip_esp_hdr(skb); 2573229Spst 2583229Spst if (esph->spi != x->id.spi) 2593229Spst return ERR_PTR(-EINVAL); 2603229Spst 2613229Spst if (!pskb_may_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead))) 26246078Simp return ERR_PTR(-EINVAL); 2633229Spst 2643229Spst __skb_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead)); 2653229Spst 2663229Spst skb->encap_hdr_csum = 1; 2673229Spst 2683229Spst if (!(features & NETIF_F_HW_ESP) || x->xso.dev != skb->dev) 2693229Spst esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK | 2703229Spst NETIF_F_SCTP_CRC); 2713229Spst else if (!(features & NETIF_F_HW_ESP_TX_CSUM)) 2723229Spst esp_features = features & ~(NETIF_F_CSUM_MASK | 2733229Spst NETIF_F_SCTP_CRC); 2743229Spst 2753229Spst xo->flags |= XFRM_GSO_SEGMENT; 2763229Spst 2773229Spst return xfrm6_outer_mode_gso_segment(x, skb, esp_features); 2783229Spst} 2793229Spst 2803229Spststatic int esp6_input_tail(struct xfrm_state *x, struct sk_buff *skb) 2813229Spst{ 2823229Spst struct crypto_aead *aead = x->data; 2833229Spst struct xfrm_offload *xo = xfrm_offload(skb); 2843229Spst 2853229Spst if (!pskb_may_pull(skb, sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead))) 2863229Spst return -EINVAL; 2873229Spst 2883229Spst if (!(xo->flags & CRYPTO_DONE)) 2893229Spst skb->ip_summed = CHECKSUM_NONE; 2903229Spst 2913229Spst return esp6_input_done2(skb, 0); 2923229Spst} 2933229Spst 2943229Spststatic int esp6_xmit(struct xfrm_state *x, struct sk_buff *skb, netdev_features_t features) 2953229Spst{ 2963229Spst int len; 2973229Spst int err; 2983229Spst int alen; 2993229Spst int blksize; 3003229Spst struct xfrm_offload *xo; 3013229Spst struct crypto_aead *aead; 3023229Spst struct esp_info esp; 3033229Spst bool hw_offload = true; 3043229Spst __u32 seq; 3053229Spst 3063229Spst esp.inplace = true; 3073229Spst 3083229Spst xo = xfrm_offload(skb); 3093229Spst 3103229Spst if (!xo) 3113229Spst return -EINVAL; 3123229Spst 3133229Spst if (!(features & NETIF_F_HW_ESP) || x->xso.dev != skb->dev) { 3143229Spst xo->flags |= CRYPTO_FALLBACK; 3153229Spst hw_offload = false; 3163229Spst } 3173229Spst 3183229Spst esp.proto = xo->proto; 31913572Spst 3203229Spst /* skb is pure payload to encrypt */ 3213229Spst 3223229Spst aead = x->data; 3233229Spst alen = crypto_aead_authsize(aead); 3243229Spst 3253229Spst esp.tfclen = 0; 3263229Spst /* XXX: Add support for tfc padding here. */ 3273229Spst 3283229Spst blksize = ALIGN(crypto_aead_blocksize(aead), 4); 3293229Spst esp.clen = ALIGN(skb->len + 2 + esp.tfclen, blksize); 3303229Spst esp.plen = esp.clen - skb->len - esp.tfclen; 3313229Spst esp.tailen = esp.tfclen + esp.plen + alen; 332 333 if (!hw_offload || !skb_is_gso(skb)) { 334 esp.nfrags = esp6_output_head(x, skb, &esp); 335 if (esp.nfrags < 0) 336 return esp.nfrags; 337 } 338 339 seq = xo->seq.low; 340 341 esp.esph = ip_esp_hdr(skb); 342 esp.esph->spi = x->id.spi; 343 344 skb_push(skb, -skb_network_offset(skb)); 345 346 if (xo->flags & XFRM_GSO_SEGMENT) { 347 esp.esph->seq_no = htonl(seq); 348 349 if (!skb_is_gso(skb)) 350 xo->seq.low++; 351 else 352 xo->seq.low += skb_shinfo(skb)->gso_segs; 353 } 354 355 if (xo->seq.low < seq) 356 xo->seq.hi++; 357 358 esp.seqno = cpu_to_be64(xo->seq.low + ((u64)xo->seq.hi << 32)); 359 360 len = skb->len - sizeof(struct ipv6hdr); 361 if (len > IPV6_MAXPLEN) 362 len = 0; 363 364 ipv6_hdr(skb)->payload_len = htons(len); 365 366 if (hw_offload) { 367 if (!skb_ext_add(skb, SKB_EXT_SEC_PATH)) 368 return -ENOMEM; 369 370 xo = xfrm_offload(skb); 371 if (!xo) 372 return -EINVAL; 373 374 xo->flags |= XFRM_XMIT; 375 return 0; 376 } 377 378 err = esp6_output_tail(x, skb, &esp); 379 if (err) 380 return err; 381 382 secpath_reset(skb); 383 384 if (skb_needs_linearize(skb, skb->dev->features) && 385 __skb_linearize(skb)) 386 return -ENOMEM; 387 return 0; 388} 389 390static const struct net_offload esp6_offload = { 391 .callbacks = { 392 .gro_receive = esp6_gro_receive, 393 .gso_segment = esp6_gso_segment, 394 }, 395}; 396 397static const struct xfrm_type_offload esp6_type_offload = { 398 .owner = THIS_MODULE, 399 .proto = IPPROTO_ESP, 400 .input_tail = esp6_input_tail, 401 .xmit = esp6_xmit, 402 .encap = esp6_gso_encap, 403}; 404 405static int __init esp6_offload_init(void) 406{ 407 if (xfrm_register_type_offload(&esp6_type_offload, AF_INET6) < 0) { 408 pr_info("%s: can't add xfrm type offload\n", __func__); 409 return -EAGAIN; 410 } 411 412 return inet6_add_offload(&esp6_offload, IPPROTO_ESP); 413} 414 415static void __exit esp6_offload_exit(void) 416{ 417 xfrm_unregister_type_offload(&esp6_type_offload, AF_INET6); 418 inet6_del_offload(&esp6_offload, IPPROTO_ESP); 419} 420 421module_init(esp6_offload_init); 422module_exit(esp6_offload_exit); 423MODULE_LICENSE("GPL"); 424MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>"); 425MODULE_ALIAS_XFRM_OFFLOAD_TYPE(AF_INET6, XFRM_PROTO_ESP); 426MODULE_DESCRIPTION("IPV6 GSO/GRO offload support"); 427