1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Routines having to do with the 'struct sk_buff' memory handlers. 4 * 5 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk> 6 * Florian La Roche <rzsfl@rz.uni-sb.de> 7 * 8 * Fixes: 9 * Alan Cox : Fixed the worst of the load 10 * balancer bugs. 11 * Dave Platt : Interrupt stacking fix. 12 * Richard Kooijman : Timestamp fixes. 13 * Alan Cox : Changed buffer format. 14 * Alan Cox : destructor hook for AF_UNIX etc. 15 * Linus Torvalds : Better skb_clone. 16 * Alan Cox : Added skb_copy. 17 * Alan Cox : Added all the changed routines Linus 18 * only put in the headers 19 * Ray VanTassle : Fixed --skb->lock in free 20 * Alan Cox : skb_copy copy arp field 21 * Andi Kleen : slabified it. 22 * Robert Olsson : Removed skb_head_pool 23 * 24 * NOTE: 25 * The __skb_ routines should be called with interrupts 26 * disabled, or you better be *real* sure that the operation is atomic 27 * with respect to whatever list is being frobbed (e.g. via lock_sock() 28 * or via disabling bottom half handlers, etc). 29 */ 30 31/* 32 * The functions in this file will not compile correctly with gcc 2.4.x 33 */ 34 35#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 36 37#include <linux/module.h> 38#include <linux/types.h> 39#include <linux/kernel.h> 40#include <linux/mm.h> 41#include <linux/interrupt.h> 42#include <linux/in.h> 43#include <linux/inet.h> 44#include <linux/slab.h> 45#include <linux/tcp.h> 46#include <linux/udp.h> 47#include <linux/sctp.h> 48#include <linux/netdevice.h> 49#ifdef CONFIG_NET_CLS_ACT 50#include <net/pkt_sched.h> 51#endif 52#include <linux/string.h> 53#include <linux/skbuff.h> 54#include <linux/splice.h> 55#include <linux/cache.h> 56#include <linux/rtnetlink.h> 57#include <linux/init.h> 58#include <linux/scatterlist.h> 59#include <linux/errqueue.h> 60#include <linux/prefetch.h> 61#include <linux/bitfield.h> 62#include <linux/if_vlan.h> 63#include <linux/mpls.h> 64#include <linux/kcov.h> 65#include <linux/iov_iter.h> 66 67#include <net/protocol.h> 68#include <net/dst.h> 69#include <net/sock.h> 70#include <net/checksum.h> 71#include <net/gso.h> 72#include <net/hotdata.h> 73#include <net/ip6_checksum.h> 74#include <net/xfrm.h> 75#include <net/mpls.h> 76#include <net/mptcp.h> 77#include <net/mctp.h> 78#include <net/page_pool/helpers.h> 79#include <net/dropreason.h> 80 81#include <linux/uaccess.h> 82#include <trace/events/skb.h> 83#include <linux/highmem.h> 84#include <linux/capability.h> 85#include <linux/user_namespace.h> 86#include <linux/indirect_call_wrapper.h> 87#include <linux/textsearch.h> 88 89#include "dev.h" 90#include "sock_destructor.h" 91 92#ifdef CONFIG_SKB_EXTENSIONS 93static struct kmem_cache *skbuff_ext_cache __ro_after_init; 94#endif 95 96#define SKB_SMALL_HEAD_SIZE SKB_HEAD_ALIGN(MAX_TCP_HEADER) 97 98/* We want SKB_SMALL_HEAD_CACHE_SIZE to not be a power of two. 99 * This should ensure that SKB_SMALL_HEAD_HEADROOM is a unique 100 * size, and we can differentiate heads from skb_small_head_cache 101 * vs system slabs by looking at their size (skb_end_offset()). 102 */ 103#define SKB_SMALL_HEAD_CACHE_SIZE \ 104 (is_power_of_2(SKB_SMALL_HEAD_SIZE) ? \ 105 (SKB_SMALL_HEAD_SIZE + L1_CACHE_BYTES) : \ 106 SKB_SMALL_HEAD_SIZE) 107 108#define SKB_SMALL_HEAD_HEADROOM \ 109 SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE) 110 111int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS; 112EXPORT_SYMBOL(sysctl_max_skb_frags); 113 114/* kcm_write_msgs() relies on casting paged frags to bio_vec to use 115 * iov_iter_bvec(). These static asserts ensure the cast is valid is long as the 116 * netmem is a page. 117 */ 118static_assert(offsetof(struct bio_vec, bv_page) == 119 offsetof(skb_frag_t, netmem)); 120static_assert(sizeof_field(struct bio_vec, bv_page) == 121 sizeof_field(skb_frag_t, netmem)); 122 123static_assert(offsetof(struct bio_vec, bv_len) == offsetof(skb_frag_t, len)); 124static_assert(sizeof_field(struct bio_vec, bv_len) == 125 sizeof_field(skb_frag_t, len)); 126 127static_assert(offsetof(struct bio_vec, bv_offset) == 128 offsetof(skb_frag_t, offset)); 129static_assert(sizeof_field(struct bio_vec, bv_offset) == 130 sizeof_field(skb_frag_t, offset)); 131 132#undef FN 133#define FN(reason) [SKB_DROP_REASON_##reason] = #reason, 134static const char * const drop_reasons[] = { 135 [SKB_CONSUMED] = "CONSUMED", 136 DEFINE_DROP_REASON(FN, FN) 137}; 138 139static const struct drop_reason_list drop_reasons_core = { 140 .reasons = drop_reasons, 141 .n_reasons = ARRAY_SIZE(drop_reasons), 142}; 143 144const struct drop_reason_list __rcu * 145drop_reasons_by_subsys[SKB_DROP_REASON_SUBSYS_NUM] = { 146 [SKB_DROP_REASON_SUBSYS_CORE] = RCU_INITIALIZER(&drop_reasons_core), 147}; 148EXPORT_SYMBOL(drop_reasons_by_subsys); 149 150/** 151 * drop_reasons_register_subsys - register another drop reason subsystem 152 * @subsys: the subsystem to register, must not be the core 153 * @list: the list of drop reasons within the subsystem, must point to 154 * a statically initialized list 155 */ 156void drop_reasons_register_subsys(enum skb_drop_reason_subsys subsys, 157 const struct drop_reason_list *list) 158{ 159 if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE || 160 subsys >= ARRAY_SIZE(drop_reasons_by_subsys), 161 "invalid subsystem %d\n", subsys)) 162 return; 163 164 /* must point to statically allocated memory, so INIT is OK */ 165 RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], list); 166} 167EXPORT_SYMBOL_GPL(drop_reasons_register_subsys); 168 169/** 170 * drop_reasons_unregister_subsys - unregister a drop reason subsystem 171 * @subsys: the subsystem to remove, must not be the core 172 * 173 * Note: This will synchronize_rcu() to ensure no users when it returns. 174 */ 175void drop_reasons_unregister_subsys(enum skb_drop_reason_subsys subsys) 176{ 177 if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE || 178 subsys >= ARRAY_SIZE(drop_reasons_by_subsys), 179 "invalid subsystem %d\n", subsys)) 180 return; 181 182 RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], NULL); 183 184 synchronize_rcu(); 185} 186EXPORT_SYMBOL_GPL(drop_reasons_unregister_subsys); 187 188/** 189 * skb_panic - private function for out-of-line support 190 * @skb: buffer 191 * @sz: size 192 * @addr: address 193 * @msg: skb_over_panic or skb_under_panic 194 * 195 * Out-of-line support for skb_put() and skb_push(). 196 * Called via the wrapper skb_over_panic() or skb_under_panic(). 197 * Keep out of line to prevent kernel bloat. 198 * __builtin_return_address is not used because it is not always reliable. 199 */ 200static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr, 201 const char msg[]) 202{ 203 pr_emerg("%s: text:%px len:%d put:%d head:%px data:%px tail:%#lx end:%#lx dev:%s\n", 204 msg, addr, skb->len, sz, skb->head, skb->data, 205 (unsigned long)skb->tail, (unsigned long)skb->end, 206 skb->dev ? skb->dev->name : "<NULL>"); 207 BUG(); 208} 209 210static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr) 211{ 212 skb_panic(skb, sz, addr, __func__); 213} 214 215static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr) 216{ 217 skb_panic(skb, sz, addr, __func__); 218} 219 220#define NAPI_SKB_CACHE_SIZE 64 221#define NAPI_SKB_CACHE_BULK 16 222#define NAPI_SKB_CACHE_HALF (NAPI_SKB_CACHE_SIZE / 2) 223 224#if PAGE_SIZE == SZ_4K 225 226#define NAPI_HAS_SMALL_PAGE_FRAG 1 227#define NAPI_SMALL_PAGE_PFMEMALLOC(nc) ((nc).pfmemalloc) 228 229/* specialized page frag allocator using a single order 0 page 230 * and slicing it into 1K sized fragment. Constrained to systems 231 * with a very limited amount of 1K fragments fitting a single 232 * page - to avoid excessive truesize underestimation 233 */ 234 235struct page_frag_1k { 236 void *va; 237 u16 offset; 238 bool pfmemalloc; 239}; 240 241static void *page_frag_alloc_1k(struct page_frag_1k *nc, gfp_t gfp) 242{ 243 struct page *page; 244 int offset; 245 246 offset = nc->offset - SZ_1K; 247 if (likely(offset >= 0)) 248 goto use_frag; 249 250 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 251 if (!page) 252 return NULL; 253 254 nc->va = page_address(page); 255 nc->pfmemalloc = page_is_pfmemalloc(page); 256 offset = PAGE_SIZE - SZ_1K; 257 page_ref_add(page, offset / SZ_1K); 258 259use_frag: 260 nc->offset = offset; 261 return nc->va + offset; 262} 263#else 264 265/* the small page is actually unused in this build; add dummy helpers 266 * to please the compiler and avoid later preprocessor's conditionals 267 */ 268#define NAPI_HAS_SMALL_PAGE_FRAG 0 269#define NAPI_SMALL_PAGE_PFMEMALLOC(nc) false 270 271struct page_frag_1k { 272}; 273 274static void *page_frag_alloc_1k(struct page_frag_1k *nc, gfp_t gfp_mask) 275{ 276 return NULL; 277} 278 279#endif 280 281struct napi_alloc_cache { 282 struct page_frag_cache page; 283 struct page_frag_1k page_small; 284 unsigned int skb_count; 285 void *skb_cache[NAPI_SKB_CACHE_SIZE]; 286}; 287 288static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache); 289static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache); 290 291/* Double check that napi_get_frags() allocates skbs with 292 * skb->head being backed by slab, not a page fragment. 293 * This is to make sure bug fixed in 3226b158e67c 294 * ("net: avoid 32 x truesize under-estimation for tiny skbs") 295 * does not accidentally come back. 296 */ 297void napi_get_frags_check(struct napi_struct *napi) 298{ 299 struct sk_buff *skb; 300 301 local_bh_disable(); 302 skb = napi_get_frags(napi); 303 WARN_ON_ONCE(!NAPI_HAS_SMALL_PAGE_FRAG && skb && skb->head_frag); 304 napi_free_frags(napi); 305 local_bh_enable(); 306} 307 308void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) 309{ 310 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 311 312 fragsz = SKB_DATA_ALIGN(fragsz); 313 314 return __page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, 315 align_mask); 316} 317EXPORT_SYMBOL(__napi_alloc_frag_align); 318 319void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) 320{ 321 void *data; 322 323 fragsz = SKB_DATA_ALIGN(fragsz); 324 if (in_hardirq() || irqs_disabled()) { 325 struct page_frag_cache *nc = this_cpu_ptr(&netdev_alloc_cache); 326 327 data = __page_frag_alloc_align(nc, fragsz, GFP_ATOMIC, 328 align_mask); 329 } else { 330 struct napi_alloc_cache *nc; 331 332 local_bh_disable(); 333 nc = this_cpu_ptr(&napi_alloc_cache); 334 data = __page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, 335 align_mask); 336 local_bh_enable(); 337 } 338 return data; 339} 340EXPORT_SYMBOL(__netdev_alloc_frag_align); 341 342static struct sk_buff *napi_skb_cache_get(void) 343{ 344 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 345 struct sk_buff *skb; 346 347 if (unlikely(!nc->skb_count)) { 348 nc->skb_count = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, 349 GFP_ATOMIC, 350 NAPI_SKB_CACHE_BULK, 351 nc->skb_cache); 352 if (unlikely(!nc->skb_count)) 353 return NULL; 354 } 355 356 skb = nc->skb_cache[--nc->skb_count]; 357 kasan_mempool_unpoison_object(skb, kmem_cache_size(net_hotdata.skbuff_cache)); 358 359 return skb; 360} 361 362static inline void __finalize_skb_around(struct sk_buff *skb, void *data, 363 unsigned int size) 364{ 365 struct skb_shared_info *shinfo; 366 367 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 368 369 /* Assumes caller memset cleared SKB */ 370 skb->truesize = SKB_TRUESIZE(size); 371 refcount_set(&skb->users, 1); 372 skb->head = data; 373 skb->data = data; 374 skb_reset_tail_pointer(skb); 375 skb_set_end_offset(skb, size); 376 skb->mac_header = (typeof(skb->mac_header))~0U; 377 skb->transport_header = (typeof(skb->transport_header))~0U; 378 skb->alloc_cpu = raw_smp_processor_id(); 379 /* make sure we initialize shinfo sequentially */ 380 shinfo = skb_shinfo(skb); 381 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 382 atomic_set(&shinfo->dataref, 1); 383 384 skb_set_kcov_handle(skb, kcov_common_handle()); 385} 386 387static inline void *__slab_build_skb(struct sk_buff *skb, void *data, 388 unsigned int *size) 389{ 390 void *resized; 391 392 /* Must find the allocation size (and grow it to match). */ 393 *size = ksize(data); 394 /* krealloc() will immediately return "data" when 395 * "ksize(data)" is requested: it is the existing upper 396 * bounds. As a result, GFP_ATOMIC will be ignored. Note 397 * that this "new" pointer needs to be passed back to the 398 * caller for use so the __alloc_size hinting will be 399 * tracked correctly. 400 */ 401 resized = krealloc(data, *size, GFP_ATOMIC); 402 WARN_ON_ONCE(resized != data); 403 return resized; 404} 405 406/* build_skb() variant which can operate on slab buffers. 407 * Note that this should be used sparingly as slab buffers 408 * cannot be combined efficiently by GRO! 409 */ 410struct sk_buff *slab_build_skb(void *data) 411{ 412 struct sk_buff *skb; 413 unsigned int size; 414 415 skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC); 416 if (unlikely(!skb)) 417 return NULL; 418 419 memset(skb, 0, offsetof(struct sk_buff, tail)); 420 data = __slab_build_skb(skb, data, &size); 421 __finalize_skb_around(skb, data, size); 422 423 return skb; 424} 425EXPORT_SYMBOL(slab_build_skb); 426 427/* Caller must provide SKB that is memset cleared */ 428static void __build_skb_around(struct sk_buff *skb, void *data, 429 unsigned int frag_size) 430{ 431 unsigned int size = frag_size; 432 433 /* frag_size == 0 is considered deprecated now. Callers 434 * using slab buffer should use slab_build_skb() instead. 435 */ 436 if (WARN_ONCE(size == 0, "Use slab_build_skb() instead")) 437 data = __slab_build_skb(skb, data, &size); 438 439 __finalize_skb_around(skb, data, size); 440} 441 442/** 443 * __build_skb - build a network buffer 444 * @data: data buffer provided by caller 445 * @frag_size: size of data (must not be 0) 446 * 447 * Allocate a new &sk_buff. Caller provides space holding head and 448 * skb_shared_info. @data must have been allocated from the page 449 * allocator or vmalloc(). (A @frag_size of 0 to indicate a kmalloc() 450 * allocation is deprecated, and callers should use slab_build_skb() 451 * instead.) 452 * The return is the new skb buffer. 453 * On a failure the return is %NULL, and @data is not freed. 454 * Notes : 455 * Before IO, driver allocates only data buffer where NIC put incoming frame 456 * Driver should add room at head (NET_SKB_PAD) and 457 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info)) 458 * After IO, driver calls build_skb(), to allocate sk_buff and populate it 459 * before giving packet to stack. 460 * RX rings only contains data buffers, not full skbs. 461 */ 462struct sk_buff *__build_skb(void *data, unsigned int frag_size) 463{ 464 struct sk_buff *skb; 465 466 skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC); 467 if (unlikely(!skb)) 468 return NULL; 469 470 memset(skb, 0, offsetof(struct sk_buff, tail)); 471 __build_skb_around(skb, data, frag_size); 472 473 return skb; 474} 475 476/* build_skb() is wrapper over __build_skb(), that specifically 477 * takes care of skb->head and skb->pfmemalloc 478 */ 479struct sk_buff *build_skb(void *data, unsigned int frag_size) 480{ 481 struct sk_buff *skb = __build_skb(data, frag_size); 482 483 if (likely(skb && frag_size)) { 484 skb->head_frag = 1; 485 skb_propagate_pfmemalloc(virt_to_head_page(data), skb); 486 } 487 return skb; 488} 489EXPORT_SYMBOL(build_skb); 490 491/** 492 * build_skb_around - build a network buffer around provided skb 493 * @skb: sk_buff provide by caller, must be memset cleared 494 * @data: data buffer provided by caller 495 * @frag_size: size of data 496 */ 497struct sk_buff *build_skb_around(struct sk_buff *skb, 498 void *data, unsigned int frag_size) 499{ 500 if (unlikely(!skb)) 501 return NULL; 502 503 __build_skb_around(skb, data, frag_size); 504 505 if (frag_size) { 506 skb->head_frag = 1; 507 skb_propagate_pfmemalloc(virt_to_head_page(data), skb); 508 } 509 return skb; 510} 511EXPORT_SYMBOL(build_skb_around); 512 513/** 514 * __napi_build_skb - build a network buffer 515 * @data: data buffer provided by caller 516 * @frag_size: size of data 517 * 518 * Version of __build_skb() that uses NAPI percpu caches to obtain 519 * skbuff_head instead of inplace allocation. 520 * 521 * Returns a new &sk_buff on success, %NULL on allocation failure. 522 */ 523static struct sk_buff *__napi_build_skb(void *data, unsigned int frag_size) 524{ 525 struct sk_buff *skb; 526 527 skb = napi_skb_cache_get(); 528 if (unlikely(!skb)) 529 return NULL; 530 531 memset(skb, 0, offsetof(struct sk_buff, tail)); 532 __build_skb_around(skb, data, frag_size); 533 534 return skb; 535} 536 537/** 538 * napi_build_skb - build a network buffer 539 * @data: data buffer provided by caller 540 * @frag_size: size of data 541 * 542 * Version of __napi_build_skb() that takes care of skb->head_frag 543 * and skb->pfmemalloc when the data is a page or page fragment. 544 * 545 * Returns a new &sk_buff on success, %NULL on allocation failure. 546 */ 547struct sk_buff *napi_build_skb(void *data, unsigned int frag_size) 548{ 549 struct sk_buff *skb = __napi_build_skb(data, frag_size); 550 551 if (likely(skb) && frag_size) { 552 skb->head_frag = 1; 553 skb_propagate_pfmemalloc(virt_to_head_page(data), skb); 554 } 555 556 return skb; 557} 558EXPORT_SYMBOL(napi_build_skb); 559 560/* 561 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells 562 * the caller if emergency pfmemalloc reserves are being used. If it is and 563 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves 564 * may be used. Otherwise, the packet data may be discarded until enough 565 * memory is free 566 */ 567static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node, 568 bool *pfmemalloc) 569{ 570 bool ret_pfmemalloc = false; 571 size_t obj_size; 572 void *obj; 573 574 obj_size = SKB_HEAD_ALIGN(*size); 575 if (obj_size <= SKB_SMALL_HEAD_CACHE_SIZE && 576 !(flags & KMALLOC_NOT_NORMAL_BITS)) { 577 obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache, 578 flags | __GFP_NOMEMALLOC | __GFP_NOWARN, 579 node); 580 *size = SKB_SMALL_HEAD_CACHE_SIZE; 581 if (obj || !(gfp_pfmemalloc_allowed(flags))) 582 goto out; 583 /* Try again but now we are using pfmemalloc reserves */ 584 ret_pfmemalloc = true; 585 obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache, flags, node); 586 goto out; 587 } 588 589 obj_size = kmalloc_size_roundup(obj_size); 590 /* The following cast might truncate high-order bits of obj_size, this 591 * is harmless because kmalloc(obj_size >= 2^32) will fail anyway. 592 */ 593 *size = (unsigned int)obj_size; 594 595 /* 596 * Try a regular allocation, when that fails and we're not entitled 597 * to the reserves, fail. 598 */ 599 obj = kmalloc_node_track_caller(obj_size, 600 flags | __GFP_NOMEMALLOC | __GFP_NOWARN, 601 node); 602 if (obj || !(gfp_pfmemalloc_allowed(flags))) 603 goto out; 604 605 /* Try again but now we are using pfmemalloc reserves */ 606 ret_pfmemalloc = true; 607 obj = kmalloc_node_track_caller(obj_size, flags, node); 608 609out: 610 if (pfmemalloc) 611 *pfmemalloc = ret_pfmemalloc; 612 613 return obj; 614} 615 616/* Allocate a new skbuff. We do this ourselves so we can fill in a few 617 * 'private' fields and also do memory statistics to find all the 618 * [BEEP] leaks. 619 * 620 */ 621 622/** 623 * __alloc_skb - allocate a network buffer 624 * @size: size to allocate 625 * @gfp_mask: allocation mask 626 * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache 627 * instead of head cache and allocate a cloned (child) skb. 628 * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for 629 * allocations in case the data is required for writeback 630 * @node: numa node to allocate memory on 631 * 632 * Allocate a new &sk_buff. The returned buffer has no headroom and a 633 * tail room of at least size bytes. The object has a reference count 634 * of one. The return is the buffer. On a failure the return is %NULL. 635 * 636 * Buffers may only be allocated from interrupts using a @gfp_mask of 637 * %GFP_ATOMIC. 638 */ 639struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, 640 int flags, int node) 641{ 642 struct kmem_cache *cache; 643 struct sk_buff *skb; 644 bool pfmemalloc; 645 u8 *data; 646 647 cache = (flags & SKB_ALLOC_FCLONE) 648 ? net_hotdata.skbuff_fclone_cache : net_hotdata.skbuff_cache; 649 650 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX)) 651 gfp_mask |= __GFP_MEMALLOC; 652 653 /* Get the HEAD */ 654 if ((flags & (SKB_ALLOC_FCLONE | SKB_ALLOC_NAPI)) == SKB_ALLOC_NAPI && 655 likely(node == NUMA_NO_NODE || node == numa_mem_id())) 656 skb = napi_skb_cache_get(); 657 else 658 skb = kmem_cache_alloc_node(cache, gfp_mask & ~GFP_DMA, node); 659 if (unlikely(!skb)) 660 return NULL; 661 prefetchw(skb); 662 663 /* We do our best to align skb_shared_info on a separate cache 664 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives 665 * aligned memory blocks, unless SLUB/SLAB debug is enabled. 666 * Both skb->head and skb_shared_info are cache line aligned. 667 */ 668 data = kmalloc_reserve(&size, gfp_mask, node, &pfmemalloc); 669 if (unlikely(!data)) 670 goto nodata; 671 /* kmalloc_size_roundup() might give us more room than requested. 672 * Put skb_shared_info exactly at the end of allocated zone, 673 * to allow max possible filling before reallocation. 674 */ 675 prefetchw(data + SKB_WITH_OVERHEAD(size)); 676 677 /* 678 * Only clear those fields we need to clear, not those that we will 679 * actually initialise below. Hence, don't put any more fields after 680 * the tail pointer in struct sk_buff! 681 */ 682 memset(skb, 0, offsetof(struct sk_buff, tail)); 683 __build_skb_around(skb, data, size); 684 skb->pfmemalloc = pfmemalloc; 685 686 if (flags & SKB_ALLOC_FCLONE) { 687 struct sk_buff_fclones *fclones; 688 689 fclones = container_of(skb, struct sk_buff_fclones, skb1); 690 691 skb->fclone = SKB_FCLONE_ORIG; 692 refcount_set(&fclones->fclone_ref, 1); 693 } 694 695 return skb; 696 697nodata: 698 kmem_cache_free(cache, skb); 699 return NULL; 700} 701EXPORT_SYMBOL(__alloc_skb); 702 703/** 704 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device 705 * @dev: network device to receive on 706 * @len: length to allocate 707 * @gfp_mask: get_free_pages mask, passed to alloc_skb 708 * 709 * Allocate a new &sk_buff and assign it a usage count of one. The 710 * buffer has NET_SKB_PAD headroom built in. Users should allocate 711 * the headroom they think they need without accounting for the 712 * built in space. The built in space is used for optimisations. 713 * 714 * %NULL is returned if there is no free memory. 715 */ 716struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len, 717 gfp_t gfp_mask) 718{ 719 struct page_frag_cache *nc; 720 struct sk_buff *skb; 721 bool pfmemalloc; 722 void *data; 723 724 len += NET_SKB_PAD; 725 726 /* If requested length is either too small or too big, 727 * we use kmalloc() for skb->head allocation. 728 */ 729 if (len <= SKB_WITH_OVERHEAD(1024) || 730 len > SKB_WITH_OVERHEAD(PAGE_SIZE) || 731 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 732 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE); 733 if (!skb) 734 goto skb_fail; 735 goto skb_success; 736 } 737 738 len = SKB_HEAD_ALIGN(len); 739 740 if (sk_memalloc_socks()) 741 gfp_mask |= __GFP_MEMALLOC; 742 743 if (in_hardirq() || irqs_disabled()) { 744 nc = this_cpu_ptr(&netdev_alloc_cache); 745 data = page_frag_alloc(nc, len, gfp_mask); 746 pfmemalloc = nc->pfmemalloc; 747 } else { 748 local_bh_disable(); 749 nc = this_cpu_ptr(&napi_alloc_cache.page); 750 data = page_frag_alloc(nc, len, gfp_mask); 751 pfmemalloc = nc->pfmemalloc; 752 local_bh_enable(); 753 } 754 755 if (unlikely(!data)) 756 return NULL; 757 758 skb = __build_skb(data, len); 759 if (unlikely(!skb)) { 760 skb_free_frag(data); 761 return NULL; 762 } 763 764 if (pfmemalloc) 765 skb->pfmemalloc = 1; 766 skb->head_frag = 1; 767 768skb_success: 769 skb_reserve(skb, NET_SKB_PAD); 770 skb->dev = dev; 771 772skb_fail: 773 return skb; 774} 775EXPORT_SYMBOL(__netdev_alloc_skb); 776 777/** 778 * __napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance 779 * @napi: napi instance this buffer was allocated for 780 * @len: length to allocate 781 * @gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages 782 * 783 * Allocate a new sk_buff for use in NAPI receive. This buffer will 784 * attempt to allocate the head from a special reserved region used 785 * only for NAPI Rx allocation. By doing this we can save several 786 * CPU cycles by avoiding having to disable and re-enable IRQs. 787 * 788 * %NULL is returned if there is no free memory. 789 */ 790struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len, 791 gfp_t gfp_mask) 792{ 793 struct napi_alloc_cache *nc; 794 struct sk_buff *skb; 795 bool pfmemalloc; 796 void *data; 797 798 DEBUG_NET_WARN_ON_ONCE(!in_softirq()); 799 len += NET_SKB_PAD + NET_IP_ALIGN; 800 801 /* If requested length is either too small or too big, 802 * we use kmalloc() for skb->head allocation. 803 * When the small frag allocator is available, prefer it over kmalloc 804 * for small fragments 805 */ 806 if ((!NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) || 807 len > SKB_WITH_OVERHEAD(PAGE_SIZE) || 808 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 809 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX | SKB_ALLOC_NAPI, 810 NUMA_NO_NODE); 811 if (!skb) 812 goto skb_fail; 813 goto skb_success; 814 } 815 816 nc = this_cpu_ptr(&napi_alloc_cache); 817 818 if (sk_memalloc_socks()) 819 gfp_mask |= __GFP_MEMALLOC; 820 821 if (NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) { 822 /* we are artificially inflating the allocation size, but 823 * that is not as bad as it may look like, as: 824 * - 'len' less than GRO_MAX_HEAD makes little sense 825 * - On most systems, larger 'len' values lead to fragment 826 * size above 512 bytes 827 * - kmalloc would use the kmalloc-1k slab for such values 828 * - Builds with smaller GRO_MAX_HEAD will very likely do 829 * little networking, as that implies no WiFi and no 830 * tunnels support, and 32 bits arches. 831 */ 832 len = SZ_1K; 833 834 data = page_frag_alloc_1k(&nc->page_small, gfp_mask); 835 pfmemalloc = NAPI_SMALL_PAGE_PFMEMALLOC(nc->page_small); 836 } else { 837 len = SKB_HEAD_ALIGN(len); 838 839 data = page_frag_alloc(&nc->page, len, gfp_mask); 840 pfmemalloc = nc->page.pfmemalloc; 841 } 842 843 if (unlikely(!data)) 844 return NULL; 845 846 skb = __napi_build_skb(data, len); 847 if (unlikely(!skb)) { 848 skb_free_frag(data); 849 return NULL; 850 } 851 852 if (pfmemalloc) 853 skb->pfmemalloc = 1; 854 skb->head_frag = 1; 855 856skb_success: 857 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 858 skb->dev = napi->dev; 859 860skb_fail: 861 return skb; 862} 863EXPORT_SYMBOL(__napi_alloc_skb); 864 865void skb_add_rx_frag_netmem(struct sk_buff *skb, int i, netmem_ref netmem, 866 int off, int size, unsigned int truesize) 867{ 868 DEBUG_NET_WARN_ON_ONCE(size > truesize); 869 870 skb_fill_netmem_desc(skb, i, netmem, off, size); 871 skb->len += size; 872 skb->data_len += size; 873 skb->truesize += truesize; 874} 875EXPORT_SYMBOL(skb_add_rx_frag_netmem); 876 877void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size, 878 unsigned int truesize) 879{ 880 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 881 882 DEBUG_NET_WARN_ON_ONCE(size > truesize); 883 884 skb_frag_size_add(frag, size); 885 skb->len += size; 886 skb->data_len += size; 887 skb->truesize += truesize; 888} 889EXPORT_SYMBOL(skb_coalesce_rx_frag); 890 891static void skb_drop_list(struct sk_buff **listp) 892{ 893 kfree_skb_list(*listp); 894 *listp = NULL; 895} 896 897static inline void skb_drop_fraglist(struct sk_buff *skb) 898{ 899 skb_drop_list(&skb_shinfo(skb)->frag_list); 900} 901 902static void skb_clone_fraglist(struct sk_buff *skb) 903{ 904 struct sk_buff *list; 905 906 skb_walk_frags(skb, list) 907 skb_get(list); 908} 909 910static bool is_pp_page(struct page *page) 911{ 912 return (page->pp_magic & ~0x3UL) == PP_SIGNATURE; 913} 914 915int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, 916 unsigned int headroom) 917{ 918#if IS_ENABLED(CONFIG_PAGE_POOL) 919 u32 size, truesize, len, max_head_size, off; 920 struct sk_buff *skb = *pskb, *nskb; 921 int err, i, head_off; 922 void *data; 923 924 /* XDP does not support fraglist so we need to linearize 925 * the skb. 926 */ 927 if (skb_has_frag_list(skb)) 928 return -EOPNOTSUPP; 929 930 max_head_size = SKB_WITH_OVERHEAD(PAGE_SIZE - headroom); 931 if (skb->len > max_head_size + MAX_SKB_FRAGS * PAGE_SIZE) 932 return -ENOMEM; 933 934 size = min_t(u32, skb->len, max_head_size); 935 truesize = SKB_HEAD_ALIGN(size) + headroom; 936 data = page_pool_dev_alloc_va(pool, &truesize); 937 if (!data) 938 return -ENOMEM; 939 940 nskb = napi_build_skb(data, truesize); 941 if (!nskb) { 942 page_pool_free_va(pool, data, true); 943 return -ENOMEM; 944 } 945 946 skb_reserve(nskb, headroom); 947 skb_copy_header(nskb, skb); 948 skb_mark_for_recycle(nskb); 949 950 err = skb_copy_bits(skb, 0, nskb->data, size); 951 if (err) { 952 consume_skb(nskb); 953 return err; 954 } 955 skb_put(nskb, size); 956 957 head_off = skb_headroom(nskb) - skb_headroom(skb); 958 skb_headers_offset_update(nskb, head_off); 959 960 off = size; 961 len = skb->len - off; 962 for (i = 0; i < MAX_SKB_FRAGS && off < skb->len; i++) { 963 struct page *page; 964 u32 page_off; 965 966 size = min_t(u32, len, PAGE_SIZE); 967 truesize = size; 968 969 page = page_pool_dev_alloc(pool, &page_off, &truesize); 970 if (!page) { 971 consume_skb(nskb); 972 return -ENOMEM; 973 } 974 975 skb_add_rx_frag(nskb, i, page, page_off, size, truesize); 976 err = skb_copy_bits(skb, off, page_address(page) + page_off, 977 size); 978 if (err) { 979 consume_skb(nskb); 980 return err; 981 } 982 983 len -= size; 984 off += size; 985 } 986 987 consume_skb(skb); 988 *pskb = nskb; 989 990 return 0; 991#else 992 return -EOPNOTSUPP; 993#endif 994} 995EXPORT_SYMBOL(skb_pp_cow_data); 996 997int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb, 998 struct bpf_prog *prog) 999{ 1000 if (!prog->aux->xdp_has_frags) 1001 return -EINVAL; 1002 1003 return skb_pp_cow_data(pool, pskb, XDP_PACKET_HEADROOM); 1004} 1005EXPORT_SYMBOL(skb_cow_data_for_xdp); 1006 1007#if IS_ENABLED(CONFIG_PAGE_POOL) 1008bool napi_pp_put_page(struct page *page, bool napi_safe) 1009{ 1010 bool allow_direct = false; 1011 struct page_pool *pp; 1012 1013 page = compound_head(page); 1014 1015 /* page->pp_magic is OR'ed with PP_SIGNATURE after the allocation 1016 * in order to preserve any existing bits, such as bit 0 for the 1017 * head page of compound page and bit 1 for pfmemalloc page, so 1018 * mask those bits for freeing side when doing below checking, 1019 * and page_is_pfmemalloc() is checked in __page_pool_put_page() 1020 * to avoid recycling the pfmemalloc page. 1021 */ 1022 if (unlikely(!is_pp_page(page))) 1023 return false; 1024 1025 pp = page->pp; 1026 1027 /* Allow direct recycle if we have reasons to believe that we are 1028 * in the same context as the consumer would run, so there's 1029 * no possible race. 1030 * __page_pool_put_page() makes sure we're not in hardirq context 1031 * and interrupts are enabled prior to accessing the cache. 1032 */ 1033 if (napi_safe || in_softirq()) { 1034 const struct napi_struct *napi = READ_ONCE(pp->p.napi); 1035 unsigned int cpuid = smp_processor_id(); 1036 1037 allow_direct = napi && READ_ONCE(napi->list_owner) == cpuid; 1038 allow_direct |= READ_ONCE(pp->cpuid) == cpuid; 1039 } 1040 1041 /* Driver set this to memory recycling info. Reset it on recycle. 1042 * This will *not* work for NIC using a split-page memory model. 1043 * The page will be returned to the pool here regardless of the 1044 * 'flipped' fragment being in use or not. 1045 */ 1046 page_pool_put_full_page(pp, page, allow_direct); 1047 1048 return true; 1049} 1050EXPORT_SYMBOL(napi_pp_put_page); 1051#endif 1052 1053static bool skb_pp_recycle(struct sk_buff *skb, void *data, bool napi_safe) 1054{ 1055 if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle) 1056 return false; 1057 return napi_pp_put_page(virt_to_page(data), napi_safe); 1058} 1059 1060/** 1061 * skb_pp_frag_ref() - Increase fragment references of a page pool aware skb 1062 * @skb: page pool aware skb 1063 * 1064 * Increase the fragment reference count (pp_ref_count) of a skb. This is 1065 * intended to gain fragment references only for page pool aware skbs, 1066 * i.e. when skb->pp_recycle is true, and not for fragments in a 1067 * non-pp-recycling skb. It has a fallback to increase references on normal 1068 * pages, as page pool aware skbs may also have normal page fragments. 1069 */ 1070static int skb_pp_frag_ref(struct sk_buff *skb) 1071{ 1072 struct skb_shared_info *shinfo; 1073 struct page *head_page; 1074 int i; 1075 1076 if (!skb->pp_recycle) 1077 return -EINVAL; 1078 1079 shinfo = skb_shinfo(skb); 1080 1081 for (i = 0; i < shinfo->nr_frags; i++) { 1082 head_page = compound_head(skb_frag_page(&shinfo->frags[i])); 1083 if (likely(is_pp_page(head_page))) 1084 page_pool_ref_page(head_page); 1085 else 1086 page_ref_inc(head_page); 1087 } 1088 return 0; 1089} 1090 1091static void skb_kfree_head(void *head, unsigned int end_offset) 1092{ 1093 if (end_offset == SKB_SMALL_HEAD_HEADROOM) 1094 kmem_cache_free(net_hotdata.skb_small_head_cache, head); 1095 else 1096 kfree(head); 1097} 1098 1099static void skb_free_head(struct sk_buff *skb, bool napi_safe) 1100{ 1101 unsigned char *head = skb->head; 1102 1103 if (skb->head_frag) { 1104 if (skb_pp_recycle(skb, head, napi_safe)) 1105 return; 1106 skb_free_frag(head); 1107 } else { 1108 skb_kfree_head(head, skb_end_offset(skb)); 1109 } 1110} 1111 1112static void skb_release_data(struct sk_buff *skb, enum skb_drop_reason reason, 1113 bool napi_safe) 1114{ 1115 struct skb_shared_info *shinfo = skb_shinfo(skb); 1116 int i; 1117 1118 if (!skb_data_unref(skb, shinfo)) 1119 goto exit; 1120 1121 if (skb_zcopy(skb)) { 1122 bool skip_unref = shinfo->flags & SKBFL_MANAGED_FRAG_REFS; 1123 1124 skb_zcopy_clear(skb, true); 1125 if (skip_unref) 1126 goto free_head; 1127 } 1128 1129 for (i = 0; i < shinfo->nr_frags; i++) 1130 napi_frag_unref(&shinfo->frags[i], skb->pp_recycle, napi_safe); 1131 1132free_head: 1133 if (shinfo->frag_list) 1134 kfree_skb_list_reason(shinfo->frag_list, reason); 1135 1136 skb_free_head(skb, napi_safe); 1137exit: 1138 /* When we clone an SKB we copy the reycling bit. The pp_recycle 1139 * bit is only set on the head though, so in order to avoid races 1140 * while trying to recycle fragments on __skb_frag_unref() we need 1141 * to make one SKB responsible for triggering the recycle path. 1142 * So disable the recycling bit if an SKB is cloned and we have 1143 * additional references to the fragmented part of the SKB. 1144 * Eventually the last SKB will have the recycling bit set and it's 1145 * dataref set to 0, which will trigger the recycling 1146 */ 1147 skb->pp_recycle = 0; 1148} 1149 1150/* 1151 * Free an skbuff by memory without cleaning the state. 1152 */ 1153static void kfree_skbmem(struct sk_buff *skb) 1154{ 1155 struct sk_buff_fclones *fclones; 1156 1157 switch (skb->fclone) { 1158 case SKB_FCLONE_UNAVAILABLE: 1159 kmem_cache_free(net_hotdata.skbuff_cache, skb); 1160 return; 1161 1162 case SKB_FCLONE_ORIG: 1163 fclones = container_of(skb, struct sk_buff_fclones, skb1); 1164 1165 /* We usually free the clone (TX completion) before original skb 1166 * This test would have no chance to be true for the clone, 1167 * while here, branch prediction will be good. 1168 */ 1169 if (refcount_read(&fclones->fclone_ref) == 1) 1170 goto fastpath; 1171 break; 1172 1173 default: /* SKB_FCLONE_CLONE */ 1174 fclones = container_of(skb, struct sk_buff_fclones, skb2); 1175 break; 1176 } 1177 if (!refcount_dec_and_test(&fclones->fclone_ref)) 1178 return; 1179fastpath: 1180 kmem_cache_free(net_hotdata.skbuff_fclone_cache, fclones); 1181} 1182 1183void skb_release_head_state(struct sk_buff *skb) 1184{ 1185 skb_dst_drop(skb); 1186 if (skb->destructor) { 1187 DEBUG_NET_WARN_ON_ONCE(in_hardirq()); 1188 skb->destructor(skb); 1189 } 1190#if IS_ENABLED(CONFIG_NF_CONNTRACK) 1191 nf_conntrack_put(skb_nfct(skb)); 1192#endif 1193 skb_ext_put(skb); 1194} 1195 1196/* Free everything but the sk_buff shell. */ 1197static void skb_release_all(struct sk_buff *skb, enum skb_drop_reason reason, 1198 bool napi_safe) 1199{ 1200 skb_release_head_state(skb); 1201 if (likely(skb->head)) 1202 skb_release_data(skb, reason, napi_safe); 1203} 1204 1205/** 1206 * __kfree_skb - private function 1207 * @skb: buffer 1208 * 1209 * Free an sk_buff. Release anything attached to the buffer. 1210 * Clean the state. This is an internal helper function. Users should 1211 * always call kfree_skb 1212 */ 1213 1214void __kfree_skb(struct sk_buff *skb) 1215{ 1216 skb_release_all(skb, SKB_DROP_REASON_NOT_SPECIFIED, false); 1217 kfree_skbmem(skb); 1218} 1219EXPORT_SYMBOL(__kfree_skb); 1220 1221static __always_inline 1222bool __kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason) 1223{ 1224 if (unlikely(!skb_unref(skb))) 1225 return false; 1226 1227 DEBUG_NET_WARN_ON_ONCE(reason == SKB_NOT_DROPPED_YET || 1228 u32_get_bits(reason, 1229 SKB_DROP_REASON_SUBSYS_MASK) >= 1230 SKB_DROP_REASON_SUBSYS_NUM); 1231 1232 if (reason == SKB_CONSUMED) 1233 trace_consume_skb(skb, __builtin_return_address(0)); 1234 else 1235 trace_kfree_skb(skb, __builtin_return_address(0), reason); 1236 return true; 1237} 1238 1239/** 1240 * kfree_skb_reason - free an sk_buff with special reason 1241 * @skb: buffer to free 1242 * @reason: reason why this skb is dropped 1243 * 1244 * Drop a reference to the buffer and free it if the usage count has 1245 * hit zero. Meanwhile, pass the drop reason to 'kfree_skb' 1246 * tracepoint. 1247 */ 1248void __fix_address 1249kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason) 1250{ 1251 if (__kfree_skb_reason(skb, reason)) 1252 __kfree_skb(skb); 1253} 1254EXPORT_SYMBOL(kfree_skb_reason); 1255 1256#define KFREE_SKB_BULK_SIZE 16 1257 1258struct skb_free_array { 1259 unsigned int skb_count; 1260 void *skb_array[KFREE_SKB_BULK_SIZE]; 1261}; 1262 1263static void kfree_skb_add_bulk(struct sk_buff *skb, 1264 struct skb_free_array *sa, 1265 enum skb_drop_reason reason) 1266{ 1267 /* if SKB is a clone, don't handle this case */ 1268 if (unlikely(skb->fclone != SKB_FCLONE_UNAVAILABLE)) { 1269 __kfree_skb(skb); 1270 return; 1271 } 1272 1273 skb_release_all(skb, reason, false); 1274 sa->skb_array[sa->skb_count++] = skb; 1275 1276 if (unlikely(sa->skb_count == KFREE_SKB_BULK_SIZE)) { 1277 kmem_cache_free_bulk(net_hotdata.skbuff_cache, KFREE_SKB_BULK_SIZE, 1278 sa->skb_array); 1279 sa->skb_count = 0; 1280 } 1281} 1282 1283void __fix_address 1284kfree_skb_list_reason(struct sk_buff *segs, enum skb_drop_reason reason) 1285{ 1286 struct skb_free_array sa; 1287 1288 sa.skb_count = 0; 1289 1290 while (segs) { 1291 struct sk_buff *next = segs->next; 1292 1293 if (__kfree_skb_reason(segs, reason)) { 1294 skb_poison_list(segs); 1295 kfree_skb_add_bulk(segs, &sa, reason); 1296 } 1297 1298 segs = next; 1299 } 1300 1301 if (sa.skb_count) 1302 kmem_cache_free_bulk(net_hotdata.skbuff_cache, sa.skb_count, sa.skb_array); 1303} 1304EXPORT_SYMBOL(kfree_skb_list_reason); 1305 1306/* Dump skb information and contents. 1307 * 1308 * Must only be called from net_ratelimit()-ed paths. 1309 * 1310 * Dumps whole packets if full_pkt, only headers otherwise. 1311 */ 1312void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt) 1313{ 1314 struct skb_shared_info *sh = skb_shinfo(skb); 1315 struct net_device *dev = skb->dev; 1316 struct sock *sk = skb->sk; 1317 struct sk_buff *list_skb; 1318 bool has_mac, has_trans; 1319 int headroom, tailroom; 1320 int i, len, seg_len; 1321 1322 if (full_pkt) 1323 len = skb->len; 1324 else 1325 len = min_t(int, skb->len, MAX_HEADER + 128); 1326 1327 headroom = skb_headroom(skb); 1328 tailroom = skb_tailroom(skb); 1329 1330 has_mac = skb_mac_header_was_set(skb); 1331 has_trans = skb_transport_header_was_set(skb); 1332 1333 printk("%sskb len=%u headroom=%u headlen=%u tailroom=%u\n" 1334 "mac=(%d,%d) net=(%d,%d) trans=%d\n" 1335 "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n" 1336 "csum(0x%x ip_summed=%u complete_sw=%u valid=%u level=%u)\n" 1337 "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n", 1338 level, skb->len, headroom, skb_headlen(skb), tailroom, 1339 has_mac ? skb->mac_header : -1, 1340 has_mac ? skb_mac_header_len(skb) : -1, 1341 skb->network_header, 1342 has_trans ? skb_network_header_len(skb) : -1, 1343 has_trans ? skb->transport_header : -1, 1344 sh->tx_flags, sh->nr_frags, 1345 sh->gso_size, sh->gso_type, sh->gso_segs, 1346 skb->csum, skb->ip_summed, skb->csum_complete_sw, 1347 skb->csum_valid, skb->csum_level, 1348 skb->hash, skb->sw_hash, skb->l4_hash, 1349 ntohs(skb->protocol), skb->pkt_type, skb->skb_iif); 1350 1351 if (dev) 1352 printk("%sdev name=%s feat=%pNF\n", 1353 level, dev->name, &dev->features); 1354 if (sk) 1355 printk("%ssk family=%hu type=%u proto=%u\n", 1356 level, sk->sk_family, sk->sk_type, sk->sk_protocol); 1357 1358 if (full_pkt && headroom) 1359 print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET, 1360 16, 1, skb->head, headroom, false); 1361 1362 seg_len = min_t(int, skb_headlen(skb), len); 1363 if (seg_len) 1364 print_hex_dump(level, "skb linear: ", DUMP_PREFIX_OFFSET, 1365 16, 1, skb->data, seg_len, false); 1366 len -= seg_len; 1367 1368 if (full_pkt && tailroom) 1369 print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET, 1370 16, 1, skb_tail_pointer(skb), tailroom, false); 1371 1372 for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) { 1373 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1374 u32 p_off, p_len, copied; 1375 struct page *p; 1376 u8 *vaddr; 1377 1378 skb_frag_foreach_page(frag, skb_frag_off(frag), 1379 skb_frag_size(frag), p, p_off, p_len, 1380 copied) { 1381 seg_len = min_t(int, p_len, len); 1382 vaddr = kmap_atomic(p); 1383 print_hex_dump(level, "skb frag: ", 1384 DUMP_PREFIX_OFFSET, 1385 16, 1, vaddr + p_off, seg_len, false); 1386 kunmap_atomic(vaddr); 1387 len -= seg_len; 1388 if (!len) 1389 break; 1390 } 1391 } 1392 1393 if (full_pkt && skb_has_frag_list(skb)) { 1394 printk("skb fraglist:\n"); 1395 skb_walk_frags(skb, list_skb) 1396 skb_dump(level, list_skb, true); 1397 } 1398} 1399EXPORT_SYMBOL(skb_dump); 1400 1401/** 1402 * skb_tx_error - report an sk_buff xmit error 1403 * @skb: buffer that triggered an error 1404 * 1405 * Report xmit error if a device callback is tracking this skb. 1406 * skb must be freed afterwards. 1407 */ 1408void skb_tx_error(struct sk_buff *skb) 1409{ 1410 if (skb) { 1411 skb_zcopy_downgrade_managed(skb); 1412 skb_zcopy_clear(skb, true); 1413 } 1414} 1415EXPORT_SYMBOL(skb_tx_error); 1416 1417#ifdef CONFIG_TRACEPOINTS 1418/** 1419 * consume_skb - free an skbuff 1420 * @skb: buffer to free 1421 * 1422 * Drop a ref to the buffer and free it if the usage count has hit zero 1423 * Functions identically to kfree_skb, but kfree_skb assumes that the frame 1424 * is being dropped after a failure and notes that 1425 */ 1426void consume_skb(struct sk_buff *skb) 1427{ 1428 if (!skb_unref(skb)) 1429 return; 1430 1431 trace_consume_skb(skb, __builtin_return_address(0)); 1432 __kfree_skb(skb); 1433} 1434EXPORT_SYMBOL(consume_skb); 1435#endif 1436 1437/** 1438 * __consume_stateless_skb - free an skbuff, assuming it is stateless 1439 * @skb: buffer to free 1440 * 1441 * Alike consume_skb(), but this variant assumes that this is the last 1442 * skb reference and all the head states have been already dropped 1443 */ 1444void __consume_stateless_skb(struct sk_buff *skb) 1445{ 1446 trace_consume_skb(skb, __builtin_return_address(0)); 1447 skb_release_data(skb, SKB_CONSUMED, false); 1448 kfree_skbmem(skb); 1449} 1450 1451static void napi_skb_cache_put(struct sk_buff *skb) 1452{ 1453 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 1454 u32 i; 1455 1456 if (!kasan_mempool_poison_object(skb)) 1457 return; 1458 1459 nc->skb_cache[nc->skb_count++] = skb; 1460 1461 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) { 1462 for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++) 1463 kasan_mempool_unpoison_object(nc->skb_cache[i], 1464 kmem_cache_size(net_hotdata.skbuff_cache)); 1465 1466 kmem_cache_free_bulk(net_hotdata.skbuff_cache, NAPI_SKB_CACHE_HALF, 1467 nc->skb_cache + NAPI_SKB_CACHE_HALF); 1468 nc->skb_count = NAPI_SKB_CACHE_HALF; 1469 } 1470} 1471 1472void __napi_kfree_skb(struct sk_buff *skb, enum skb_drop_reason reason) 1473{ 1474 skb_release_all(skb, reason, true); 1475 napi_skb_cache_put(skb); 1476} 1477 1478void napi_skb_free_stolen_head(struct sk_buff *skb) 1479{ 1480 if (unlikely(skb->slow_gro)) { 1481 nf_reset_ct(skb); 1482 skb_dst_drop(skb); 1483 skb_ext_put(skb); 1484 skb_orphan(skb); 1485 skb->slow_gro = 0; 1486 } 1487 napi_skb_cache_put(skb); 1488} 1489 1490void napi_consume_skb(struct sk_buff *skb, int budget) 1491{ 1492 /* Zero budget indicate non-NAPI context called us, like netpoll */ 1493 if (unlikely(!budget)) { 1494 dev_consume_skb_any(skb); 1495 return; 1496 } 1497 1498 DEBUG_NET_WARN_ON_ONCE(!in_softirq()); 1499 1500 if (!skb_unref(skb)) 1501 return; 1502 1503 /* if reaching here SKB is ready to free */ 1504 trace_consume_skb(skb, __builtin_return_address(0)); 1505 1506 /* if SKB is a clone, don't handle this case */ 1507 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) { 1508 __kfree_skb(skb); 1509 return; 1510 } 1511 1512 skb_release_all(skb, SKB_CONSUMED, !!budget); 1513 napi_skb_cache_put(skb); 1514} 1515EXPORT_SYMBOL(napi_consume_skb); 1516 1517/* Make sure a field is contained by headers group */ 1518#define CHECK_SKB_FIELD(field) \ 1519 BUILD_BUG_ON(offsetof(struct sk_buff, field) != \ 1520 offsetof(struct sk_buff, headers.field)); \ 1521 1522static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 1523{ 1524 new->tstamp = old->tstamp; 1525 /* We do not copy old->sk */ 1526 new->dev = old->dev; 1527 memcpy(new->cb, old->cb, sizeof(old->cb)); 1528 skb_dst_copy(new, old); 1529 __skb_ext_copy(new, old); 1530 __nf_copy(new, old, false); 1531 1532 /* Note : this field could be in the headers group. 1533 * It is not yet because we do not want to have a 16 bit hole 1534 */ 1535 new->queue_mapping = old->queue_mapping; 1536 1537 memcpy(&new->headers, &old->headers, sizeof(new->headers)); 1538 CHECK_SKB_FIELD(protocol); 1539 CHECK_SKB_FIELD(csum); 1540 CHECK_SKB_FIELD(hash); 1541 CHECK_SKB_FIELD(priority); 1542 CHECK_SKB_FIELD(skb_iif); 1543 CHECK_SKB_FIELD(vlan_proto); 1544 CHECK_SKB_FIELD(vlan_tci); 1545 CHECK_SKB_FIELD(transport_header); 1546 CHECK_SKB_FIELD(network_header); 1547 CHECK_SKB_FIELD(mac_header); 1548 CHECK_SKB_FIELD(inner_protocol); 1549 CHECK_SKB_FIELD(inner_transport_header); 1550 CHECK_SKB_FIELD(inner_network_header); 1551 CHECK_SKB_FIELD(inner_mac_header); 1552 CHECK_SKB_FIELD(mark); 1553#ifdef CONFIG_NETWORK_SECMARK 1554 CHECK_SKB_FIELD(secmark); 1555#endif 1556#ifdef CONFIG_NET_RX_BUSY_POLL 1557 CHECK_SKB_FIELD(napi_id); 1558#endif 1559 CHECK_SKB_FIELD(alloc_cpu); 1560#ifdef CONFIG_XPS 1561 CHECK_SKB_FIELD(sender_cpu); 1562#endif 1563#ifdef CONFIG_NET_SCHED 1564 CHECK_SKB_FIELD(tc_index); 1565#endif 1566 1567} 1568 1569/* 1570 * You should not add any new code to this function. Add it to 1571 * __copy_skb_header above instead. 1572 */ 1573static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb) 1574{ 1575#define C(x) n->x = skb->x 1576 1577 n->next = n->prev = NULL; 1578 n->sk = NULL; 1579 __copy_skb_header(n, skb); 1580 1581 C(len); 1582 C(data_len); 1583 C(mac_len); 1584 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len; 1585 n->cloned = 1; 1586 n->nohdr = 0; 1587 n->peeked = 0; 1588 C(pfmemalloc); 1589 C(pp_recycle); 1590 n->destructor = NULL; 1591 C(tail); 1592 C(end); 1593 C(head); 1594 C(head_frag); 1595 C(data); 1596 C(truesize); 1597 refcount_set(&n->users, 1); 1598 1599 atomic_inc(&(skb_shinfo(skb)->dataref)); 1600 skb->cloned = 1; 1601 1602 return n; 1603#undef C 1604} 1605 1606/** 1607 * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg 1608 * @first: first sk_buff of the msg 1609 */ 1610struct sk_buff *alloc_skb_for_msg(struct sk_buff *first) 1611{ 1612 struct sk_buff *n; 1613 1614 n = alloc_skb(0, GFP_ATOMIC); 1615 if (!n) 1616 return NULL; 1617 1618 n->len = first->len; 1619 n->data_len = first->len; 1620 n->truesize = first->truesize; 1621 1622 skb_shinfo(n)->frag_list = first; 1623 1624 __copy_skb_header(n, first); 1625 n->destructor = NULL; 1626 1627 return n; 1628} 1629EXPORT_SYMBOL_GPL(alloc_skb_for_msg); 1630 1631/** 1632 * skb_morph - morph one skb into another 1633 * @dst: the skb to receive the contents 1634 * @src: the skb to supply the contents 1635 * 1636 * This is identical to skb_clone except that the target skb is 1637 * supplied by the user. 1638 * 1639 * The target skb is returned upon exit. 1640 */ 1641struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src) 1642{ 1643 skb_release_all(dst, SKB_CONSUMED, false); 1644 return __skb_clone(dst, src); 1645} 1646EXPORT_SYMBOL_GPL(skb_morph); 1647 1648int mm_account_pinned_pages(struct mmpin *mmp, size_t size) 1649{ 1650 unsigned long max_pg, num_pg, new_pg, old_pg, rlim; 1651 struct user_struct *user; 1652 1653 if (capable(CAP_IPC_LOCK) || !size) 1654 return 0; 1655 1656 rlim = rlimit(RLIMIT_MEMLOCK); 1657 if (rlim == RLIM_INFINITY) 1658 return 0; 1659 1660 num_pg = (size >> PAGE_SHIFT) + 2; /* worst case */ 1661 max_pg = rlim >> PAGE_SHIFT; 1662 user = mmp->user ? : current_user(); 1663 1664 old_pg = atomic_long_read(&user->locked_vm); 1665 do { 1666 new_pg = old_pg + num_pg; 1667 if (new_pg > max_pg) 1668 return -ENOBUFS; 1669 } while (!atomic_long_try_cmpxchg(&user->locked_vm, &old_pg, new_pg)); 1670 1671 if (!mmp->user) { 1672 mmp->user = get_uid(user); 1673 mmp->num_pg = num_pg; 1674 } else { 1675 mmp->num_pg += num_pg; 1676 } 1677 1678 return 0; 1679} 1680EXPORT_SYMBOL_GPL(mm_account_pinned_pages); 1681 1682void mm_unaccount_pinned_pages(struct mmpin *mmp) 1683{ 1684 if (mmp->user) { 1685 atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm); 1686 free_uid(mmp->user); 1687 } 1688} 1689EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages); 1690 1691static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size) 1692{ 1693 struct ubuf_info_msgzc *uarg; 1694 struct sk_buff *skb; 1695 1696 WARN_ON_ONCE(!in_task()); 1697 1698 skb = sock_omalloc(sk, 0, GFP_KERNEL); 1699 if (!skb) 1700 return NULL; 1701 1702 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb)); 1703 uarg = (void *)skb->cb; 1704 uarg->mmp.user = NULL; 1705 1706 if (mm_account_pinned_pages(&uarg->mmp, size)) { 1707 kfree_skb(skb); 1708 return NULL; 1709 } 1710 1711 uarg->ubuf.callback = msg_zerocopy_callback; 1712 uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1; 1713 uarg->len = 1; 1714 uarg->bytelen = size; 1715 uarg->zerocopy = 1; 1716 uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; 1717 refcount_set(&uarg->ubuf.refcnt, 1); 1718 sock_hold(sk); 1719 1720 return &uarg->ubuf; 1721} 1722 1723static inline struct sk_buff *skb_from_uarg(struct ubuf_info_msgzc *uarg) 1724{ 1725 return container_of((void *)uarg, struct sk_buff, cb); 1726} 1727 1728struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, 1729 struct ubuf_info *uarg) 1730{ 1731 if (uarg) { 1732 struct ubuf_info_msgzc *uarg_zc; 1733 const u32 byte_limit = 1 << 19; /* limit to a few TSO */ 1734 u32 bytelen, next; 1735 1736 /* there might be non MSG_ZEROCOPY users */ 1737 if (uarg->callback != msg_zerocopy_callback) 1738 return NULL; 1739 1740 /* realloc only when socket is locked (TCP, UDP cork), 1741 * so uarg->len and sk_zckey access is serialized 1742 */ 1743 if (!sock_owned_by_user(sk)) { 1744 WARN_ON_ONCE(1); 1745 return NULL; 1746 } 1747 1748 uarg_zc = uarg_to_msgzc(uarg); 1749 bytelen = uarg_zc->bytelen + size; 1750 if (uarg_zc->len == USHRT_MAX - 1 || bytelen > byte_limit) { 1751 /* TCP can create new skb to attach new uarg */ 1752 if (sk->sk_type == SOCK_STREAM) 1753 goto new_alloc; 1754 return NULL; 1755 } 1756 1757 next = (u32)atomic_read(&sk->sk_zckey); 1758 if ((u32)(uarg_zc->id + uarg_zc->len) == next) { 1759 if (mm_account_pinned_pages(&uarg_zc->mmp, size)) 1760 return NULL; 1761 uarg_zc->len++; 1762 uarg_zc->bytelen = bytelen; 1763 atomic_set(&sk->sk_zckey, ++next); 1764 1765 /* no extra ref when appending to datagram (MSG_MORE) */ 1766 if (sk->sk_type == SOCK_STREAM) 1767 net_zcopy_get(uarg); 1768 1769 return uarg; 1770 } 1771 } 1772 1773new_alloc: 1774 return msg_zerocopy_alloc(sk, size); 1775} 1776EXPORT_SYMBOL_GPL(msg_zerocopy_realloc); 1777 1778static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len) 1779{ 1780 struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); 1781 u32 old_lo, old_hi; 1782 u64 sum_len; 1783 1784 old_lo = serr->ee.ee_info; 1785 old_hi = serr->ee.ee_data; 1786 sum_len = old_hi - old_lo + 1ULL + len; 1787 1788 if (sum_len >= (1ULL << 32)) 1789 return false; 1790 1791 if (lo != old_hi + 1) 1792 return false; 1793 1794 serr->ee.ee_data += len; 1795 return true; 1796} 1797 1798static void __msg_zerocopy_callback(struct ubuf_info_msgzc *uarg) 1799{ 1800 struct sk_buff *tail, *skb = skb_from_uarg(uarg); 1801 struct sock_exterr_skb *serr; 1802 struct sock *sk = skb->sk; 1803 struct sk_buff_head *q; 1804 unsigned long flags; 1805 bool is_zerocopy; 1806 u32 lo, hi; 1807 u16 len; 1808 1809 mm_unaccount_pinned_pages(&uarg->mmp); 1810 1811 /* if !len, there was only 1 call, and it was aborted 1812 * so do not queue a completion notification 1813 */ 1814 if (!uarg->len || sock_flag(sk, SOCK_DEAD)) 1815 goto release; 1816 1817 len = uarg->len; 1818 lo = uarg->id; 1819 hi = uarg->id + len - 1; 1820 is_zerocopy = uarg->zerocopy; 1821 1822 serr = SKB_EXT_ERR(skb); 1823 memset(serr, 0, sizeof(*serr)); 1824 serr->ee.ee_errno = 0; 1825 serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY; 1826 serr->ee.ee_data = hi; 1827 serr->ee.ee_info = lo; 1828 if (!is_zerocopy) 1829 serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED; 1830 1831 q = &sk->sk_error_queue; 1832 spin_lock_irqsave(&q->lock, flags); 1833 tail = skb_peek_tail(q); 1834 if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY || 1835 !skb_zerocopy_notify_extend(tail, lo, len)) { 1836 __skb_queue_tail(q, skb); 1837 skb = NULL; 1838 } 1839 spin_unlock_irqrestore(&q->lock, flags); 1840 1841 sk_error_report(sk); 1842 1843release: 1844 consume_skb(skb); 1845 sock_put(sk); 1846} 1847 1848void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, 1849 bool success) 1850{ 1851 struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg); 1852 1853 uarg_zc->zerocopy = uarg_zc->zerocopy & success; 1854 1855 if (refcount_dec_and_test(&uarg->refcnt)) 1856 __msg_zerocopy_callback(uarg_zc); 1857} 1858EXPORT_SYMBOL_GPL(msg_zerocopy_callback); 1859 1860void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref) 1861{ 1862 struct sock *sk = skb_from_uarg(uarg_to_msgzc(uarg))->sk; 1863 1864 atomic_dec(&sk->sk_zckey); 1865 uarg_to_msgzc(uarg)->len--; 1866 1867 if (have_uref) 1868 msg_zerocopy_callback(NULL, uarg, true); 1869} 1870EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort); 1871 1872int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, 1873 struct msghdr *msg, int len, 1874 struct ubuf_info *uarg) 1875{ 1876 struct ubuf_info *orig_uarg = skb_zcopy(skb); 1877 int err, orig_len = skb->len; 1878 1879 /* An skb can only point to one uarg. This edge case happens when 1880 * TCP appends to an skb, but zerocopy_realloc triggered a new alloc. 1881 */ 1882 if (orig_uarg && uarg != orig_uarg) 1883 return -EEXIST; 1884 1885 err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len); 1886 if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) { 1887 struct sock *save_sk = skb->sk; 1888 1889 /* Streams do not free skb on error. Reset to prev state. */ 1890 iov_iter_revert(&msg->msg_iter, skb->len - orig_len); 1891 skb->sk = sk; 1892 ___pskb_trim(skb, orig_len); 1893 skb->sk = save_sk; 1894 return err; 1895 } 1896 1897 skb_zcopy_set(skb, uarg, NULL); 1898 return skb->len - orig_len; 1899} 1900EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream); 1901 1902void __skb_zcopy_downgrade_managed(struct sk_buff *skb) 1903{ 1904 int i; 1905 1906 skb_shinfo(skb)->flags &= ~SKBFL_MANAGED_FRAG_REFS; 1907 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 1908 skb_frag_ref(skb, i); 1909} 1910EXPORT_SYMBOL_GPL(__skb_zcopy_downgrade_managed); 1911 1912static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig, 1913 gfp_t gfp_mask) 1914{ 1915 if (skb_zcopy(orig)) { 1916 if (skb_zcopy(nskb)) { 1917 /* !gfp_mask callers are verified to !skb_zcopy(nskb) */ 1918 if (!gfp_mask) { 1919 WARN_ON_ONCE(1); 1920 return -ENOMEM; 1921 } 1922 if (skb_uarg(nskb) == skb_uarg(orig)) 1923 return 0; 1924 if (skb_copy_ubufs(nskb, GFP_ATOMIC)) 1925 return -EIO; 1926 } 1927 skb_zcopy_set(nskb, skb_uarg(orig), NULL); 1928 } 1929 return 0; 1930} 1931 1932/** 1933 * skb_copy_ubufs - copy userspace skb frags buffers to kernel 1934 * @skb: the skb to modify 1935 * @gfp_mask: allocation priority 1936 * 1937 * This must be called on skb with SKBFL_ZEROCOPY_ENABLE. 1938 * It will copy all frags into kernel and drop the reference 1939 * to userspace pages. 1940 * 1941 * If this function is called from an interrupt gfp_mask() must be 1942 * %GFP_ATOMIC. 1943 * 1944 * Returns 0 on success or a negative error code on failure 1945 * to allocate kernel memory to copy to. 1946 */ 1947int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) 1948{ 1949 int num_frags = skb_shinfo(skb)->nr_frags; 1950 struct page *page, *head = NULL; 1951 int i, order, psize, new_frags; 1952 u32 d_off; 1953 1954 if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) 1955 return -EINVAL; 1956 1957 if (!num_frags) 1958 goto release; 1959 1960 /* We might have to allocate high order pages, so compute what minimum 1961 * page order is needed. 1962 */ 1963 order = 0; 1964 while ((PAGE_SIZE << order) * MAX_SKB_FRAGS < __skb_pagelen(skb)) 1965 order++; 1966 psize = (PAGE_SIZE << order); 1967 1968 new_frags = (__skb_pagelen(skb) + psize - 1) >> (PAGE_SHIFT + order); 1969 for (i = 0; i < new_frags; i++) { 1970 page = alloc_pages(gfp_mask | __GFP_COMP, order); 1971 if (!page) { 1972 while (head) { 1973 struct page *next = (struct page *)page_private(head); 1974 put_page(head); 1975 head = next; 1976 } 1977 return -ENOMEM; 1978 } 1979 set_page_private(page, (unsigned long)head); 1980 head = page; 1981 } 1982 1983 page = head; 1984 d_off = 0; 1985 for (i = 0; i < num_frags; i++) { 1986 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 1987 u32 p_off, p_len, copied; 1988 struct page *p; 1989 u8 *vaddr; 1990 1991 skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f), 1992 p, p_off, p_len, copied) { 1993 u32 copy, done = 0; 1994 vaddr = kmap_atomic(p); 1995 1996 while (done < p_len) { 1997 if (d_off == psize) { 1998 d_off = 0; 1999 page = (struct page *)page_private(page); 2000 } 2001 copy = min_t(u32, psize - d_off, p_len - done); 2002 memcpy(page_address(page) + d_off, 2003 vaddr + p_off + done, copy); 2004 done += copy; 2005 d_off += copy; 2006 } 2007 kunmap_atomic(vaddr); 2008 } 2009 } 2010 2011 /* skb frags release userspace buffers */ 2012 for (i = 0; i < num_frags; i++) 2013 skb_frag_unref(skb, i); 2014 2015 /* skb frags point to kernel buffers */ 2016 for (i = 0; i < new_frags - 1; i++) { 2017 __skb_fill_netmem_desc(skb, i, page_to_netmem(head), 0, psize); 2018 head = (struct page *)page_private(head); 2019 } 2020 __skb_fill_netmem_desc(skb, new_frags - 1, page_to_netmem(head), 0, 2021 d_off); 2022 skb_shinfo(skb)->nr_frags = new_frags; 2023 2024release: 2025 skb_zcopy_clear(skb, false); 2026 return 0; 2027} 2028EXPORT_SYMBOL_GPL(skb_copy_ubufs); 2029 2030/** 2031 * skb_clone - duplicate an sk_buff 2032 * @skb: buffer to clone 2033 * @gfp_mask: allocation priority 2034 * 2035 * Duplicate an &sk_buff. The new one is not owned by a socket. Both 2036 * copies share the same packet data but not structure. The new 2037 * buffer has a reference count of 1. If the allocation fails the 2038 * function returns %NULL otherwise the new buffer is returned. 2039 * 2040 * If this function is called from an interrupt gfp_mask() must be 2041 * %GFP_ATOMIC. 2042 */ 2043 2044struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) 2045{ 2046 struct sk_buff_fclones *fclones = container_of(skb, 2047 struct sk_buff_fclones, 2048 skb1); 2049 struct sk_buff *n; 2050 2051 if (skb_orphan_frags(skb, gfp_mask)) 2052 return NULL; 2053 2054 if (skb->fclone == SKB_FCLONE_ORIG && 2055 refcount_read(&fclones->fclone_ref) == 1) { 2056 n = &fclones->skb2; 2057 refcount_set(&fclones->fclone_ref, 2); 2058 n->fclone = SKB_FCLONE_CLONE; 2059 } else { 2060 if (skb_pfmemalloc(skb)) 2061 gfp_mask |= __GFP_MEMALLOC; 2062 2063 n = kmem_cache_alloc(net_hotdata.skbuff_cache, gfp_mask); 2064 if (!n) 2065 return NULL; 2066 2067 n->fclone = SKB_FCLONE_UNAVAILABLE; 2068 } 2069 2070 return __skb_clone(n, skb); 2071} 2072EXPORT_SYMBOL(skb_clone); 2073 2074void skb_headers_offset_update(struct sk_buff *skb, int off) 2075{ 2076 /* Only adjust this if it actually is csum_start rather than csum */ 2077 if (skb->ip_summed == CHECKSUM_PARTIAL) 2078 skb->csum_start += off; 2079 /* {transport,network,mac}_header and tail are relative to skb->head */ 2080 skb->transport_header += off; 2081 skb->network_header += off; 2082 if (skb_mac_header_was_set(skb)) 2083 skb->mac_header += off; 2084 skb->inner_transport_header += off; 2085 skb->inner_network_header += off; 2086 skb->inner_mac_header += off; 2087} 2088EXPORT_SYMBOL(skb_headers_offset_update); 2089 2090void skb_copy_header(struct sk_buff *new, const struct sk_buff *old) 2091{ 2092 __copy_skb_header(new, old); 2093 2094 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size; 2095 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs; 2096 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type; 2097} 2098EXPORT_SYMBOL(skb_copy_header); 2099 2100static inline int skb_alloc_rx_flag(const struct sk_buff *skb) 2101{ 2102 if (skb_pfmemalloc(skb)) 2103 return SKB_ALLOC_RX; 2104 return 0; 2105} 2106 2107/** 2108 * skb_copy - create private copy of an sk_buff 2109 * @skb: buffer to copy 2110 * @gfp_mask: allocation priority 2111 * 2112 * Make a copy of both an &sk_buff and its data. This is used when the 2113 * caller wishes to modify the data and needs a private copy of the 2114 * data to alter. Returns %NULL on failure or the pointer to the buffer 2115 * on success. The returned buffer has a reference count of 1. 2116 * 2117 * As by-product this function converts non-linear &sk_buff to linear 2118 * one, so that &sk_buff becomes completely private and caller is allowed 2119 * to modify all the data of returned buffer. This means that this 2120 * function is not recommended for use in circumstances when only 2121 * header is going to be modified. Use pskb_copy() instead. 2122 */ 2123 2124struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask) 2125{ 2126 int headerlen = skb_headroom(skb); 2127 unsigned int size = skb_end_offset(skb) + skb->data_len; 2128 struct sk_buff *n = __alloc_skb(size, gfp_mask, 2129 skb_alloc_rx_flag(skb), NUMA_NO_NODE); 2130 2131 if (!n) 2132 return NULL; 2133 2134 /* Set the data pointer */ 2135 skb_reserve(n, headerlen); 2136 /* Set the tail pointer and length */ 2137 skb_put(n, skb->len); 2138 2139 BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len)); 2140 2141 skb_copy_header(n, skb); 2142 return n; 2143} 2144EXPORT_SYMBOL(skb_copy); 2145 2146/** 2147 * __pskb_copy_fclone - create copy of an sk_buff with private head. 2148 * @skb: buffer to copy 2149 * @headroom: headroom of new skb 2150 * @gfp_mask: allocation priority 2151 * @fclone: if true allocate the copy of the skb from the fclone 2152 * cache instead of the head cache; it is recommended to set this 2153 * to true for the cases where the copy will likely be cloned 2154 * 2155 * Make a copy of both an &sk_buff and part of its data, located 2156 * in header. Fragmented data remain shared. This is used when 2157 * the caller wishes to modify only header of &sk_buff and needs 2158 * private copy of the header to alter. Returns %NULL on failure 2159 * or the pointer to the buffer on success. 2160 * The returned buffer has a reference count of 1. 2161 */ 2162 2163struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, 2164 gfp_t gfp_mask, bool fclone) 2165{ 2166 unsigned int size = skb_headlen(skb) + headroom; 2167 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0); 2168 struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE); 2169 2170 if (!n) 2171 goto out; 2172 2173 /* Set the data pointer */ 2174 skb_reserve(n, headroom); 2175 /* Set the tail pointer and length */ 2176 skb_put(n, skb_headlen(skb)); 2177 /* Copy the bytes */ 2178 skb_copy_from_linear_data(skb, n->data, n->len); 2179 2180 n->truesize += skb->data_len; 2181 n->data_len = skb->data_len; 2182 n->len = skb->len; 2183 2184 if (skb_shinfo(skb)->nr_frags) { 2185 int i; 2186 2187 if (skb_orphan_frags(skb, gfp_mask) || 2188 skb_zerocopy_clone(n, skb, gfp_mask)) { 2189 kfree_skb(n); 2190 n = NULL; 2191 goto out; 2192 } 2193 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2194 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; 2195 skb_frag_ref(skb, i); 2196 } 2197 skb_shinfo(n)->nr_frags = i; 2198 } 2199 2200 if (skb_has_frag_list(skb)) { 2201 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list; 2202 skb_clone_fraglist(n); 2203 } 2204 2205 skb_copy_header(n, skb); 2206out: 2207 return n; 2208} 2209EXPORT_SYMBOL(__pskb_copy_fclone); 2210 2211/** 2212 * pskb_expand_head - reallocate header of &sk_buff 2213 * @skb: buffer to reallocate 2214 * @nhead: room to add at head 2215 * @ntail: room to add at tail 2216 * @gfp_mask: allocation priority 2217 * 2218 * Expands (or creates identical copy, if @nhead and @ntail are zero) 2219 * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have 2220 * reference count of 1. Returns zero in the case of success or error, 2221 * if expansion failed. In the last case, &sk_buff is not changed. 2222 * 2223 * All the pointers pointing into skb header may change and must be 2224 * reloaded after call to this function. 2225 */ 2226 2227int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, 2228 gfp_t gfp_mask) 2229{ 2230 unsigned int osize = skb_end_offset(skb); 2231 unsigned int size = osize + nhead + ntail; 2232 long off; 2233 u8 *data; 2234 int i; 2235 2236 BUG_ON(nhead < 0); 2237 2238 BUG_ON(skb_shared(skb)); 2239 2240 skb_zcopy_downgrade_managed(skb); 2241 2242 if (skb_pfmemalloc(skb)) 2243 gfp_mask |= __GFP_MEMALLOC; 2244 2245 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL); 2246 if (!data) 2247 goto nodata; 2248 size = SKB_WITH_OVERHEAD(size); 2249 2250 /* Copy only real data... and, alas, header. This should be 2251 * optimized for the cases when header is void. 2252 */ 2253 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head); 2254 2255 memcpy((struct skb_shared_info *)(data + size), 2256 skb_shinfo(skb), 2257 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags])); 2258 2259 /* 2260 * if shinfo is shared we must drop the old head gracefully, but if it 2261 * is not we can just drop the old head and let the existing refcount 2262 * be since all we did is relocate the values 2263 */ 2264 if (skb_cloned(skb)) { 2265 if (skb_orphan_frags(skb, gfp_mask)) 2266 goto nofrags; 2267 if (skb_zcopy(skb)) 2268 refcount_inc(&skb_uarg(skb)->refcnt); 2269 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 2270 skb_frag_ref(skb, i); 2271 2272 if (skb_has_frag_list(skb)) 2273 skb_clone_fraglist(skb); 2274 2275 skb_release_data(skb, SKB_CONSUMED, false); 2276 } else { 2277 skb_free_head(skb, false); 2278 } 2279 off = (data + nhead) - skb->head; 2280 2281 skb->head = data; 2282 skb->head_frag = 0; 2283 skb->data += off; 2284 2285 skb_set_end_offset(skb, size); 2286#ifdef NET_SKBUFF_DATA_USES_OFFSET 2287 off = nhead; 2288#endif 2289 skb->tail += off; 2290 skb_headers_offset_update(skb, nhead); 2291 skb->cloned = 0; 2292 skb->hdr_len = 0; 2293 skb->nohdr = 0; 2294 atomic_set(&skb_shinfo(skb)->dataref, 1); 2295 2296 skb_metadata_clear(skb); 2297 2298 /* It is not generally safe to change skb->truesize. 2299 * For the moment, we really care of rx path, or 2300 * when skb is orphaned (not attached to a socket). 2301 */ 2302 if (!skb->sk || skb->destructor == sock_edemux) 2303 skb->truesize += size - osize; 2304 2305 return 0; 2306 2307nofrags: 2308 skb_kfree_head(data, size); 2309nodata: 2310 return -ENOMEM; 2311} 2312EXPORT_SYMBOL(pskb_expand_head); 2313 2314/* Make private copy of skb with writable head and some headroom */ 2315 2316struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) 2317{ 2318 struct sk_buff *skb2; 2319 int delta = headroom - skb_headroom(skb); 2320 2321 if (delta <= 0) 2322 skb2 = pskb_copy(skb, GFP_ATOMIC); 2323 else { 2324 skb2 = skb_clone(skb, GFP_ATOMIC); 2325 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0, 2326 GFP_ATOMIC)) { 2327 kfree_skb(skb2); 2328 skb2 = NULL; 2329 } 2330 } 2331 return skb2; 2332} 2333EXPORT_SYMBOL(skb_realloc_headroom); 2334 2335/* Note: We plan to rework this in linux-6.4 */ 2336int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri) 2337{ 2338 unsigned int saved_end_offset, saved_truesize; 2339 struct skb_shared_info *shinfo; 2340 int res; 2341 2342 saved_end_offset = skb_end_offset(skb); 2343 saved_truesize = skb->truesize; 2344 2345 res = pskb_expand_head(skb, 0, 0, pri); 2346 if (res) 2347 return res; 2348 2349 skb->truesize = saved_truesize; 2350 2351 if (likely(skb_end_offset(skb) == saved_end_offset)) 2352 return 0; 2353 2354 /* We can not change skb->end if the original or new value 2355 * is SKB_SMALL_HEAD_HEADROOM, as it might break skb_kfree_head(). 2356 */ 2357 if (saved_end_offset == SKB_SMALL_HEAD_HEADROOM || 2358 skb_end_offset(skb) == SKB_SMALL_HEAD_HEADROOM) { 2359 /* We think this path should not be taken. 2360 * Add a temporary trace to warn us just in case. 2361 */ 2362 pr_err_once("__skb_unclone_keeptruesize() skb_end_offset() %u -> %u\n", 2363 saved_end_offset, skb_end_offset(skb)); 2364 WARN_ON_ONCE(1); 2365 return 0; 2366 } 2367 2368 shinfo = skb_shinfo(skb); 2369 2370 /* We are about to change back skb->end, 2371 * we need to move skb_shinfo() to its new location. 2372 */ 2373 memmove(skb->head + saved_end_offset, 2374 shinfo, 2375 offsetof(struct skb_shared_info, frags[shinfo->nr_frags])); 2376 2377 skb_set_end_offset(skb, saved_end_offset); 2378 2379 return 0; 2380} 2381 2382/** 2383 * skb_expand_head - reallocate header of &sk_buff 2384 * @skb: buffer to reallocate 2385 * @headroom: needed headroom 2386 * 2387 * Unlike skb_realloc_headroom, this one does not allocate a new skb 2388 * if possible; copies skb->sk to new skb as needed 2389 * and frees original skb in case of failures. 2390 * 2391 * It expect increased headroom and generates warning otherwise. 2392 */ 2393 2394struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom) 2395{ 2396 int delta = headroom - skb_headroom(skb); 2397 int osize = skb_end_offset(skb); 2398 struct sock *sk = skb->sk; 2399 2400 if (WARN_ONCE(delta <= 0, 2401 "%s is expecting an increase in the headroom", __func__)) 2402 return skb; 2403 2404 delta = SKB_DATA_ALIGN(delta); 2405 /* pskb_expand_head() might crash, if skb is shared. */ 2406 if (skb_shared(skb) || !is_skb_wmem(skb)) { 2407 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC); 2408 2409 if (unlikely(!nskb)) 2410 goto fail; 2411 2412 if (sk) 2413 skb_set_owner_w(nskb, sk); 2414 consume_skb(skb); 2415 skb = nskb; 2416 } 2417 if (pskb_expand_head(skb, delta, 0, GFP_ATOMIC)) 2418 goto fail; 2419 2420 if (sk && is_skb_wmem(skb)) { 2421 delta = skb_end_offset(skb) - osize; 2422 refcount_add(delta, &sk->sk_wmem_alloc); 2423 skb->truesize += delta; 2424 } 2425 return skb; 2426 2427fail: 2428 kfree_skb(skb); 2429 return NULL; 2430} 2431EXPORT_SYMBOL(skb_expand_head); 2432 2433/** 2434 * skb_copy_expand - copy and expand sk_buff 2435 * @skb: buffer to copy 2436 * @newheadroom: new free bytes at head 2437 * @newtailroom: new free bytes at tail 2438 * @gfp_mask: allocation priority 2439 * 2440 * Make a copy of both an &sk_buff and its data and while doing so 2441 * allocate additional space. 2442 * 2443 * This is used when the caller wishes to modify the data and needs a 2444 * private copy of the data to alter as well as more space for new fields. 2445 * Returns %NULL on failure or the pointer to the buffer 2446 * on success. The returned buffer has a reference count of 1. 2447 * 2448 * You must pass %GFP_ATOMIC as the allocation priority if this function 2449 * is called from an interrupt. 2450 */ 2451struct sk_buff *skb_copy_expand(const struct sk_buff *skb, 2452 int newheadroom, int newtailroom, 2453 gfp_t gfp_mask) 2454{ 2455 /* 2456 * Allocate the copy buffer 2457 */ 2458 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom, 2459 gfp_mask, skb_alloc_rx_flag(skb), 2460 NUMA_NO_NODE); 2461 int oldheadroom = skb_headroom(skb); 2462 int head_copy_len, head_copy_off; 2463 2464 if (!n) 2465 return NULL; 2466 2467 skb_reserve(n, newheadroom); 2468 2469 /* Set the tail pointer and length */ 2470 skb_put(n, skb->len); 2471 2472 head_copy_len = oldheadroom; 2473 head_copy_off = 0; 2474 if (newheadroom <= head_copy_len) 2475 head_copy_len = newheadroom; 2476 else 2477 head_copy_off = newheadroom - head_copy_len; 2478 2479 /* Copy the linear header and data. */ 2480 BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off, 2481 skb->len + head_copy_len)); 2482 2483 skb_copy_header(n, skb); 2484 2485 skb_headers_offset_update(n, newheadroom - oldheadroom); 2486 2487 return n; 2488} 2489EXPORT_SYMBOL(skb_copy_expand); 2490 2491/** 2492 * __skb_pad - zero pad the tail of an skb 2493 * @skb: buffer to pad 2494 * @pad: space to pad 2495 * @free_on_error: free buffer on error 2496 * 2497 * Ensure that a buffer is followed by a padding area that is zero 2498 * filled. Used by network drivers which may DMA or transfer data 2499 * beyond the buffer end onto the wire. 2500 * 2501 * May return error in out of memory cases. The skb is freed on error 2502 * if @free_on_error is true. 2503 */ 2504 2505int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error) 2506{ 2507 int err; 2508 int ntail; 2509 2510 /* If the skbuff is non linear tailroom is always zero.. */ 2511 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) { 2512 memset(skb->data+skb->len, 0, pad); 2513 return 0; 2514 } 2515 2516 ntail = skb->data_len + pad - (skb->end - skb->tail); 2517 if (likely(skb_cloned(skb) || ntail > 0)) { 2518 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC); 2519 if (unlikely(err)) 2520 goto free_skb; 2521 } 2522 2523 /* FIXME: The use of this function with non-linear skb's really needs 2524 * to be audited. 2525 */ 2526 err = skb_linearize(skb); 2527 if (unlikely(err)) 2528 goto free_skb; 2529 2530 memset(skb->data + skb->len, 0, pad); 2531 return 0; 2532 2533free_skb: 2534 if (free_on_error) 2535 kfree_skb(skb); 2536 return err; 2537} 2538EXPORT_SYMBOL(__skb_pad); 2539 2540/** 2541 * pskb_put - add data to the tail of a potentially fragmented buffer 2542 * @skb: start of the buffer to use 2543 * @tail: tail fragment of the buffer to use 2544 * @len: amount of data to add 2545 * 2546 * This function extends the used data area of the potentially 2547 * fragmented buffer. @tail must be the last fragment of @skb -- or 2548 * @skb itself. If this would exceed the total buffer size the kernel 2549 * will panic. A pointer to the first byte of the extra data is 2550 * returned. 2551 */ 2552 2553void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len) 2554{ 2555 if (tail != skb) { 2556 skb->data_len += len; 2557 skb->len += len; 2558 } 2559 return skb_put(tail, len); 2560} 2561EXPORT_SYMBOL_GPL(pskb_put); 2562 2563/** 2564 * skb_put - add data to a buffer 2565 * @skb: buffer to use 2566 * @len: amount of data to add 2567 * 2568 * This function extends the used data area of the buffer. If this would 2569 * exceed the total buffer size the kernel will panic. A pointer to the 2570 * first byte of the extra data is returned. 2571 */ 2572void *skb_put(struct sk_buff *skb, unsigned int len) 2573{ 2574 void *tmp = skb_tail_pointer(skb); 2575 SKB_LINEAR_ASSERT(skb); 2576 skb->tail += len; 2577 skb->len += len; 2578 if (unlikely(skb->tail > skb->end)) 2579 skb_over_panic(skb, len, __builtin_return_address(0)); 2580 return tmp; 2581} 2582EXPORT_SYMBOL(skb_put); 2583 2584/** 2585 * skb_push - add data to the start of a buffer 2586 * @skb: buffer to use 2587 * @len: amount of data to add 2588 * 2589 * This function extends the used data area of the buffer at the buffer 2590 * start. If this would exceed the total buffer headroom the kernel will 2591 * panic. A pointer to the first byte of the extra data is returned. 2592 */ 2593void *skb_push(struct sk_buff *skb, unsigned int len) 2594{ 2595 skb->data -= len; 2596 skb->len += len; 2597 if (unlikely(skb->data < skb->head)) 2598 skb_under_panic(skb, len, __builtin_return_address(0)); 2599 return skb->data; 2600} 2601EXPORT_SYMBOL(skb_push); 2602 2603/** 2604 * skb_pull - remove data from the start of a buffer 2605 * @skb: buffer to use 2606 * @len: amount of data to remove 2607 * 2608 * This function removes data from the start of a buffer, returning 2609 * the memory to the headroom. A pointer to the next data in the buffer 2610 * is returned. Once the data has been pulled future pushes will overwrite 2611 * the old data. 2612 */ 2613void *skb_pull(struct sk_buff *skb, unsigned int len) 2614{ 2615 return skb_pull_inline(skb, len); 2616} 2617EXPORT_SYMBOL(skb_pull); 2618 2619/** 2620 * skb_pull_data - remove data from the start of a buffer returning its 2621 * original position. 2622 * @skb: buffer to use 2623 * @len: amount of data to remove 2624 * 2625 * This function removes data from the start of a buffer, returning 2626 * the memory to the headroom. A pointer to the original data in the buffer 2627 * is returned after checking if there is enough data to pull. Once the 2628 * data has been pulled future pushes will overwrite the old data. 2629 */ 2630void *skb_pull_data(struct sk_buff *skb, size_t len) 2631{ 2632 void *data = skb->data; 2633 2634 if (skb->len < len) 2635 return NULL; 2636 2637 skb_pull(skb, len); 2638 2639 return data; 2640} 2641EXPORT_SYMBOL(skb_pull_data); 2642 2643/** 2644 * skb_trim - remove end from a buffer 2645 * @skb: buffer to alter 2646 * @len: new length 2647 * 2648 * Cut the length of a buffer down by removing data from the tail. If 2649 * the buffer is already under the length specified it is not modified. 2650 * The skb must be linear. 2651 */ 2652void skb_trim(struct sk_buff *skb, unsigned int len) 2653{ 2654 if (skb->len > len) 2655 __skb_trim(skb, len); 2656} 2657EXPORT_SYMBOL(skb_trim); 2658 2659/* Trims skb to length len. It can change skb pointers. 2660 */ 2661 2662int ___pskb_trim(struct sk_buff *skb, unsigned int len) 2663{ 2664 struct sk_buff **fragp; 2665 struct sk_buff *frag; 2666 int offset = skb_headlen(skb); 2667 int nfrags = skb_shinfo(skb)->nr_frags; 2668 int i; 2669 int err; 2670 2671 if (skb_cloned(skb) && 2672 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))) 2673 return err; 2674 2675 i = 0; 2676 if (offset >= len) 2677 goto drop_pages; 2678 2679 for (; i < nfrags; i++) { 2680 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]); 2681 2682 if (end < len) { 2683 offset = end; 2684 continue; 2685 } 2686 2687 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset); 2688 2689drop_pages: 2690 skb_shinfo(skb)->nr_frags = i; 2691 2692 for (; i < nfrags; i++) 2693 skb_frag_unref(skb, i); 2694 2695 if (skb_has_frag_list(skb)) 2696 skb_drop_fraglist(skb); 2697 goto done; 2698 } 2699 2700 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp); 2701 fragp = &frag->next) { 2702 int end = offset + frag->len; 2703 2704 if (skb_shared(frag)) { 2705 struct sk_buff *nfrag; 2706 2707 nfrag = skb_clone(frag, GFP_ATOMIC); 2708 if (unlikely(!nfrag)) 2709 return -ENOMEM; 2710 2711 nfrag->next = frag->next; 2712 consume_skb(frag); 2713 frag = nfrag; 2714 *fragp = frag; 2715 } 2716 2717 if (end < len) { 2718 offset = end; 2719 continue; 2720 } 2721 2722 if (end > len && 2723 unlikely((err = pskb_trim(frag, len - offset)))) 2724 return err; 2725 2726 if (frag->next) 2727 skb_drop_list(&frag->next); 2728 break; 2729 } 2730 2731done: 2732 if (len > skb_headlen(skb)) { 2733 skb->data_len -= skb->len - len; 2734 skb->len = len; 2735 } else { 2736 skb->len = len; 2737 skb->data_len = 0; 2738 skb_set_tail_pointer(skb, len); 2739 } 2740 2741 if (!skb->sk || skb->destructor == sock_edemux) 2742 skb_condense(skb); 2743 return 0; 2744} 2745EXPORT_SYMBOL(___pskb_trim); 2746 2747/* Note : use pskb_trim_rcsum() instead of calling this directly 2748 */ 2749int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len) 2750{ 2751 if (skb->ip_summed == CHECKSUM_COMPLETE) { 2752 int delta = skb->len - len; 2753 2754 skb->csum = csum_block_sub(skb->csum, 2755 skb_checksum(skb, len, delta, 0), 2756 len); 2757 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 2758 int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len; 2759 int offset = skb_checksum_start_offset(skb) + skb->csum_offset; 2760 2761 if (offset + sizeof(__sum16) > hdlen) 2762 return -EINVAL; 2763 } 2764 return __pskb_trim(skb, len); 2765} 2766EXPORT_SYMBOL(pskb_trim_rcsum_slow); 2767 2768/** 2769 * __pskb_pull_tail - advance tail of skb header 2770 * @skb: buffer to reallocate 2771 * @delta: number of bytes to advance tail 2772 * 2773 * The function makes a sense only on a fragmented &sk_buff, 2774 * it expands header moving its tail forward and copying necessary 2775 * data from fragmented part. 2776 * 2777 * &sk_buff MUST have reference count of 1. 2778 * 2779 * Returns %NULL (and &sk_buff does not change) if pull failed 2780 * or value of new tail of skb in the case of success. 2781 * 2782 * All the pointers pointing into skb header may change and must be 2783 * reloaded after call to this function. 2784 */ 2785 2786/* Moves tail of skb head forward, copying data from fragmented part, 2787 * when it is necessary. 2788 * 1. It may fail due to malloc failure. 2789 * 2. It may change skb pointers. 2790 * 2791 * It is pretty complicated. Luckily, it is called only in exceptional cases. 2792 */ 2793void *__pskb_pull_tail(struct sk_buff *skb, int delta) 2794{ 2795 /* If skb has not enough free space at tail, get new one 2796 * plus 128 bytes for future expansions. If we have enough 2797 * room at tail, reallocate without expansion only if skb is cloned. 2798 */ 2799 int i, k, eat = (skb->tail + delta) - skb->end; 2800 2801 if (eat > 0 || skb_cloned(skb)) { 2802 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0, 2803 GFP_ATOMIC)) 2804 return NULL; 2805 } 2806 2807 BUG_ON(skb_copy_bits(skb, skb_headlen(skb), 2808 skb_tail_pointer(skb), delta)); 2809 2810 /* Optimization: no fragments, no reasons to preestimate 2811 * size of pulled pages. Superb. 2812 */ 2813 if (!skb_has_frag_list(skb)) 2814 goto pull_pages; 2815 2816 /* Estimate size of pulled pages. */ 2817 eat = delta; 2818 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2819 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2820 2821 if (size >= eat) 2822 goto pull_pages; 2823 eat -= size; 2824 } 2825 2826 /* If we need update frag list, we are in troubles. 2827 * Certainly, it is possible to add an offset to skb data, 2828 * but taking into account that pulling is expected to 2829 * be very rare operation, it is worth to fight against 2830 * further bloating skb head and crucify ourselves here instead. 2831 * Pure masohism, indeed. 8)8) 2832 */ 2833 if (eat) { 2834 struct sk_buff *list = skb_shinfo(skb)->frag_list; 2835 struct sk_buff *clone = NULL; 2836 struct sk_buff *insp = NULL; 2837 2838 do { 2839 if (list->len <= eat) { 2840 /* Eaten as whole. */ 2841 eat -= list->len; 2842 list = list->next; 2843 insp = list; 2844 } else { 2845 /* Eaten partially. */ 2846 if (skb_is_gso(skb) && !list->head_frag && 2847 skb_headlen(list)) 2848 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY; 2849 2850 if (skb_shared(list)) { 2851 /* Sucks! We need to fork list. :-( */ 2852 clone = skb_clone(list, GFP_ATOMIC); 2853 if (!clone) 2854 return NULL; 2855 insp = list->next; 2856 list = clone; 2857 } else { 2858 /* This may be pulled without 2859 * problems. */ 2860 insp = list; 2861 } 2862 if (!pskb_pull(list, eat)) { 2863 kfree_skb(clone); 2864 return NULL; 2865 } 2866 break; 2867 } 2868 } while (eat); 2869 2870 /* Free pulled out fragments. */ 2871 while ((list = skb_shinfo(skb)->frag_list) != insp) { 2872 skb_shinfo(skb)->frag_list = list->next; 2873 consume_skb(list); 2874 } 2875 /* And insert new clone at head. */ 2876 if (clone) { 2877 clone->next = list; 2878 skb_shinfo(skb)->frag_list = clone; 2879 } 2880 } 2881 /* Success! Now we may commit changes to skb data. */ 2882 2883pull_pages: 2884 eat = delta; 2885 k = 0; 2886 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2887 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2888 2889 if (size <= eat) { 2890 skb_frag_unref(skb, i); 2891 eat -= size; 2892 } else { 2893 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 2894 2895 *frag = skb_shinfo(skb)->frags[i]; 2896 if (eat) { 2897 skb_frag_off_add(frag, eat); 2898 skb_frag_size_sub(frag, eat); 2899 if (!i) 2900 goto end; 2901 eat = 0; 2902 } 2903 k++; 2904 } 2905 } 2906 skb_shinfo(skb)->nr_frags = k; 2907 2908end: 2909 skb->tail += delta; 2910 skb->data_len -= delta; 2911 2912 if (!skb->data_len) 2913 skb_zcopy_clear(skb, false); 2914 2915 return skb_tail_pointer(skb); 2916} 2917EXPORT_SYMBOL(__pskb_pull_tail); 2918 2919/** 2920 * skb_copy_bits - copy bits from skb to kernel buffer 2921 * @skb: source skb 2922 * @offset: offset in source 2923 * @to: destination buffer 2924 * @len: number of bytes to copy 2925 * 2926 * Copy the specified number of bytes from the source skb to the 2927 * destination buffer. 2928 * 2929 * CAUTION ! : 2930 * If its prototype is ever changed, 2931 * check arch/{*}/net/{*}.S files, 2932 * since it is called from BPF assembly code. 2933 */ 2934int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len) 2935{ 2936 int start = skb_headlen(skb); 2937 struct sk_buff *frag_iter; 2938 int i, copy; 2939 2940 if (offset > (int)skb->len - len) 2941 goto fault; 2942 2943 /* Copy header. */ 2944 if ((copy = start - offset) > 0) { 2945 if (copy > len) 2946 copy = len; 2947 skb_copy_from_linear_data_offset(skb, offset, to, copy); 2948 if ((len -= copy) == 0) 2949 return 0; 2950 offset += copy; 2951 to += copy; 2952 } 2953 2954 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2955 int end; 2956 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 2957 2958 WARN_ON(start > offset + len); 2959 2960 end = start + skb_frag_size(f); 2961 if ((copy = end - offset) > 0) { 2962 u32 p_off, p_len, copied; 2963 struct page *p; 2964 u8 *vaddr; 2965 2966 if (copy > len) 2967 copy = len; 2968 2969 skb_frag_foreach_page(f, 2970 skb_frag_off(f) + offset - start, 2971 copy, p, p_off, p_len, copied) { 2972 vaddr = kmap_atomic(p); 2973 memcpy(to + copied, vaddr + p_off, p_len); 2974 kunmap_atomic(vaddr); 2975 } 2976 2977 if ((len -= copy) == 0) 2978 return 0; 2979 offset += copy; 2980 to += copy; 2981 } 2982 start = end; 2983 } 2984 2985 skb_walk_frags(skb, frag_iter) { 2986 int end; 2987 2988 WARN_ON(start > offset + len); 2989 2990 end = start + frag_iter->len; 2991 if ((copy = end - offset) > 0) { 2992 if (copy > len) 2993 copy = len; 2994 if (skb_copy_bits(frag_iter, offset - start, to, copy)) 2995 goto fault; 2996 if ((len -= copy) == 0) 2997 return 0; 2998 offset += copy; 2999 to += copy; 3000 } 3001 start = end; 3002 } 3003 3004 if (!len) 3005 return 0; 3006 3007fault: 3008 return -EFAULT; 3009} 3010EXPORT_SYMBOL(skb_copy_bits); 3011 3012/* 3013 * Callback from splice_to_pipe(), if we need to release some pages 3014 * at the end of the spd in case we error'ed out in filling the pipe. 3015 */ 3016static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i) 3017{ 3018 put_page(spd->pages[i]); 3019} 3020 3021static struct page *linear_to_page(struct page *page, unsigned int *len, 3022 unsigned int *offset, 3023 struct sock *sk) 3024{ 3025 struct page_frag *pfrag = sk_page_frag(sk); 3026 3027 if (!sk_page_frag_refill(sk, pfrag)) 3028 return NULL; 3029 3030 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset); 3031 3032 memcpy(page_address(pfrag->page) + pfrag->offset, 3033 page_address(page) + *offset, *len); 3034 *offset = pfrag->offset; 3035 pfrag->offset += *len; 3036 3037 return pfrag->page; 3038} 3039 3040static bool spd_can_coalesce(const struct splice_pipe_desc *spd, 3041 struct page *page, 3042 unsigned int offset) 3043{ 3044 return spd->nr_pages && 3045 spd->pages[spd->nr_pages - 1] == page && 3046 (spd->partial[spd->nr_pages - 1].offset + 3047 spd->partial[spd->nr_pages - 1].len == offset); 3048} 3049 3050/* 3051 * Fill page/offset/length into spd, if it can hold more pages. 3052 */ 3053static bool spd_fill_page(struct splice_pipe_desc *spd, 3054 struct pipe_inode_info *pipe, struct page *page, 3055 unsigned int *len, unsigned int offset, 3056 bool linear, 3057 struct sock *sk) 3058{ 3059 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS)) 3060 return true; 3061 3062 if (linear) { 3063 page = linear_to_page(page, len, &offset, sk); 3064 if (!page) 3065 return true; 3066 } 3067 if (spd_can_coalesce(spd, page, offset)) { 3068 spd->partial[spd->nr_pages - 1].len += *len; 3069 return false; 3070 } 3071 get_page(page); 3072 spd->pages[spd->nr_pages] = page; 3073 spd->partial[spd->nr_pages].len = *len; 3074 spd->partial[spd->nr_pages].offset = offset; 3075 spd->nr_pages++; 3076 3077 return false; 3078} 3079 3080static bool __splice_segment(struct page *page, unsigned int poff, 3081 unsigned int plen, unsigned int *off, 3082 unsigned int *len, 3083 struct splice_pipe_desc *spd, bool linear, 3084 struct sock *sk, 3085 struct pipe_inode_info *pipe) 3086{ 3087 if (!*len) 3088 return true; 3089 3090 /* skip this segment if already processed */ 3091 if (*off >= plen) { 3092 *off -= plen; 3093 return false; 3094 } 3095 3096 /* ignore any bits we already processed */ 3097 poff += *off; 3098 plen -= *off; 3099 *off = 0; 3100 3101 do { 3102 unsigned int flen = min(*len, plen); 3103 3104 if (spd_fill_page(spd, pipe, page, &flen, poff, 3105 linear, sk)) 3106 return true; 3107 poff += flen; 3108 plen -= flen; 3109 *len -= flen; 3110 } while (*len && plen); 3111 3112 return false; 3113} 3114 3115/* 3116 * Map linear and fragment data from the skb to spd. It reports true if the 3117 * pipe is full or if we already spliced the requested length. 3118 */ 3119static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, 3120 unsigned int *offset, unsigned int *len, 3121 struct splice_pipe_desc *spd, struct sock *sk) 3122{ 3123 int seg; 3124 struct sk_buff *iter; 3125 3126 /* map the linear part : 3127 * If skb->head_frag is set, this 'linear' part is backed by a 3128 * fragment, and if the head is not shared with any clones then 3129 * we can avoid a copy since we own the head portion of this page. 3130 */ 3131 if (__splice_segment(virt_to_page(skb->data), 3132 (unsigned long) skb->data & (PAGE_SIZE - 1), 3133 skb_headlen(skb), 3134 offset, len, spd, 3135 skb_head_is_locked(skb), 3136 sk, pipe)) 3137 return true; 3138 3139 /* 3140 * then map the fragments 3141 */ 3142 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) { 3143 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg]; 3144 3145 if (__splice_segment(skb_frag_page(f), 3146 skb_frag_off(f), skb_frag_size(f), 3147 offset, len, spd, false, sk, pipe)) 3148 return true; 3149 } 3150 3151 skb_walk_frags(skb, iter) { 3152 if (*offset >= iter->len) { 3153 *offset -= iter->len; 3154 continue; 3155 } 3156 /* __skb_splice_bits() only fails if the output has no room 3157 * left, so no point in going over the frag_list for the error 3158 * case. 3159 */ 3160 if (__skb_splice_bits(iter, pipe, offset, len, spd, sk)) 3161 return true; 3162 } 3163 3164 return false; 3165} 3166 3167/* 3168 * Map data from the skb to a pipe. Should handle both the linear part, 3169 * the fragments, and the frag list. 3170 */ 3171int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, 3172 struct pipe_inode_info *pipe, unsigned int tlen, 3173 unsigned int flags) 3174{ 3175 struct partial_page partial[MAX_SKB_FRAGS]; 3176 struct page *pages[MAX_SKB_FRAGS]; 3177 struct splice_pipe_desc spd = { 3178 .pages = pages, 3179 .partial = partial, 3180 .nr_pages_max = MAX_SKB_FRAGS, 3181 .ops = &nosteal_pipe_buf_ops, 3182 .spd_release = sock_spd_release, 3183 }; 3184 int ret = 0; 3185 3186 __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk); 3187 3188 if (spd.nr_pages) 3189 ret = splice_to_pipe(pipe, &spd); 3190 3191 return ret; 3192} 3193EXPORT_SYMBOL_GPL(skb_splice_bits); 3194 3195static int sendmsg_locked(struct sock *sk, struct msghdr *msg) 3196{ 3197 struct socket *sock = sk->sk_socket; 3198 size_t size = msg_data_left(msg); 3199 3200 if (!sock) 3201 return -EINVAL; 3202 3203 if (!sock->ops->sendmsg_locked) 3204 return sock_no_sendmsg_locked(sk, msg, size); 3205 3206 return sock->ops->sendmsg_locked(sk, msg, size); 3207} 3208 3209static int sendmsg_unlocked(struct sock *sk, struct msghdr *msg) 3210{ 3211 struct socket *sock = sk->sk_socket; 3212 3213 if (!sock) 3214 return -EINVAL; 3215 return sock_sendmsg(sock, msg); 3216} 3217 3218typedef int (*sendmsg_func)(struct sock *sk, struct msghdr *msg); 3219static int __skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, 3220 int len, sendmsg_func sendmsg) 3221{ 3222 unsigned int orig_len = len; 3223 struct sk_buff *head = skb; 3224 unsigned short fragidx; 3225 int slen, ret; 3226 3227do_frag_list: 3228 3229 /* Deal with head data */ 3230 while (offset < skb_headlen(skb) && len) { 3231 struct kvec kv; 3232 struct msghdr msg; 3233 3234 slen = min_t(int, len, skb_headlen(skb) - offset); 3235 kv.iov_base = skb->data + offset; 3236 kv.iov_len = slen; 3237 memset(&msg, 0, sizeof(msg)); 3238 msg.msg_flags = MSG_DONTWAIT; 3239 3240 iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, &kv, 1, slen); 3241 ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked, 3242 sendmsg_unlocked, sk, &msg); 3243 if (ret <= 0) 3244 goto error; 3245 3246 offset += ret; 3247 len -= ret; 3248 } 3249 3250 /* All the data was skb head? */ 3251 if (!len) 3252 goto out; 3253 3254 /* Make offset relative to start of frags */ 3255 offset -= skb_headlen(skb); 3256 3257 /* Find where we are in frag list */ 3258 for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 3259 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 3260 3261 if (offset < skb_frag_size(frag)) 3262 break; 3263 3264 offset -= skb_frag_size(frag); 3265 } 3266 3267 for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 3268 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 3269 3270 slen = min_t(size_t, len, skb_frag_size(frag) - offset); 3271 3272 while (slen) { 3273 struct bio_vec bvec; 3274 struct msghdr msg = { 3275 .msg_flags = MSG_SPLICE_PAGES | MSG_DONTWAIT, 3276 }; 3277 3278 bvec_set_page(&bvec, skb_frag_page(frag), slen, 3279 skb_frag_off(frag) + offset); 3280 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, 3281 slen); 3282 3283 ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked, 3284 sendmsg_unlocked, sk, &msg); 3285 if (ret <= 0) 3286 goto error; 3287 3288 len -= ret; 3289 offset += ret; 3290 slen -= ret; 3291 } 3292 3293 offset = 0; 3294 } 3295 3296 if (len) { 3297 /* Process any frag lists */ 3298 3299 if (skb == head) { 3300 if (skb_has_frag_list(skb)) { 3301 skb = skb_shinfo(skb)->frag_list; 3302 goto do_frag_list; 3303 } 3304 } else if (skb->next) { 3305 skb = skb->next; 3306 goto do_frag_list; 3307 } 3308 } 3309 3310out: 3311 return orig_len - len; 3312 3313error: 3314 return orig_len == len ? ret : orig_len - len; 3315} 3316 3317/* Send skb data on a socket. Socket must be locked. */ 3318int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset, 3319 int len) 3320{ 3321 return __skb_send_sock(sk, skb, offset, len, sendmsg_locked); 3322} 3323EXPORT_SYMBOL_GPL(skb_send_sock_locked); 3324 3325/* Send skb data on a socket. Socket must be unlocked. */ 3326int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len) 3327{ 3328 return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked); 3329} 3330 3331/** 3332 * skb_store_bits - store bits from kernel buffer to skb 3333 * @skb: destination buffer 3334 * @offset: offset in destination 3335 * @from: source buffer 3336 * @len: number of bytes to copy 3337 * 3338 * Copy the specified number of bytes from the source buffer to the 3339 * destination skb. This function handles all the messy bits of 3340 * traversing fragment lists and such. 3341 */ 3342 3343int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len) 3344{ 3345 int start = skb_headlen(skb); 3346 struct sk_buff *frag_iter; 3347 int i, copy; 3348 3349 if (offset > (int)skb->len - len) 3350 goto fault; 3351 3352 if ((copy = start - offset) > 0) { 3353 if (copy > len) 3354 copy = len; 3355 skb_copy_to_linear_data_offset(skb, offset, from, copy); 3356 if ((len -= copy) == 0) 3357 return 0; 3358 offset += copy; 3359 from += copy; 3360 } 3361 3362 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3363 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3364 int end; 3365 3366 WARN_ON(start > offset + len); 3367 3368 end = start + skb_frag_size(frag); 3369 if ((copy = end - offset) > 0) { 3370 u32 p_off, p_len, copied; 3371 struct page *p; 3372 u8 *vaddr; 3373 3374 if (copy > len) 3375 copy = len; 3376 3377 skb_frag_foreach_page(frag, 3378 skb_frag_off(frag) + offset - start, 3379 copy, p, p_off, p_len, copied) { 3380 vaddr = kmap_atomic(p); 3381 memcpy(vaddr + p_off, from + copied, p_len); 3382 kunmap_atomic(vaddr); 3383 } 3384 3385 if ((len -= copy) == 0) 3386 return 0; 3387 offset += copy; 3388 from += copy; 3389 } 3390 start = end; 3391 } 3392 3393 skb_walk_frags(skb, frag_iter) { 3394 int end; 3395 3396 WARN_ON(start > offset + len); 3397 3398 end = start + frag_iter->len; 3399 if ((copy = end - offset) > 0) { 3400 if (copy > len) 3401 copy = len; 3402 if (skb_store_bits(frag_iter, offset - start, 3403 from, copy)) 3404 goto fault; 3405 if ((len -= copy) == 0) 3406 return 0; 3407 offset += copy; 3408 from += copy; 3409 } 3410 start = end; 3411 } 3412 if (!len) 3413 return 0; 3414 3415fault: 3416 return -EFAULT; 3417} 3418EXPORT_SYMBOL(skb_store_bits); 3419 3420/* Checksum skb data. */ 3421__wsum __skb_checksum(const struct sk_buff *skb, int offset, int len, 3422 __wsum csum, const struct skb_checksum_ops *ops) 3423{ 3424 int start = skb_headlen(skb); 3425 int i, copy = start - offset; 3426 struct sk_buff *frag_iter; 3427 int pos = 0; 3428 3429 /* Checksum header. */ 3430 if (copy > 0) { 3431 if (copy > len) 3432 copy = len; 3433 csum = INDIRECT_CALL_1(ops->update, csum_partial_ext, 3434 skb->data + offset, copy, csum); 3435 if ((len -= copy) == 0) 3436 return csum; 3437 offset += copy; 3438 pos = copy; 3439 } 3440 3441 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3442 int end; 3443 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3444 3445 WARN_ON(start > offset + len); 3446 3447 end = start + skb_frag_size(frag); 3448 if ((copy = end - offset) > 0) { 3449 u32 p_off, p_len, copied; 3450 struct page *p; 3451 __wsum csum2; 3452 u8 *vaddr; 3453 3454 if (copy > len) 3455 copy = len; 3456 3457 skb_frag_foreach_page(frag, 3458 skb_frag_off(frag) + offset - start, 3459 copy, p, p_off, p_len, copied) { 3460 vaddr = kmap_atomic(p); 3461 csum2 = INDIRECT_CALL_1(ops->update, 3462 csum_partial_ext, 3463 vaddr + p_off, p_len, 0); 3464 kunmap_atomic(vaddr); 3465 csum = INDIRECT_CALL_1(ops->combine, 3466 csum_block_add_ext, csum, 3467 csum2, pos, p_len); 3468 pos += p_len; 3469 } 3470 3471 if (!(len -= copy)) 3472 return csum; 3473 offset += copy; 3474 } 3475 start = end; 3476 } 3477 3478 skb_walk_frags(skb, frag_iter) { 3479 int end; 3480 3481 WARN_ON(start > offset + len); 3482 3483 end = start + frag_iter->len; 3484 if ((copy = end - offset) > 0) { 3485 __wsum csum2; 3486 if (copy > len) 3487 copy = len; 3488 csum2 = __skb_checksum(frag_iter, offset - start, 3489 copy, 0, ops); 3490 csum = INDIRECT_CALL_1(ops->combine, csum_block_add_ext, 3491 csum, csum2, pos, copy); 3492 if ((len -= copy) == 0) 3493 return csum; 3494 offset += copy; 3495 pos += copy; 3496 } 3497 start = end; 3498 } 3499 BUG_ON(len); 3500 3501 return csum; 3502} 3503EXPORT_SYMBOL(__skb_checksum); 3504 3505__wsum skb_checksum(const struct sk_buff *skb, int offset, 3506 int len, __wsum csum) 3507{ 3508 const struct skb_checksum_ops ops = { 3509 .update = csum_partial_ext, 3510 .combine = csum_block_add_ext, 3511 }; 3512 3513 return __skb_checksum(skb, offset, len, csum, &ops); 3514} 3515EXPORT_SYMBOL(skb_checksum); 3516 3517/* Both of above in one bottle. */ 3518 3519__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, 3520 u8 *to, int len) 3521{ 3522 int start = skb_headlen(skb); 3523 int i, copy = start - offset; 3524 struct sk_buff *frag_iter; 3525 int pos = 0; 3526 __wsum csum = 0; 3527 3528 /* Copy header. */ 3529 if (copy > 0) { 3530 if (copy > len) 3531 copy = len; 3532 csum = csum_partial_copy_nocheck(skb->data + offset, to, 3533 copy); 3534 if ((len -= copy) == 0) 3535 return csum; 3536 offset += copy; 3537 to += copy; 3538 pos = copy; 3539 } 3540 3541 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3542 int end; 3543 3544 WARN_ON(start > offset + len); 3545 3546 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 3547 if ((copy = end - offset) > 0) { 3548 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3549 u32 p_off, p_len, copied; 3550 struct page *p; 3551 __wsum csum2; 3552 u8 *vaddr; 3553 3554 if (copy > len) 3555 copy = len; 3556 3557 skb_frag_foreach_page(frag, 3558 skb_frag_off(frag) + offset - start, 3559 copy, p, p_off, p_len, copied) { 3560 vaddr = kmap_atomic(p); 3561 csum2 = csum_partial_copy_nocheck(vaddr + p_off, 3562 to + copied, 3563 p_len); 3564 kunmap_atomic(vaddr); 3565 csum = csum_block_add(csum, csum2, pos); 3566 pos += p_len; 3567 } 3568 3569 if (!(len -= copy)) 3570 return csum; 3571 offset += copy; 3572 to += copy; 3573 } 3574 start = end; 3575 } 3576 3577 skb_walk_frags(skb, frag_iter) { 3578 __wsum csum2; 3579 int end; 3580 3581 WARN_ON(start > offset + len); 3582 3583 end = start + frag_iter->len; 3584 if ((copy = end - offset) > 0) { 3585 if (copy > len) 3586 copy = len; 3587 csum2 = skb_copy_and_csum_bits(frag_iter, 3588 offset - start, 3589 to, copy); 3590 csum = csum_block_add(csum, csum2, pos); 3591 if ((len -= copy) == 0) 3592 return csum; 3593 offset += copy; 3594 to += copy; 3595 pos += copy; 3596 } 3597 start = end; 3598 } 3599 BUG_ON(len); 3600 return csum; 3601} 3602EXPORT_SYMBOL(skb_copy_and_csum_bits); 3603 3604__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len) 3605{ 3606 __sum16 sum; 3607 3608 sum = csum_fold(skb_checksum(skb, 0, len, skb->csum)); 3609 /* See comments in __skb_checksum_complete(). */ 3610 if (likely(!sum)) { 3611 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 3612 !skb->csum_complete_sw) 3613 netdev_rx_csum_fault(skb->dev, skb); 3614 } 3615 if (!skb_shared(skb)) 3616 skb->csum_valid = !sum; 3617 return sum; 3618} 3619EXPORT_SYMBOL(__skb_checksum_complete_head); 3620 3621/* This function assumes skb->csum already holds pseudo header's checksum, 3622 * which has been changed from the hardware checksum, for example, by 3623 * __skb_checksum_validate_complete(). And, the original skb->csum must 3624 * have been validated unsuccessfully for CHECKSUM_COMPLETE case. 3625 * 3626 * It returns non-zero if the recomputed checksum is still invalid, otherwise 3627 * zero. The new checksum is stored back into skb->csum unless the skb is 3628 * shared. 3629 */ 3630__sum16 __skb_checksum_complete(struct sk_buff *skb) 3631{ 3632 __wsum csum; 3633 __sum16 sum; 3634 3635 csum = skb_checksum(skb, 0, skb->len, 0); 3636 3637 sum = csum_fold(csum_add(skb->csum, csum)); 3638 /* This check is inverted, because we already knew the hardware 3639 * checksum is invalid before calling this function. So, if the 3640 * re-computed checksum is valid instead, then we have a mismatch 3641 * between the original skb->csum and skb_checksum(). This means either 3642 * the original hardware checksum is incorrect or we screw up skb->csum 3643 * when moving skb->data around. 3644 */ 3645 if (likely(!sum)) { 3646 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 3647 !skb->csum_complete_sw) 3648 netdev_rx_csum_fault(skb->dev, skb); 3649 } 3650 3651 if (!skb_shared(skb)) { 3652 /* Save full packet checksum */ 3653 skb->csum = csum; 3654 skb->ip_summed = CHECKSUM_COMPLETE; 3655 skb->csum_complete_sw = 1; 3656 skb->csum_valid = !sum; 3657 } 3658 3659 return sum; 3660} 3661EXPORT_SYMBOL(__skb_checksum_complete); 3662 3663static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum) 3664{ 3665 net_warn_ratelimited( 3666 "%s: attempt to compute crc32c without libcrc32c.ko\n", 3667 __func__); 3668 return 0; 3669} 3670 3671static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2, 3672 int offset, int len) 3673{ 3674 net_warn_ratelimited( 3675 "%s: attempt to compute crc32c without libcrc32c.ko\n", 3676 __func__); 3677 return 0; 3678} 3679 3680static const struct skb_checksum_ops default_crc32c_ops = { 3681 .update = warn_crc32c_csum_update, 3682 .combine = warn_crc32c_csum_combine, 3683}; 3684 3685const struct skb_checksum_ops *crc32c_csum_stub __read_mostly = 3686 &default_crc32c_ops; 3687EXPORT_SYMBOL(crc32c_csum_stub); 3688 3689 /** 3690 * skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy() 3691 * @from: source buffer 3692 * 3693 * Calculates the amount of linear headroom needed in the 'to' skb passed 3694 * into skb_zerocopy(). 3695 */ 3696unsigned int 3697skb_zerocopy_headlen(const struct sk_buff *from) 3698{ 3699 unsigned int hlen = 0; 3700 3701 if (!from->head_frag || 3702 skb_headlen(from) < L1_CACHE_BYTES || 3703 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) { 3704 hlen = skb_headlen(from); 3705 if (!hlen) 3706 hlen = from->len; 3707 } 3708 3709 if (skb_has_frag_list(from)) 3710 hlen = from->len; 3711 3712 return hlen; 3713} 3714EXPORT_SYMBOL_GPL(skb_zerocopy_headlen); 3715 3716/** 3717 * skb_zerocopy - Zero copy skb to skb 3718 * @to: destination buffer 3719 * @from: source buffer 3720 * @len: number of bytes to copy from source buffer 3721 * @hlen: size of linear headroom in destination buffer 3722 * 3723 * Copies up to `len` bytes from `from` to `to` by creating references 3724 * to the frags in the source buffer. 3725 * 3726 * The `hlen` as calculated by skb_zerocopy_headlen() specifies the 3727 * headroom in the `to` buffer. 3728 * 3729 * Return value: 3730 * 0: everything is OK 3731 * -ENOMEM: couldn't orphan frags of @from due to lack of memory 3732 * -EFAULT: skb_copy_bits() found some problem with skb geometry 3733 */ 3734int 3735skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) 3736{ 3737 int i, j = 0; 3738 int plen = 0; /* length of skb->head fragment */ 3739 int ret; 3740 struct page *page; 3741 unsigned int offset; 3742 3743 BUG_ON(!from->head_frag && !hlen); 3744 3745 /* dont bother with small payloads */ 3746 if (len <= skb_tailroom(to)) 3747 return skb_copy_bits(from, 0, skb_put(to, len), len); 3748 3749 if (hlen) { 3750 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen); 3751 if (unlikely(ret)) 3752 return ret; 3753 len -= hlen; 3754 } else { 3755 plen = min_t(int, skb_headlen(from), len); 3756 if (plen) { 3757 page = virt_to_head_page(from->head); 3758 offset = from->data - (unsigned char *)page_address(page); 3759 __skb_fill_netmem_desc(to, 0, page_to_netmem(page), 3760 offset, plen); 3761 get_page(page); 3762 j = 1; 3763 len -= plen; 3764 } 3765 } 3766 3767 skb_len_add(to, len + plen); 3768 3769 if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) { 3770 skb_tx_error(from); 3771 return -ENOMEM; 3772 } 3773 skb_zerocopy_clone(to, from, GFP_ATOMIC); 3774 3775 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) { 3776 int size; 3777 3778 if (!len) 3779 break; 3780 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i]; 3781 size = min_t(int, skb_frag_size(&skb_shinfo(to)->frags[j]), 3782 len); 3783 skb_frag_size_set(&skb_shinfo(to)->frags[j], size); 3784 len -= size; 3785 skb_frag_ref(to, j); 3786 j++; 3787 } 3788 skb_shinfo(to)->nr_frags = j; 3789 3790 return 0; 3791} 3792EXPORT_SYMBOL_GPL(skb_zerocopy); 3793 3794void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to) 3795{ 3796 __wsum csum; 3797 long csstart; 3798 3799 if (skb->ip_summed == CHECKSUM_PARTIAL) 3800 csstart = skb_checksum_start_offset(skb); 3801 else 3802 csstart = skb_headlen(skb); 3803 3804 BUG_ON(csstart > skb_headlen(skb)); 3805 3806 skb_copy_from_linear_data(skb, to, csstart); 3807 3808 csum = 0; 3809 if (csstart != skb->len) 3810 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart, 3811 skb->len - csstart); 3812 3813 if (skb->ip_summed == CHECKSUM_PARTIAL) { 3814 long csstuff = csstart + skb->csum_offset; 3815 3816 *((__sum16 *)(to + csstuff)) = csum_fold(csum); 3817 } 3818} 3819EXPORT_SYMBOL(skb_copy_and_csum_dev); 3820 3821/** 3822 * skb_dequeue - remove from the head of the queue 3823 * @list: list to dequeue from 3824 * 3825 * Remove the head of the list. The list lock is taken so the function 3826 * may be used safely with other locking list functions. The head item is 3827 * returned or %NULL if the list is empty. 3828 */ 3829 3830struct sk_buff *skb_dequeue(struct sk_buff_head *list) 3831{ 3832 unsigned long flags; 3833 struct sk_buff *result; 3834 3835 spin_lock_irqsave(&list->lock, flags); 3836 result = __skb_dequeue(list); 3837 spin_unlock_irqrestore(&list->lock, flags); 3838 return result; 3839} 3840EXPORT_SYMBOL(skb_dequeue); 3841 3842/** 3843 * skb_dequeue_tail - remove from the tail of the queue 3844 * @list: list to dequeue from 3845 * 3846 * Remove the tail of the list. The list lock is taken so the function 3847 * may be used safely with other locking list functions. The tail item is 3848 * returned or %NULL if the list is empty. 3849 */ 3850struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list) 3851{ 3852 unsigned long flags; 3853 struct sk_buff *result; 3854 3855 spin_lock_irqsave(&list->lock, flags); 3856 result = __skb_dequeue_tail(list); 3857 spin_unlock_irqrestore(&list->lock, flags); 3858 return result; 3859} 3860EXPORT_SYMBOL(skb_dequeue_tail); 3861 3862/** 3863 * skb_queue_purge_reason - empty a list 3864 * @list: list to empty 3865 * @reason: drop reason 3866 * 3867 * Delete all buffers on an &sk_buff list. Each buffer is removed from 3868 * the list and one reference dropped. This function takes the list 3869 * lock and is atomic with respect to other list locking functions. 3870 */ 3871void skb_queue_purge_reason(struct sk_buff_head *list, 3872 enum skb_drop_reason reason) 3873{ 3874 struct sk_buff_head tmp; 3875 unsigned long flags; 3876 3877 if (skb_queue_empty_lockless(list)) 3878 return; 3879 3880 __skb_queue_head_init(&tmp); 3881 3882 spin_lock_irqsave(&list->lock, flags); 3883 skb_queue_splice_init(list, &tmp); 3884 spin_unlock_irqrestore(&list->lock, flags); 3885 3886 __skb_queue_purge_reason(&tmp, reason); 3887} 3888EXPORT_SYMBOL(skb_queue_purge_reason); 3889 3890/** 3891 * skb_rbtree_purge - empty a skb rbtree 3892 * @root: root of the rbtree to empty 3893 * Return value: the sum of truesizes of all purged skbs. 3894 * 3895 * Delete all buffers on an &sk_buff rbtree. Each buffer is removed from 3896 * the list and one reference dropped. This function does not take 3897 * any lock. Synchronization should be handled by the caller (e.g., TCP 3898 * out-of-order queue is protected by the socket lock). 3899 */ 3900unsigned int skb_rbtree_purge(struct rb_root *root) 3901{ 3902 struct rb_node *p = rb_first(root); 3903 unsigned int sum = 0; 3904 3905 while (p) { 3906 struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode); 3907 3908 p = rb_next(p); 3909 rb_erase(&skb->rbnode, root); 3910 sum += skb->truesize; 3911 kfree_skb(skb); 3912 } 3913 return sum; 3914} 3915 3916void skb_errqueue_purge(struct sk_buff_head *list) 3917{ 3918 struct sk_buff *skb, *next; 3919 struct sk_buff_head kill; 3920 unsigned long flags; 3921 3922 __skb_queue_head_init(&kill); 3923 3924 spin_lock_irqsave(&list->lock, flags); 3925 skb_queue_walk_safe(list, skb, next) { 3926 if (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ZEROCOPY || 3927 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_TIMESTAMPING) 3928 continue; 3929 __skb_unlink(skb, list); 3930 __skb_queue_tail(&kill, skb); 3931 } 3932 spin_unlock_irqrestore(&list->lock, flags); 3933 __skb_queue_purge(&kill); 3934} 3935EXPORT_SYMBOL(skb_errqueue_purge); 3936 3937/** 3938 * skb_queue_head - queue a buffer at the list head 3939 * @list: list to use 3940 * @newsk: buffer to queue 3941 * 3942 * Queue a buffer at the start of the list. This function takes the 3943 * list lock and can be used safely with other locking &sk_buff functions 3944 * safely. 3945 * 3946 * A buffer cannot be placed on two lists at the same time. 3947 */ 3948void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk) 3949{ 3950 unsigned long flags; 3951 3952 spin_lock_irqsave(&list->lock, flags); 3953 __skb_queue_head(list, newsk); 3954 spin_unlock_irqrestore(&list->lock, flags); 3955} 3956EXPORT_SYMBOL(skb_queue_head); 3957 3958/** 3959 * skb_queue_tail - queue a buffer at the list tail 3960 * @list: list to use 3961 * @newsk: buffer to queue 3962 * 3963 * Queue a buffer at the tail of the list. This function takes the 3964 * list lock and can be used safely with other locking &sk_buff functions 3965 * safely. 3966 * 3967 * A buffer cannot be placed on two lists at the same time. 3968 */ 3969void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk) 3970{ 3971 unsigned long flags; 3972 3973 spin_lock_irqsave(&list->lock, flags); 3974 __skb_queue_tail(list, newsk); 3975 spin_unlock_irqrestore(&list->lock, flags); 3976} 3977EXPORT_SYMBOL(skb_queue_tail); 3978 3979/** 3980 * skb_unlink - remove a buffer from a list 3981 * @skb: buffer to remove 3982 * @list: list to use 3983 * 3984 * Remove a packet from a list. The list locks are taken and this 3985 * function is atomic with respect to other list locked calls 3986 * 3987 * You must know what list the SKB is on. 3988 */ 3989void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) 3990{ 3991 unsigned long flags; 3992 3993 spin_lock_irqsave(&list->lock, flags); 3994 __skb_unlink(skb, list); 3995 spin_unlock_irqrestore(&list->lock, flags); 3996} 3997EXPORT_SYMBOL(skb_unlink); 3998 3999/** 4000 * skb_append - append a buffer 4001 * @old: buffer to insert after 4002 * @newsk: buffer to insert 4003 * @list: list to use 4004 * 4005 * Place a packet after a given packet in a list. The list locks are taken 4006 * and this function is atomic with respect to other list locked calls. 4007 * A buffer cannot be placed on two lists at the same time. 4008 */ 4009void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list) 4010{ 4011 unsigned long flags; 4012 4013 spin_lock_irqsave(&list->lock, flags); 4014 __skb_queue_after(list, old, newsk); 4015 spin_unlock_irqrestore(&list->lock, flags); 4016} 4017EXPORT_SYMBOL(skb_append); 4018 4019static inline void skb_split_inside_header(struct sk_buff *skb, 4020 struct sk_buff* skb1, 4021 const u32 len, const int pos) 4022{ 4023 int i; 4024 4025 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len), 4026 pos - len); 4027 /* And move data appendix as is. */ 4028 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 4029 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i]; 4030 4031 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags; 4032 skb_shinfo(skb)->nr_frags = 0; 4033 skb1->data_len = skb->data_len; 4034 skb1->len += skb1->data_len; 4035 skb->data_len = 0; 4036 skb->len = len; 4037 skb_set_tail_pointer(skb, len); 4038} 4039 4040static inline void skb_split_no_header(struct sk_buff *skb, 4041 struct sk_buff* skb1, 4042 const u32 len, int pos) 4043{ 4044 int i, k = 0; 4045 const int nfrags = skb_shinfo(skb)->nr_frags; 4046 4047 skb_shinfo(skb)->nr_frags = 0; 4048 skb1->len = skb1->data_len = skb->len - len; 4049 skb->len = len; 4050 skb->data_len = len - pos; 4051 4052 for (i = 0; i < nfrags; i++) { 4053 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 4054 4055 if (pos + size > len) { 4056 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i]; 4057 4058 if (pos < len) { 4059 /* Split frag. 4060 * We have two variants in this case: 4061 * 1. Move all the frag to the second 4062 * part, if it is possible. F.e. 4063 * this approach is mandatory for TUX, 4064 * where splitting is expensive. 4065 * 2. Split is accurately. We make this. 4066 */ 4067 skb_frag_ref(skb, i); 4068 skb_frag_off_add(&skb_shinfo(skb1)->frags[0], len - pos); 4069 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos); 4070 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos); 4071 skb_shinfo(skb)->nr_frags++; 4072 } 4073 k++; 4074 } else 4075 skb_shinfo(skb)->nr_frags++; 4076 pos += size; 4077 } 4078 skb_shinfo(skb1)->nr_frags = k; 4079} 4080 4081/** 4082 * skb_split - Split fragmented skb to two parts at length len. 4083 * @skb: the buffer to split 4084 * @skb1: the buffer to receive the second part 4085 * @len: new length for skb 4086 */ 4087void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len) 4088{ 4089 int pos = skb_headlen(skb); 4090 const int zc_flags = SKBFL_SHARED_FRAG | SKBFL_PURE_ZEROCOPY; 4091 4092 skb_zcopy_downgrade_managed(skb); 4093 4094 skb_shinfo(skb1)->flags |= skb_shinfo(skb)->flags & zc_flags; 4095 skb_zerocopy_clone(skb1, skb, 0); 4096 if (len < pos) /* Split line is inside header. */ 4097 skb_split_inside_header(skb, skb1, len, pos); 4098 else /* Second chunk has no header, nothing to copy. */ 4099 skb_split_no_header(skb, skb1, len, pos); 4100} 4101EXPORT_SYMBOL(skb_split); 4102 4103/* Shifting from/to a cloned skb is a no-go. 4104 * 4105 * Caller cannot keep skb_shinfo related pointers past calling here! 4106 */ 4107static int skb_prepare_for_shift(struct sk_buff *skb) 4108{ 4109 return skb_unclone_keeptruesize(skb, GFP_ATOMIC); 4110} 4111 4112/** 4113 * skb_shift - Shifts paged data partially from skb to another 4114 * @tgt: buffer into which tail data gets added 4115 * @skb: buffer from which the paged data comes from 4116 * @shiftlen: shift up to this many bytes 4117 * 4118 * Attempts to shift up to shiftlen worth of bytes, which may be less than 4119 * the length of the skb, from skb to tgt. Returns number bytes shifted. 4120 * It's up to caller to free skb if everything was shifted. 4121 * 4122 * If @tgt runs out of frags, the whole operation is aborted. 4123 * 4124 * Skb cannot include anything else but paged data while tgt is allowed 4125 * to have non-paged data as well. 4126 * 4127 * TODO: full sized shift could be optimized but that would need 4128 * specialized skb free'er to handle frags without up-to-date nr_frags. 4129 */ 4130int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) 4131{ 4132 int from, to, merge, todo; 4133 skb_frag_t *fragfrom, *fragto; 4134 4135 BUG_ON(shiftlen > skb->len); 4136 4137 if (skb_headlen(skb)) 4138 return 0; 4139 if (skb_zcopy(tgt) || skb_zcopy(skb)) 4140 return 0; 4141 4142 todo = shiftlen; 4143 from = 0; 4144 to = skb_shinfo(tgt)->nr_frags; 4145 fragfrom = &skb_shinfo(skb)->frags[from]; 4146 4147 /* Actual merge is delayed until the point when we know we can 4148 * commit all, so that we don't have to undo partial changes 4149 */ 4150 if (!to || 4151 !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom), 4152 skb_frag_off(fragfrom))) { 4153 merge = -1; 4154 } else { 4155 merge = to - 1; 4156 4157 todo -= skb_frag_size(fragfrom); 4158 if (todo < 0) { 4159 if (skb_prepare_for_shift(skb) || 4160 skb_prepare_for_shift(tgt)) 4161 return 0; 4162 4163 /* All previous frag pointers might be stale! */ 4164 fragfrom = &skb_shinfo(skb)->frags[from]; 4165 fragto = &skb_shinfo(tgt)->frags[merge]; 4166 4167 skb_frag_size_add(fragto, shiftlen); 4168 skb_frag_size_sub(fragfrom, shiftlen); 4169 skb_frag_off_add(fragfrom, shiftlen); 4170 4171 goto onlymerged; 4172 } 4173 4174 from++; 4175 } 4176 4177 /* Skip full, not-fitting skb to avoid expensive operations */ 4178 if ((shiftlen == skb->len) && 4179 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to)) 4180 return 0; 4181 4182 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt)) 4183 return 0; 4184 4185 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) { 4186 if (to == MAX_SKB_FRAGS) 4187 return 0; 4188 4189 fragfrom = &skb_shinfo(skb)->frags[from]; 4190 fragto = &skb_shinfo(tgt)->frags[to]; 4191 4192 if (todo >= skb_frag_size(fragfrom)) { 4193 *fragto = *fragfrom; 4194 todo -= skb_frag_size(fragfrom); 4195 from++; 4196 to++; 4197 4198 } else { 4199 __skb_frag_ref(fragfrom); 4200 skb_frag_page_copy(fragto, fragfrom); 4201 skb_frag_off_copy(fragto, fragfrom); 4202 skb_frag_size_set(fragto, todo); 4203 4204 skb_frag_off_add(fragfrom, todo); 4205 skb_frag_size_sub(fragfrom, todo); 4206 todo = 0; 4207 4208 to++; 4209 break; 4210 } 4211 } 4212 4213 /* Ready to "commit" this state change to tgt */ 4214 skb_shinfo(tgt)->nr_frags = to; 4215 4216 if (merge >= 0) { 4217 fragfrom = &skb_shinfo(skb)->frags[0]; 4218 fragto = &skb_shinfo(tgt)->frags[merge]; 4219 4220 skb_frag_size_add(fragto, skb_frag_size(fragfrom)); 4221 __skb_frag_unref(fragfrom, skb->pp_recycle); 4222 } 4223 4224 /* Reposition in the original skb */ 4225 to = 0; 4226 while (from < skb_shinfo(skb)->nr_frags) 4227 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++]; 4228 skb_shinfo(skb)->nr_frags = to; 4229 4230 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags); 4231 4232onlymerged: 4233 /* Most likely the tgt won't ever need its checksum anymore, skb on 4234 * the other hand might need it if it needs to be resent 4235 */ 4236 tgt->ip_summed = CHECKSUM_PARTIAL; 4237 skb->ip_summed = CHECKSUM_PARTIAL; 4238 4239 skb_len_add(skb, -shiftlen); 4240 skb_len_add(tgt, shiftlen); 4241 4242 return shiftlen; 4243} 4244 4245/** 4246 * skb_prepare_seq_read - Prepare a sequential read of skb data 4247 * @skb: the buffer to read 4248 * @from: lower offset of data to be read 4249 * @to: upper offset of data to be read 4250 * @st: state variable 4251 * 4252 * Initializes the specified state variable. Must be called before 4253 * invoking skb_seq_read() for the first time. 4254 */ 4255void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from, 4256 unsigned int to, struct skb_seq_state *st) 4257{ 4258 st->lower_offset = from; 4259 st->upper_offset = to; 4260 st->root_skb = st->cur_skb = skb; 4261 st->frag_idx = st->stepped_offset = 0; 4262 st->frag_data = NULL; 4263 st->frag_off = 0; 4264} 4265EXPORT_SYMBOL(skb_prepare_seq_read); 4266 4267/** 4268 * skb_seq_read - Sequentially read skb data 4269 * @consumed: number of bytes consumed by the caller so far 4270 * @data: destination pointer for data to be returned 4271 * @st: state variable 4272 * 4273 * Reads a block of skb data at @consumed relative to the 4274 * lower offset specified to skb_prepare_seq_read(). Assigns 4275 * the head of the data block to @data and returns the length 4276 * of the block or 0 if the end of the skb data or the upper 4277 * offset has been reached. 4278 * 4279 * The caller is not required to consume all of the data 4280 * returned, i.e. @consumed is typically set to the number 4281 * of bytes already consumed and the next call to 4282 * skb_seq_read() will return the remaining part of the block. 4283 * 4284 * Note 1: The size of each block of data returned can be arbitrary, 4285 * this limitation is the cost for zerocopy sequential 4286 * reads of potentially non linear data. 4287 * 4288 * Note 2: Fragment lists within fragments are not implemented 4289 * at the moment, state->root_skb could be replaced with 4290 * a stack for this purpose. 4291 */ 4292unsigned int skb_seq_read(unsigned int consumed, const u8 **data, 4293 struct skb_seq_state *st) 4294{ 4295 unsigned int block_limit, abs_offset = consumed + st->lower_offset; 4296 skb_frag_t *frag; 4297 4298 if (unlikely(abs_offset >= st->upper_offset)) { 4299 if (st->frag_data) { 4300 kunmap_atomic(st->frag_data); 4301 st->frag_data = NULL; 4302 } 4303 return 0; 4304 } 4305 4306next_skb: 4307 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset; 4308 4309 if (abs_offset < block_limit && !st->frag_data) { 4310 *data = st->cur_skb->data + (abs_offset - st->stepped_offset); 4311 return block_limit - abs_offset; 4312 } 4313 4314 if (st->frag_idx == 0 && !st->frag_data) 4315 st->stepped_offset += skb_headlen(st->cur_skb); 4316 4317 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) { 4318 unsigned int pg_idx, pg_off, pg_sz; 4319 4320 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx]; 4321 4322 pg_idx = 0; 4323 pg_off = skb_frag_off(frag); 4324 pg_sz = skb_frag_size(frag); 4325 4326 if (skb_frag_must_loop(skb_frag_page(frag))) { 4327 pg_idx = (pg_off + st->frag_off) >> PAGE_SHIFT; 4328 pg_off = offset_in_page(pg_off + st->frag_off); 4329 pg_sz = min_t(unsigned int, pg_sz - st->frag_off, 4330 PAGE_SIZE - pg_off); 4331 } 4332 4333 block_limit = pg_sz + st->stepped_offset; 4334 if (abs_offset < block_limit) { 4335 if (!st->frag_data) 4336 st->frag_data = kmap_atomic(skb_frag_page(frag) + pg_idx); 4337 4338 *data = (u8 *)st->frag_data + pg_off + 4339 (abs_offset - st->stepped_offset); 4340 4341 return block_limit - abs_offset; 4342 } 4343 4344 if (st->frag_data) { 4345 kunmap_atomic(st->frag_data); 4346 st->frag_data = NULL; 4347 } 4348 4349 st->stepped_offset += pg_sz; 4350 st->frag_off += pg_sz; 4351 if (st->frag_off == skb_frag_size(frag)) { 4352 st->frag_off = 0; 4353 st->frag_idx++; 4354 } 4355 } 4356 4357 if (st->frag_data) { 4358 kunmap_atomic(st->frag_data); 4359 st->frag_data = NULL; 4360 } 4361 4362 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) { 4363 st->cur_skb = skb_shinfo(st->root_skb)->frag_list; 4364 st->frag_idx = 0; 4365 goto next_skb; 4366 } else if (st->cur_skb->next) { 4367 st->cur_skb = st->cur_skb->next; 4368 st->frag_idx = 0; 4369 goto next_skb; 4370 } 4371 4372 return 0; 4373} 4374EXPORT_SYMBOL(skb_seq_read); 4375 4376/** 4377 * skb_abort_seq_read - Abort a sequential read of skb data 4378 * @st: state variable 4379 * 4380 * Must be called if skb_seq_read() was not called until it 4381 * returned 0. 4382 */ 4383void skb_abort_seq_read(struct skb_seq_state *st) 4384{ 4385 if (st->frag_data) 4386 kunmap_atomic(st->frag_data); 4387} 4388EXPORT_SYMBOL(skb_abort_seq_read); 4389 4390#define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb)) 4391 4392static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text, 4393 struct ts_config *conf, 4394 struct ts_state *state) 4395{ 4396 return skb_seq_read(offset, text, TS_SKB_CB(state)); 4397} 4398 4399static void skb_ts_finish(struct ts_config *conf, struct ts_state *state) 4400{ 4401 skb_abort_seq_read(TS_SKB_CB(state)); 4402} 4403 4404/** 4405 * skb_find_text - Find a text pattern in skb data 4406 * @skb: the buffer to look in 4407 * @from: search offset 4408 * @to: search limit 4409 * @config: textsearch configuration 4410 * 4411 * Finds a pattern in the skb data according to the specified 4412 * textsearch configuration. Use textsearch_next() to retrieve 4413 * subsequent occurrences of the pattern. Returns the offset 4414 * to the first occurrence or UINT_MAX if no match was found. 4415 */ 4416unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, 4417 unsigned int to, struct ts_config *config) 4418{ 4419 unsigned int patlen = config->ops->get_pattern_len(config); 4420 struct ts_state state; 4421 unsigned int ret; 4422 4423 BUILD_BUG_ON(sizeof(struct skb_seq_state) > sizeof(state.cb)); 4424 4425 config->get_next_block = skb_ts_get_next_block; 4426 config->finish = skb_ts_finish; 4427 4428 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state)); 4429 4430 ret = textsearch_find(config, &state); 4431 return (ret + patlen <= to - from ? ret : UINT_MAX); 4432} 4433EXPORT_SYMBOL(skb_find_text); 4434 4435int skb_append_pagefrags(struct sk_buff *skb, struct page *page, 4436 int offset, size_t size, size_t max_frags) 4437{ 4438 int i = skb_shinfo(skb)->nr_frags; 4439 4440 if (skb_can_coalesce(skb, i, page, offset)) { 4441 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size); 4442 } else if (i < max_frags) { 4443 skb_zcopy_downgrade_managed(skb); 4444 get_page(page); 4445 skb_fill_page_desc_noacc(skb, i, page, offset, size); 4446 } else { 4447 return -EMSGSIZE; 4448 } 4449 4450 return 0; 4451} 4452EXPORT_SYMBOL_GPL(skb_append_pagefrags); 4453 4454/** 4455 * skb_pull_rcsum - pull skb and update receive checksum 4456 * @skb: buffer to update 4457 * @len: length of data pulled 4458 * 4459 * This function performs an skb_pull on the packet and updates 4460 * the CHECKSUM_COMPLETE checksum. It should be used on 4461 * receive path processing instead of skb_pull unless you know 4462 * that the checksum difference is zero (e.g., a valid IP header) 4463 * or you are setting ip_summed to CHECKSUM_NONE. 4464 */ 4465void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len) 4466{ 4467 unsigned char *data = skb->data; 4468 4469 BUG_ON(len > skb->len); 4470 __skb_pull(skb, len); 4471 skb_postpull_rcsum(skb, data, len); 4472 return skb->data; 4473} 4474EXPORT_SYMBOL_GPL(skb_pull_rcsum); 4475 4476static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb) 4477{ 4478 skb_frag_t head_frag; 4479 struct page *page; 4480 4481 page = virt_to_head_page(frag_skb->head); 4482 skb_frag_fill_page_desc(&head_frag, page, frag_skb->data - 4483 (unsigned char *)page_address(page), 4484 skb_headlen(frag_skb)); 4485 return head_frag; 4486} 4487 4488struct sk_buff *skb_segment_list(struct sk_buff *skb, 4489 netdev_features_t features, 4490 unsigned int offset) 4491{ 4492 struct sk_buff *list_skb = skb_shinfo(skb)->frag_list; 4493 unsigned int tnl_hlen = skb_tnl_header_len(skb); 4494 unsigned int delta_truesize = 0; 4495 unsigned int delta_len = 0; 4496 struct sk_buff *tail = NULL; 4497 struct sk_buff *nskb, *tmp; 4498 int len_diff, err; 4499 4500 skb_push(skb, -skb_network_offset(skb) + offset); 4501 4502 /* Ensure the head is writeable before touching the shared info */ 4503 err = skb_unclone(skb, GFP_ATOMIC); 4504 if (err) 4505 goto err_linearize; 4506 4507 skb_shinfo(skb)->frag_list = NULL; 4508 4509 while (list_skb) { 4510 nskb = list_skb; 4511 list_skb = list_skb->next; 4512 4513 err = 0; 4514 delta_truesize += nskb->truesize; 4515 if (skb_shared(nskb)) { 4516 tmp = skb_clone(nskb, GFP_ATOMIC); 4517 if (tmp) { 4518 consume_skb(nskb); 4519 nskb = tmp; 4520 err = skb_unclone(nskb, GFP_ATOMIC); 4521 } else { 4522 err = -ENOMEM; 4523 } 4524 } 4525 4526 if (!tail) 4527 skb->next = nskb; 4528 else 4529 tail->next = nskb; 4530 4531 if (unlikely(err)) { 4532 nskb->next = list_skb; 4533 goto err_linearize; 4534 } 4535 4536 tail = nskb; 4537 4538 delta_len += nskb->len; 4539 4540 skb_push(nskb, -skb_network_offset(nskb) + offset); 4541 4542 skb_release_head_state(nskb); 4543 len_diff = skb_network_header_len(nskb) - skb_network_header_len(skb); 4544 __copy_skb_header(nskb, skb); 4545 4546 skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb)); 4547 nskb->transport_header += len_diff; 4548 skb_copy_from_linear_data_offset(skb, -tnl_hlen, 4549 nskb->data - tnl_hlen, 4550 offset + tnl_hlen); 4551 4552 if (skb_needs_linearize(nskb, features) && 4553 __skb_linearize(nskb)) 4554 goto err_linearize; 4555 } 4556 4557 skb->truesize = skb->truesize - delta_truesize; 4558 skb->data_len = skb->data_len - delta_len; 4559 skb->len = skb->len - delta_len; 4560 4561 skb_gso_reset(skb); 4562 4563 skb->prev = tail; 4564 4565 if (skb_needs_linearize(skb, features) && 4566 __skb_linearize(skb)) 4567 goto err_linearize; 4568 4569 skb_get(skb); 4570 4571 return skb; 4572 4573err_linearize: 4574 kfree_skb_list(skb->next); 4575 skb->next = NULL; 4576 return ERR_PTR(-ENOMEM); 4577} 4578EXPORT_SYMBOL_GPL(skb_segment_list); 4579 4580/** 4581 * skb_segment - Perform protocol segmentation on skb. 4582 * @head_skb: buffer to segment 4583 * @features: features for the output path (see dev->features) 4584 * 4585 * This function performs segmentation on the given skb. It returns 4586 * a pointer to the first in a list of new skbs for the segments. 4587 * In case of error it returns ERR_PTR(err). 4588 */ 4589struct sk_buff *skb_segment(struct sk_buff *head_skb, 4590 netdev_features_t features) 4591{ 4592 struct sk_buff *segs = NULL; 4593 struct sk_buff *tail = NULL; 4594 struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list; 4595 unsigned int mss = skb_shinfo(head_skb)->gso_size; 4596 unsigned int doffset = head_skb->data - skb_mac_header(head_skb); 4597 unsigned int offset = doffset; 4598 unsigned int tnl_hlen = skb_tnl_header_len(head_skb); 4599 unsigned int partial_segs = 0; 4600 unsigned int headroom; 4601 unsigned int len = head_skb->len; 4602 struct sk_buff *frag_skb; 4603 skb_frag_t *frag; 4604 __be16 proto; 4605 bool csum, sg; 4606 int err = -ENOMEM; 4607 int i = 0; 4608 int nfrags, pos; 4609 4610 if ((skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY) && 4611 mss != GSO_BY_FRAGS && mss != skb_headlen(head_skb)) { 4612 struct sk_buff *check_skb; 4613 4614 for (check_skb = list_skb; check_skb; check_skb = check_skb->next) { 4615 if (skb_headlen(check_skb) && !check_skb->head_frag) { 4616 /* gso_size is untrusted, and we have a frag_list with 4617 * a linear non head_frag item. 4618 * 4619 * If head_skb's headlen does not fit requested gso_size, 4620 * it means that the frag_list members do NOT terminate 4621 * on exact gso_size boundaries. Hence we cannot perform 4622 * skb_frag_t page sharing. Therefore we must fallback to 4623 * copying the frag_list skbs; we do so by disabling SG. 4624 */ 4625 features &= ~NETIF_F_SG; 4626 break; 4627 } 4628 } 4629 } 4630 4631 __skb_push(head_skb, doffset); 4632 proto = skb_network_protocol(head_skb, NULL); 4633 if (unlikely(!proto)) 4634 return ERR_PTR(-EINVAL); 4635 4636 sg = !!(features & NETIF_F_SG); 4637 csum = !!can_checksum_protocol(features, proto); 4638 4639 if (sg && csum && (mss != GSO_BY_FRAGS)) { 4640 if (!(features & NETIF_F_GSO_PARTIAL)) { 4641 struct sk_buff *iter; 4642 unsigned int frag_len; 4643 4644 if (!list_skb || 4645 !net_gso_ok(features, skb_shinfo(head_skb)->gso_type)) 4646 goto normal; 4647 4648 /* If we get here then all the required 4649 * GSO features except frag_list are supported. 4650 * Try to split the SKB to multiple GSO SKBs 4651 * with no frag_list. 4652 * Currently we can do that only when the buffers don't 4653 * have a linear part and all the buffers except 4654 * the last are of the same length. 4655 */ 4656 frag_len = list_skb->len; 4657 skb_walk_frags(head_skb, iter) { 4658 if (frag_len != iter->len && iter->next) 4659 goto normal; 4660 if (skb_headlen(iter) && !iter->head_frag) 4661 goto normal; 4662 4663 len -= iter->len; 4664 } 4665 4666 if (len != frag_len) 4667 goto normal; 4668 } 4669 4670 /* GSO partial only requires that we trim off any excess that 4671 * doesn't fit into an MSS sized block, so take care of that 4672 * now. 4673 * Cap len to not accidentally hit GSO_BY_FRAGS. 4674 */ 4675 partial_segs = min(len, GSO_BY_FRAGS - 1) / mss; 4676 if (partial_segs > 1) 4677 mss *= partial_segs; 4678 else 4679 partial_segs = 0; 4680 } 4681 4682normal: 4683 headroom = skb_headroom(head_skb); 4684 pos = skb_headlen(head_skb); 4685 4686 if (skb_orphan_frags(head_skb, GFP_ATOMIC)) 4687 return ERR_PTR(-ENOMEM); 4688 4689 nfrags = skb_shinfo(head_skb)->nr_frags; 4690 frag = skb_shinfo(head_skb)->frags; 4691 frag_skb = head_skb; 4692 4693 do { 4694 struct sk_buff *nskb; 4695 skb_frag_t *nskb_frag; 4696 int hsize; 4697 int size; 4698 4699 if (unlikely(mss == GSO_BY_FRAGS)) { 4700 len = list_skb->len; 4701 } else { 4702 len = head_skb->len - offset; 4703 if (len > mss) 4704 len = mss; 4705 } 4706 4707 hsize = skb_headlen(head_skb) - offset; 4708 4709 if (hsize <= 0 && i >= nfrags && skb_headlen(list_skb) && 4710 (skb_headlen(list_skb) == len || sg)) { 4711 BUG_ON(skb_headlen(list_skb) > len); 4712 4713 nskb = skb_clone(list_skb, GFP_ATOMIC); 4714 if (unlikely(!nskb)) 4715 goto err; 4716 4717 i = 0; 4718 nfrags = skb_shinfo(list_skb)->nr_frags; 4719 frag = skb_shinfo(list_skb)->frags; 4720 frag_skb = list_skb; 4721 pos += skb_headlen(list_skb); 4722 4723 while (pos < offset + len) { 4724 BUG_ON(i >= nfrags); 4725 4726 size = skb_frag_size(frag); 4727 if (pos + size > offset + len) 4728 break; 4729 4730 i++; 4731 pos += size; 4732 frag++; 4733 } 4734 4735 list_skb = list_skb->next; 4736 4737 if (unlikely(pskb_trim(nskb, len))) { 4738 kfree_skb(nskb); 4739 goto err; 4740 } 4741 4742 hsize = skb_end_offset(nskb); 4743 if (skb_cow_head(nskb, doffset + headroom)) { 4744 kfree_skb(nskb); 4745 goto err; 4746 } 4747 4748 nskb->truesize += skb_end_offset(nskb) - hsize; 4749 skb_release_head_state(nskb); 4750 __skb_push(nskb, doffset); 4751 } else { 4752 if (hsize < 0) 4753 hsize = 0; 4754 if (hsize > len || !sg) 4755 hsize = len; 4756 4757 nskb = __alloc_skb(hsize + doffset + headroom, 4758 GFP_ATOMIC, skb_alloc_rx_flag(head_skb), 4759 NUMA_NO_NODE); 4760 4761 if (unlikely(!nskb)) 4762 goto err; 4763 4764 skb_reserve(nskb, headroom); 4765 __skb_put(nskb, doffset); 4766 } 4767 4768 if (segs) 4769 tail->next = nskb; 4770 else 4771 segs = nskb; 4772 tail = nskb; 4773 4774 __copy_skb_header(nskb, head_skb); 4775 4776 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom); 4777 skb_reset_mac_len(nskb); 4778 4779 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen, 4780 nskb->data - tnl_hlen, 4781 doffset + tnl_hlen); 4782 4783 if (nskb->len == len + doffset) 4784 goto perform_csum_check; 4785 4786 if (!sg) { 4787 if (!csum) { 4788 if (!nskb->remcsum_offload) 4789 nskb->ip_summed = CHECKSUM_NONE; 4790 SKB_GSO_CB(nskb)->csum = 4791 skb_copy_and_csum_bits(head_skb, offset, 4792 skb_put(nskb, 4793 len), 4794 len); 4795 SKB_GSO_CB(nskb)->csum_start = 4796 skb_headroom(nskb) + doffset; 4797 } else { 4798 if (skb_copy_bits(head_skb, offset, skb_put(nskb, len), len)) 4799 goto err; 4800 } 4801 continue; 4802 } 4803 4804 nskb_frag = skb_shinfo(nskb)->frags; 4805 4806 skb_copy_from_linear_data_offset(head_skb, offset, 4807 skb_put(nskb, hsize), hsize); 4808 4809 skb_shinfo(nskb)->flags |= skb_shinfo(head_skb)->flags & 4810 SKBFL_SHARED_FRAG; 4811 4812 if (skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC)) 4813 goto err; 4814 4815 while (pos < offset + len) { 4816 if (i >= nfrags) { 4817 if (skb_orphan_frags(list_skb, GFP_ATOMIC) || 4818 skb_zerocopy_clone(nskb, list_skb, 4819 GFP_ATOMIC)) 4820 goto err; 4821 4822 i = 0; 4823 nfrags = skb_shinfo(list_skb)->nr_frags; 4824 frag = skb_shinfo(list_skb)->frags; 4825 frag_skb = list_skb; 4826 if (!skb_headlen(list_skb)) { 4827 BUG_ON(!nfrags); 4828 } else { 4829 BUG_ON(!list_skb->head_frag); 4830 4831 /* to make room for head_frag. */ 4832 i--; 4833 frag--; 4834 } 4835 4836 list_skb = list_skb->next; 4837 } 4838 4839 if (unlikely(skb_shinfo(nskb)->nr_frags >= 4840 MAX_SKB_FRAGS)) { 4841 net_warn_ratelimited( 4842 "skb_segment: too many frags: %u %u\n", 4843 pos, mss); 4844 err = -EINVAL; 4845 goto err; 4846 } 4847 4848 *nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag; 4849 __skb_frag_ref(nskb_frag); 4850 size = skb_frag_size(nskb_frag); 4851 4852 if (pos < offset) { 4853 skb_frag_off_add(nskb_frag, offset - pos); 4854 skb_frag_size_sub(nskb_frag, offset - pos); 4855 } 4856 4857 skb_shinfo(nskb)->nr_frags++; 4858 4859 if (pos + size <= offset + len) { 4860 i++; 4861 frag++; 4862 pos += size; 4863 } else { 4864 skb_frag_size_sub(nskb_frag, pos + size - (offset + len)); 4865 goto skip_fraglist; 4866 } 4867 4868 nskb_frag++; 4869 } 4870 4871skip_fraglist: 4872 nskb->data_len = len - hsize; 4873 nskb->len += nskb->data_len; 4874 nskb->truesize += nskb->data_len; 4875 4876perform_csum_check: 4877 if (!csum) { 4878 if (skb_has_shared_frag(nskb) && 4879 __skb_linearize(nskb)) 4880 goto err; 4881 4882 if (!nskb->remcsum_offload) 4883 nskb->ip_summed = CHECKSUM_NONE; 4884 SKB_GSO_CB(nskb)->csum = 4885 skb_checksum(nskb, doffset, 4886 nskb->len - doffset, 0); 4887 SKB_GSO_CB(nskb)->csum_start = 4888 skb_headroom(nskb) + doffset; 4889 } 4890 } while ((offset += len) < head_skb->len); 4891 4892 /* Some callers want to get the end of the list. 4893 * Put it in segs->prev to avoid walking the list. 4894 * (see validate_xmit_skb_list() for example) 4895 */ 4896 segs->prev = tail; 4897 4898 if (partial_segs) { 4899 struct sk_buff *iter; 4900 int type = skb_shinfo(head_skb)->gso_type; 4901 unsigned short gso_size = skb_shinfo(head_skb)->gso_size; 4902 4903 /* Update type to add partial and then remove dodgy if set */ 4904 type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL; 4905 type &= ~SKB_GSO_DODGY; 4906 4907 /* Update GSO info and prepare to start updating headers on 4908 * our way back down the stack of protocols. 4909 */ 4910 for (iter = segs; iter; iter = iter->next) { 4911 skb_shinfo(iter)->gso_size = gso_size; 4912 skb_shinfo(iter)->gso_segs = partial_segs; 4913 skb_shinfo(iter)->gso_type = type; 4914 SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset; 4915 } 4916 4917 if (tail->len - doffset <= gso_size) 4918 skb_shinfo(tail)->gso_size = 0; 4919 else if (tail != segs) 4920 skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size); 4921 } 4922 4923 /* Following permits correct backpressure, for protocols 4924 * using skb_set_owner_w(). 4925 * Idea is to tranfert ownership from head_skb to last segment. 4926 */ 4927 if (head_skb->destructor == sock_wfree) { 4928 swap(tail->truesize, head_skb->truesize); 4929 swap(tail->destructor, head_skb->destructor); 4930 swap(tail->sk, head_skb->sk); 4931 } 4932 return segs; 4933 4934err: 4935 kfree_skb_list(segs); 4936 return ERR_PTR(err); 4937} 4938EXPORT_SYMBOL_GPL(skb_segment); 4939 4940#ifdef CONFIG_SKB_EXTENSIONS 4941#define SKB_EXT_ALIGN_VALUE 8 4942#define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE) 4943 4944static const u8 skb_ext_type_len[] = { 4945#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 4946 [SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info), 4947#endif 4948#ifdef CONFIG_XFRM 4949 [SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path), 4950#endif 4951#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 4952 [TC_SKB_EXT] = SKB_EXT_CHUNKSIZEOF(struct tc_skb_ext), 4953#endif 4954#if IS_ENABLED(CONFIG_MPTCP) 4955 [SKB_EXT_MPTCP] = SKB_EXT_CHUNKSIZEOF(struct mptcp_ext), 4956#endif 4957#if IS_ENABLED(CONFIG_MCTP_FLOWS) 4958 [SKB_EXT_MCTP] = SKB_EXT_CHUNKSIZEOF(struct mctp_flow), 4959#endif 4960}; 4961 4962static __always_inline unsigned int skb_ext_total_length(void) 4963{ 4964 unsigned int l = SKB_EXT_CHUNKSIZEOF(struct skb_ext); 4965 int i; 4966 4967 for (i = 0; i < ARRAY_SIZE(skb_ext_type_len); i++) 4968 l += skb_ext_type_len[i]; 4969 4970 return l; 4971} 4972 4973static void skb_extensions_init(void) 4974{ 4975 BUILD_BUG_ON(SKB_EXT_NUM >= 8); 4976#if !IS_ENABLED(CONFIG_KCOV_INSTRUMENT_ALL) 4977 BUILD_BUG_ON(skb_ext_total_length() > 255); 4978#endif 4979 4980 skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache", 4981 SKB_EXT_ALIGN_VALUE * skb_ext_total_length(), 4982 0, 4983 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 4984 NULL); 4985} 4986#else 4987static void skb_extensions_init(void) {} 4988#endif 4989 4990/* The SKB kmem_cache slab is critical for network performance. Never 4991 * merge/alias the slab with similar sized objects. This avoids fragmentation 4992 * that hurts performance of kmem_cache_{alloc,free}_bulk APIs. 4993 */ 4994#ifndef CONFIG_SLUB_TINY 4995#define FLAG_SKB_NO_MERGE SLAB_NO_MERGE 4996#else /* CONFIG_SLUB_TINY - simple loop in kmem_cache_alloc_bulk */ 4997#define FLAG_SKB_NO_MERGE 0 4998#endif 4999 5000void __init skb_init(void) 5001{ 5002 net_hotdata.skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache", 5003 sizeof(struct sk_buff), 5004 0, 5005 SLAB_HWCACHE_ALIGN|SLAB_PANIC| 5006 FLAG_SKB_NO_MERGE, 5007 offsetof(struct sk_buff, cb), 5008 sizeof_field(struct sk_buff, cb), 5009 NULL); 5010 net_hotdata.skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache", 5011 sizeof(struct sk_buff_fclones), 5012 0, 5013 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 5014 NULL); 5015 /* usercopy should only access first SKB_SMALL_HEAD_HEADROOM bytes. 5016 * struct skb_shared_info is located at the end of skb->head, 5017 * and should not be copied to/from user. 5018 */ 5019 net_hotdata.skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head", 5020 SKB_SMALL_HEAD_CACHE_SIZE, 5021 0, 5022 SLAB_HWCACHE_ALIGN | SLAB_PANIC, 5023 0, 5024 SKB_SMALL_HEAD_HEADROOM, 5025 NULL); 5026 skb_extensions_init(); 5027} 5028 5029static int 5030__skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len, 5031 unsigned int recursion_level) 5032{ 5033 int start = skb_headlen(skb); 5034 int i, copy = start - offset; 5035 struct sk_buff *frag_iter; 5036 int elt = 0; 5037 5038 if (unlikely(recursion_level >= 24)) 5039 return -EMSGSIZE; 5040 5041 if (copy > 0) { 5042 if (copy > len) 5043 copy = len; 5044 sg_set_buf(sg, skb->data + offset, copy); 5045 elt++; 5046 if ((len -= copy) == 0) 5047 return elt; 5048 offset += copy; 5049 } 5050 5051 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 5052 int end; 5053 5054 WARN_ON(start > offset + len); 5055 5056 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 5057 if ((copy = end - offset) > 0) { 5058 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 5059 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 5060 return -EMSGSIZE; 5061 5062 if (copy > len) 5063 copy = len; 5064 sg_set_page(&sg[elt], skb_frag_page(frag), copy, 5065 skb_frag_off(frag) + offset - start); 5066 elt++; 5067 if (!(len -= copy)) 5068 return elt; 5069 offset += copy; 5070 } 5071 start = end; 5072 } 5073 5074 skb_walk_frags(skb, frag_iter) { 5075 int end, ret; 5076 5077 WARN_ON(start > offset + len); 5078 5079 end = start + frag_iter->len; 5080 if ((copy = end - offset) > 0) { 5081 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 5082 return -EMSGSIZE; 5083 5084 if (copy > len) 5085 copy = len; 5086 ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start, 5087 copy, recursion_level + 1); 5088 if (unlikely(ret < 0)) 5089 return ret; 5090 elt += ret; 5091 if ((len -= copy) == 0) 5092 return elt; 5093 offset += copy; 5094 } 5095 start = end; 5096 } 5097 BUG_ON(len); 5098 return elt; 5099} 5100 5101/** 5102 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer 5103 * @skb: Socket buffer containing the buffers to be mapped 5104 * @sg: The scatter-gather list to map into 5105 * @offset: The offset into the buffer's contents to start mapping 5106 * @len: Length of buffer space to be mapped 5107 * 5108 * Fill the specified scatter-gather list with mappings/pointers into a 5109 * region of the buffer space attached to a socket buffer. Returns either 5110 * the number of scatterlist items used, or -EMSGSIZE if the contents 5111 * could not fit. 5112 */ 5113int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) 5114{ 5115 int nsg = __skb_to_sgvec(skb, sg, offset, len, 0); 5116 5117 if (nsg <= 0) 5118 return nsg; 5119 5120 sg_mark_end(&sg[nsg - 1]); 5121 5122 return nsg; 5123} 5124EXPORT_SYMBOL_GPL(skb_to_sgvec); 5125 5126/* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given 5127 * sglist without mark the sg which contain last skb data as the end. 5128 * So the caller can mannipulate sg list as will when padding new data after 5129 * the first call without calling sg_unmark_end to expend sg list. 5130 * 5131 * Scenario to use skb_to_sgvec_nomark: 5132 * 1. sg_init_table 5133 * 2. skb_to_sgvec_nomark(payload1) 5134 * 3. skb_to_sgvec_nomark(payload2) 5135 * 5136 * This is equivalent to: 5137 * 1. sg_init_table 5138 * 2. skb_to_sgvec(payload1) 5139 * 3. sg_unmark_end 5140 * 4. skb_to_sgvec(payload2) 5141 * 5142 * When mapping mutilple payload conditionally, skb_to_sgvec_nomark 5143 * is more preferable. 5144 */ 5145int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg, 5146 int offset, int len) 5147{ 5148 return __skb_to_sgvec(skb, sg, offset, len, 0); 5149} 5150EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark); 5151 5152 5153 5154/** 5155 * skb_cow_data - Check that a socket buffer's data buffers are writable 5156 * @skb: The socket buffer to check. 5157 * @tailbits: Amount of trailing space to be added 5158 * @trailer: Returned pointer to the skb where the @tailbits space begins 5159 * 5160 * Make sure that the data buffers attached to a socket buffer are 5161 * writable. If they are not, private copies are made of the data buffers 5162 * and the socket buffer is set to use these instead. 5163 * 5164 * If @tailbits is given, make sure that there is space to write @tailbits 5165 * bytes of data beyond current end of socket buffer. @trailer will be 5166 * set to point to the skb in which this space begins. 5167 * 5168 * The number of scatterlist elements required to completely map the 5169 * COW'd and extended socket buffer will be returned. 5170 */ 5171int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer) 5172{ 5173 int copyflag; 5174 int elt; 5175 struct sk_buff *skb1, **skb_p; 5176 5177 /* If skb is cloned or its head is paged, reallocate 5178 * head pulling out all the pages (pages are considered not writable 5179 * at the moment even if they are anonymous). 5180 */ 5181 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) && 5182 !__pskb_pull_tail(skb, __skb_pagelen(skb))) 5183 return -ENOMEM; 5184 5185 /* Easy case. Most of packets will go this way. */ 5186 if (!skb_has_frag_list(skb)) { 5187 /* A little of trouble, not enough of space for trailer. 5188 * This should not happen, when stack is tuned to generate 5189 * good frames. OK, on miss we reallocate and reserve even more 5190 * space, 128 bytes is fair. */ 5191 5192 if (skb_tailroom(skb) < tailbits && 5193 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC)) 5194 return -ENOMEM; 5195 5196 /* Voila! */ 5197 *trailer = skb; 5198 return 1; 5199 } 5200 5201 /* Misery. We are in troubles, going to mincer fragments... */ 5202 5203 elt = 1; 5204 skb_p = &skb_shinfo(skb)->frag_list; 5205 copyflag = 0; 5206 5207 while ((skb1 = *skb_p) != NULL) { 5208 int ntail = 0; 5209 5210 /* The fragment is partially pulled by someone, 5211 * this can happen on input. Copy it and everything 5212 * after it. */ 5213 5214 if (skb_shared(skb1)) 5215 copyflag = 1; 5216 5217 /* If the skb is the last, worry about trailer. */ 5218 5219 if (skb1->next == NULL && tailbits) { 5220 if (skb_shinfo(skb1)->nr_frags || 5221 skb_has_frag_list(skb1) || 5222 skb_tailroom(skb1) < tailbits) 5223 ntail = tailbits + 128; 5224 } 5225 5226 if (copyflag || 5227 skb_cloned(skb1) || 5228 ntail || 5229 skb_shinfo(skb1)->nr_frags || 5230 skb_has_frag_list(skb1)) { 5231 struct sk_buff *skb2; 5232 5233 /* Fuck, we are miserable poor guys... */ 5234 if (ntail == 0) 5235 skb2 = skb_copy(skb1, GFP_ATOMIC); 5236 else 5237 skb2 = skb_copy_expand(skb1, 5238 skb_headroom(skb1), 5239 ntail, 5240 GFP_ATOMIC); 5241 if (unlikely(skb2 == NULL)) 5242 return -ENOMEM; 5243 5244 if (skb1->sk) 5245 skb_set_owner_w(skb2, skb1->sk); 5246 5247 /* Looking around. Are we still alive? 5248 * OK, link new skb, drop old one */ 5249 5250 skb2->next = skb1->next; 5251 *skb_p = skb2; 5252 kfree_skb(skb1); 5253 skb1 = skb2; 5254 } 5255 elt++; 5256 *trailer = skb1; 5257 skb_p = &skb1->next; 5258 } 5259 5260 return elt; 5261} 5262EXPORT_SYMBOL_GPL(skb_cow_data); 5263 5264static void sock_rmem_free(struct sk_buff *skb) 5265{ 5266 struct sock *sk = skb->sk; 5267 5268 atomic_sub(skb->truesize, &sk->sk_rmem_alloc); 5269} 5270 5271static void skb_set_err_queue(struct sk_buff *skb) 5272{ 5273 /* pkt_type of skbs received on local sockets is never PACKET_OUTGOING. 5274 * So, it is safe to (mis)use it to mark skbs on the error queue. 5275 */ 5276 skb->pkt_type = PACKET_OUTGOING; 5277 BUILD_BUG_ON(PACKET_OUTGOING == 0); 5278} 5279 5280/* 5281 * Note: We dont mem charge error packets (no sk_forward_alloc changes) 5282 */ 5283int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb) 5284{ 5285 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >= 5286 (unsigned int)READ_ONCE(sk->sk_rcvbuf)) 5287 return -ENOMEM; 5288 5289 skb_orphan(skb); 5290 skb->sk = sk; 5291 skb->destructor = sock_rmem_free; 5292 atomic_add(skb->truesize, &sk->sk_rmem_alloc); 5293 skb_set_err_queue(skb); 5294 5295 /* before exiting rcu section, make sure dst is refcounted */ 5296 skb_dst_force(skb); 5297 5298 skb_queue_tail(&sk->sk_error_queue, skb); 5299 if (!sock_flag(sk, SOCK_DEAD)) 5300 sk_error_report(sk); 5301 return 0; 5302} 5303EXPORT_SYMBOL(sock_queue_err_skb); 5304 5305static bool is_icmp_err_skb(const struct sk_buff *skb) 5306{ 5307 return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP || 5308 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6); 5309} 5310 5311struct sk_buff *sock_dequeue_err_skb(struct sock *sk) 5312{ 5313 struct sk_buff_head *q = &sk->sk_error_queue; 5314 struct sk_buff *skb, *skb_next = NULL; 5315 bool icmp_next = false; 5316 unsigned long flags; 5317 5318 if (skb_queue_empty_lockless(q)) 5319 return NULL; 5320 5321 spin_lock_irqsave(&q->lock, flags); 5322 skb = __skb_dequeue(q); 5323 if (skb && (skb_next = skb_peek(q))) { 5324 icmp_next = is_icmp_err_skb(skb_next); 5325 if (icmp_next) 5326 sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_errno; 5327 } 5328 spin_unlock_irqrestore(&q->lock, flags); 5329 5330 if (is_icmp_err_skb(skb) && !icmp_next) 5331 sk->sk_err = 0; 5332 5333 if (skb_next) 5334 sk_error_report(sk); 5335 5336 return skb; 5337} 5338EXPORT_SYMBOL(sock_dequeue_err_skb); 5339 5340/** 5341 * skb_clone_sk - create clone of skb, and take reference to socket 5342 * @skb: the skb to clone 5343 * 5344 * This function creates a clone of a buffer that holds a reference on 5345 * sk_refcnt. Buffers created via this function are meant to be 5346 * returned using sock_queue_err_skb, or free via kfree_skb. 5347 * 5348 * When passing buffers allocated with this function to sock_queue_err_skb 5349 * it is necessary to wrap the call with sock_hold/sock_put in order to 5350 * prevent the socket from being released prior to being enqueued on 5351 * the sk_error_queue. 5352 */ 5353struct sk_buff *skb_clone_sk(struct sk_buff *skb) 5354{ 5355 struct sock *sk = skb->sk; 5356 struct sk_buff *clone; 5357 5358 if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt)) 5359 return NULL; 5360 5361 clone = skb_clone(skb, GFP_ATOMIC); 5362 if (!clone) { 5363 sock_put(sk); 5364 return NULL; 5365 } 5366 5367 clone->sk = sk; 5368 clone->destructor = sock_efree; 5369 5370 return clone; 5371} 5372EXPORT_SYMBOL(skb_clone_sk); 5373 5374static void __skb_complete_tx_timestamp(struct sk_buff *skb, 5375 struct sock *sk, 5376 int tstype, 5377 bool opt_stats) 5378{ 5379 struct sock_exterr_skb *serr; 5380 int err; 5381 5382 BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb)); 5383 5384 serr = SKB_EXT_ERR(skb); 5385 memset(serr, 0, sizeof(*serr)); 5386 serr->ee.ee_errno = ENOMSG; 5387 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING; 5388 serr->ee.ee_info = tstype; 5389 serr->opt_stats = opt_stats; 5390 serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0; 5391 if (READ_ONCE(sk->sk_tsflags) & SOF_TIMESTAMPING_OPT_ID) { 5392 serr->ee.ee_data = skb_shinfo(skb)->tskey; 5393 if (sk_is_tcp(sk)) 5394 serr->ee.ee_data -= atomic_read(&sk->sk_tskey); 5395 } 5396 5397 err = sock_queue_err_skb(sk, skb); 5398 5399 if (err) 5400 kfree_skb(skb); 5401} 5402 5403static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly) 5404{ 5405 bool ret; 5406 5407 if (likely(READ_ONCE(sysctl_tstamp_allow_data) || tsonly)) 5408 return true; 5409 5410 read_lock_bh(&sk->sk_callback_lock); 5411 ret = sk->sk_socket && sk->sk_socket->file && 5412 file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW); 5413 read_unlock_bh(&sk->sk_callback_lock); 5414 return ret; 5415} 5416 5417void skb_complete_tx_timestamp(struct sk_buff *skb, 5418 struct skb_shared_hwtstamps *hwtstamps) 5419{ 5420 struct sock *sk = skb->sk; 5421 5422 if (!skb_may_tx_timestamp(sk, false)) 5423 goto err; 5424 5425 /* Take a reference to prevent skb_orphan() from freeing the socket, 5426 * but only if the socket refcount is not zero. 5427 */ 5428 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 5429 *skb_hwtstamps(skb) = *hwtstamps; 5430 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false); 5431 sock_put(sk); 5432 return; 5433 } 5434 5435err: 5436 kfree_skb(skb); 5437} 5438EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp); 5439 5440void __skb_tstamp_tx(struct sk_buff *orig_skb, 5441 const struct sk_buff *ack_skb, 5442 struct skb_shared_hwtstamps *hwtstamps, 5443 struct sock *sk, int tstype) 5444{ 5445 struct sk_buff *skb; 5446 bool tsonly, opt_stats = false; 5447 u32 tsflags; 5448 5449 if (!sk) 5450 return; 5451 5452 tsflags = READ_ONCE(sk->sk_tsflags); 5453 if (!hwtstamps && !(tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) && 5454 skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS) 5455 return; 5456 5457 tsonly = tsflags & SOF_TIMESTAMPING_OPT_TSONLY; 5458 if (!skb_may_tx_timestamp(sk, tsonly)) 5459 return; 5460 5461 if (tsonly) { 5462#ifdef CONFIG_INET 5463 if ((tsflags & SOF_TIMESTAMPING_OPT_STATS) && 5464 sk_is_tcp(sk)) { 5465 skb = tcp_get_timestamping_opt_stats(sk, orig_skb, 5466 ack_skb); 5467 opt_stats = true; 5468 } else 5469#endif 5470 skb = alloc_skb(0, GFP_ATOMIC); 5471 } else { 5472 skb = skb_clone(orig_skb, GFP_ATOMIC); 5473 5474 if (skb_orphan_frags_rx(skb, GFP_ATOMIC)) { 5475 kfree_skb(skb); 5476 return; 5477 } 5478 } 5479 if (!skb) 5480 return; 5481 5482 if (tsonly) { 5483 skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags & 5484 SKBTX_ANY_TSTAMP; 5485 skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey; 5486 } 5487 5488 if (hwtstamps) 5489 *skb_hwtstamps(skb) = *hwtstamps; 5490 else 5491 __net_timestamp(skb); 5492 5493 __skb_complete_tx_timestamp(skb, sk, tstype, opt_stats); 5494} 5495EXPORT_SYMBOL_GPL(__skb_tstamp_tx); 5496 5497void skb_tstamp_tx(struct sk_buff *orig_skb, 5498 struct skb_shared_hwtstamps *hwtstamps) 5499{ 5500 return __skb_tstamp_tx(orig_skb, NULL, hwtstamps, orig_skb->sk, 5501 SCM_TSTAMP_SND); 5502} 5503EXPORT_SYMBOL_GPL(skb_tstamp_tx); 5504 5505#ifdef CONFIG_WIRELESS 5506void skb_complete_wifi_ack(struct sk_buff *skb, bool acked) 5507{ 5508 struct sock *sk = skb->sk; 5509 struct sock_exterr_skb *serr; 5510 int err = 1; 5511 5512 skb->wifi_acked_valid = 1; 5513 skb->wifi_acked = acked; 5514 5515 serr = SKB_EXT_ERR(skb); 5516 memset(serr, 0, sizeof(*serr)); 5517 serr->ee.ee_errno = ENOMSG; 5518 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS; 5519 5520 /* Take a reference to prevent skb_orphan() from freeing the socket, 5521 * but only if the socket refcount is not zero. 5522 */ 5523 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 5524 err = sock_queue_err_skb(sk, skb); 5525 sock_put(sk); 5526 } 5527 if (err) 5528 kfree_skb(skb); 5529} 5530EXPORT_SYMBOL_GPL(skb_complete_wifi_ack); 5531#endif /* CONFIG_WIRELESS */ 5532 5533/** 5534 * skb_partial_csum_set - set up and verify partial csum values for packet 5535 * @skb: the skb to set 5536 * @start: the number of bytes after skb->data to start checksumming. 5537 * @off: the offset from start to place the checksum. 5538 * 5539 * For untrusted partially-checksummed packets, we need to make sure the values 5540 * for skb->csum_start and skb->csum_offset are valid so we don't oops. 5541 * 5542 * This function checks and sets those values and skb->ip_summed: if this 5543 * returns false you should drop the packet. 5544 */ 5545bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off) 5546{ 5547 u32 csum_end = (u32)start + (u32)off + sizeof(__sum16); 5548 u32 csum_start = skb_headroom(skb) + (u32)start; 5549 5550 if (unlikely(csum_start >= U16_MAX || csum_end > skb_headlen(skb))) { 5551 net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n", 5552 start, off, skb_headroom(skb), skb_headlen(skb)); 5553 return false; 5554 } 5555 skb->ip_summed = CHECKSUM_PARTIAL; 5556 skb->csum_start = csum_start; 5557 skb->csum_offset = off; 5558 skb->transport_header = csum_start; 5559 return true; 5560} 5561EXPORT_SYMBOL_GPL(skb_partial_csum_set); 5562 5563static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len, 5564 unsigned int max) 5565{ 5566 if (skb_headlen(skb) >= len) 5567 return 0; 5568 5569 /* If we need to pullup then pullup to the max, so we 5570 * won't need to do it again. 5571 */ 5572 if (max > skb->len) 5573 max = skb->len; 5574 5575 if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL) 5576 return -ENOMEM; 5577 5578 if (skb_headlen(skb) < len) 5579 return -EPROTO; 5580 5581 return 0; 5582} 5583 5584#define MAX_TCP_HDR_LEN (15 * 4) 5585 5586static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb, 5587 typeof(IPPROTO_IP) proto, 5588 unsigned int off) 5589{ 5590 int err; 5591 5592 switch (proto) { 5593 case IPPROTO_TCP: 5594 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr), 5595 off + MAX_TCP_HDR_LEN); 5596 if (!err && !skb_partial_csum_set(skb, off, 5597 offsetof(struct tcphdr, 5598 check))) 5599 err = -EPROTO; 5600 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check; 5601 5602 case IPPROTO_UDP: 5603 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr), 5604 off + sizeof(struct udphdr)); 5605 if (!err && !skb_partial_csum_set(skb, off, 5606 offsetof(struct udphdr, 5607 check))) 5608 err = -EPROTO; 5609 return err ? ERR_PTR(err) : &udp_hdr(skb)->check; 5610 } 5611 5612 return ERR_PTR(-EPROTO); 5613} 5614 5615/* This value should be large enough to cover a tagged ethernet header plus 5616 * maximally sized IP and TCP or UDP headers. 5617 */ 5618#define MAX_IP_HDR_LEN 128 5619 5620static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate) 5621{ 5622 unsigned int off; 5623 bool fragment; 5624 __sum16 *csum; 5625 int err; 5626 5627 fragment = false; 5628 5629 err = skb_maybe_pull_tail(skb, 5630 sizeof(struct iphdr), 5631 MAX_IP_HDR_LEN); 5632 if (err < 0) 5633 goto out; 5634 5635 if (ip_is_fragment(ip_hdr(skb))) 5636 fragment = true; 5637 5638 off = ip_hdrlen(skb); 5639 5640 err = -EPROTO; 5641 5642 if (fragment) 5643 goto out; 5644 5645 csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off); 5646 if (IS_ERR(csum)) 5647 return PTR_ERR(csum); 5648 5649 if (recalculate) 5650 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 5651 ip_hdr(skb)->daddr, 5652 skb->len - off, 5653 ip_hdr(skb)->protocol, 0); 5654 err = 0; 5655 5656out: 5657 return err; 5658} 5659 5660/* This value should be large enough to cover a tagged ethernet header plus 5661 * an IPv6 header, all options, and a maximal TCP or UDP header. 5662 */ 5663#define MAX_IPV6_HDR_LEN 256 5664 5665#define OPT_HDR(type, skb, off) \ 5666 (type *)(skb_network_header(skb) + (off)) 5667 5668static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) 5669{ 5670 int err; 5671 u8 nexthdr; 5672 unsigned int off; 5673 unsigned int len; 5674 bool fragment; 5675 bool done; 5676 __sum16 *csum; 5677 5678 fragment = false; 5679 done = false; 5680 5681 off = sizeof(struct ipv6hdr); 5682 5683 err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN); 5684 if (err < 0) 5685 goto out; 5686 5687 nexthdr = ipv6_hdr(skb)->nexthdr; 5688 5689 len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len); 5690 while (off <= len && !done) { 5691 switch (nexthdr) { 5692 case IPPROTO_DSTOPTS: 5693 case IPPROTO_HOPOPTS: 5694 case IPPROTO_ROUTING: { 5695 struct ipv6_opt_hdr *hp; 5696 5697 err = skb_maybe_pull_tail(skb, 5698 off + 5699 sizeof(struct ipv6_opt_hdr), 5700 MAX_IPV6_HDR_LEN); 5701 if (err < 0) 5702 goto out; 5703 5704 hp = OPT_HDR(struct ipv6_opt_hdr, skb, off); 5705 nexthdr = hp->nexthdr; 5706 off += ipv6_optlen(hp); 5707 break; 5708 } 5709 case IPPROTO_AH: { 5710 struct ip_auth_hdr *hp; 5711 5712 err = skb_maybe_pull_tail(skb, 5713 off + 5714 sizeof(struct ip_auth_hdr), 5715 MAX_IPV6_HDR_LEN); 5716 if (err < 0) 5717 goto out; 5718 5719 hp = OPT_HDR(struct ip_auth_hdr, skb, off); 5720 nexthdr = hp->nexthdr; 5721 off += ipv6_authlen(hp); 5722 break; 5723 } 5724 case IPPROTO_FRAGMENT: { 5725 struct frag_hdr *hp; 5726 5727 err = skb_maybe_pull_tail(skb, 5728 off + 5729 sizeof(struct frag_hdr), 5730 MAX_IPV6_HDR_LEN); 5731 if (err < 0) 5732 goto out; 5733 5734 hp = OPT_HDR(struct frag_hdr, skb, off); 5735 5736 if (hp->frag_off & htons(IP6_OFFSET | IP6_MF)) 5737 fragment = true; 5738 5739 nexthdr = hp->nexthdr; 5740 off += sizeof(struct frag_hdr); 5741 break; 5742 } 5743 default: 5744 done = true; 5745 break; 5746 } 5747 } 5748 5749 err = -EPROTO; 5750 5751 if (!done || fragment) 5752 goto out; 5753 5754 csum = skb_checksum_setup_ip(skb, nexthdr, off); 5755 if (IS_ERR(csum)) 5756 return PTR_ERR(csum); 5757 5758 if (recalculate) 5759 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 5760 &ipv6_hdr(skb)->daddr, 5761 skb->len - off, nexthdr, 0); 5762 err = 0; 5763 5764out: 5765 return err; 5766} 5767 5768/** 5769 * skb_checksum_setup - set up partial checksum offset 5770 * @skb: the skb to set up 5771 * @recalculate: if true the pseudo-header checksum will be recalculated 5772 */ 5773int skb_checksum_setup(struct sk_buff *skb, bool recalculate) 5774{ 5775 int err; 5776 5777 switch (skb->protocol) { 5778 case htons(ETH_P_IP): 5779 err = skb_checksum_setup_ipv4(skb, recalculate); 5780 break; 5781 5782 case htons(ETH_P_IPV6): 5783 err = skb_checksum_setup_ipv6(skb, recalculate); 5784 break; 5785 5786 default: 5787 err = -EPROTO; 5788 break; 5789 } 5790 5791 return err; 5792} 5793EXPORT_SYMBOL(skb_checksum_setup); 5794 5795/** 5796 * skb_checksum_maybe_trim - maybe trims the given skb 5797 * @skb: the skb to check 5798 * @transport_len: the data length beyond the network header 5799 * 5800 * Checks whether the given skb has data beyond the given transport length. 5801 * If so, returns a cloned skb trimmed to this transport length. 5802 * Otherwise returns the provided skb. Returns NULL in error cases 5803 * (e.g. transport_len exceeds skb length or out-of-memory). 5804 * 5805 * Caller needs to set the skb transport header and free any returned skb if it 5806 * differs from the provided skb. 5807 */ 5808static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb, 5809 unsigned int transport_len) 5810{ 5811 struct sk_buff *skb_chk; 5812 unsigned int len = skb_transport_offset(skb) + transport_len; 5813 int ret; 5814 5815 if (skb->len < len) 5816 return NULL; 5817 else if (skb->len == len) 5818 return skb; 5819 5820 skb_chk = skb_clone(skb, GFP_ATOMIC); 5821 if (!skb_chk) 5822 return NULL; 5823 5824 ret = pskb_trim_rcsum(skb_chk, len); 5825 if (ret) { 5826 kfree_skb(skb_chk); 5827 return NULL; 5828 } 5829 5830 return skb_chk; 5831} 5832 5833/** 5834 * skb_checksum_trimmed - validate checksum of an skb 5835 * @skb: the skb to check 5836 * @transport_len: the data length beyond the network header 5837 * @skb_chkf: checksum function to use 5838 * 5839 * Applies the given checksum function skb_chkf to the provided skb. 5840 * Returns a checked and maybe trimmed skb. Returns NULL on error. 5841 * 5842 * If the skb has data beyond the given transport length, then a 5843 * trimmed & cloned skb is checked and returned. 5844 * 5845 * Caller needs to set the skb transport header and free any returned skb if it 5846 * differs from the provided skb. 5847 */ 5848struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb, 5849 unsigned int transport_len, 5850 __sum16(*skb_chkf)(struct sk_buff *skb)) 5851{ 5852 struct sk_buff *skb_chk; 5853 unsigned int offset = skb_transport_offset(skb); 5854 __sum16 ret; 5855 5856 skb_chk = skb_checksum_maybe_trim(skb, transport_len); 5857 if (!skb_chk) 5858 goto err; 5859 5860 if (!pskb_may_pull(skb_chk, offset)) 5861 goto err; 5862 5863 skb_pull_rcsum(skb_chk, offset); 5864 ret = skb_chkf(skb_chk); 5865 skb_push_rcsum(skb_chk, offset); 5866 5867 if (ret) 5868 goto err; 5869 5870 return skb_chk; 5871 5872err: 5873 if (skb_chk && skb_chk != skb) 5874 kfree_skb(skb_chk); 5875 5876 return NULL; 5877 5878} 5879EXPORT_SYMBOL(skb_checksum_trimmed); 5880 5881void __skb_warn_lro_forwarding(const struct sk_buff *skb) 5882{ 5883 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n", 5884 skb->dev->name); 5885} 5886EXPORT_SYMBOL(__skb_warn_lro_forwarding); 5887 5888void kfree_skb_partial(struct sk_buff *skb, bool head_stolen) 5889{ 5890 if (head_stolen) { 5891 skb_release_head_state(skb); 5892 kmem_cache_free(net_hotdata.skbuff_cache, skb); 5893 } else { 5894 __kfree_skb(skb); 5895 } 5896} 5897EXPORT_SYMBOL(kfree_skb_partial); 5898 5899/** 5900 * skb_try_coalesce - try to merge skb to prior one 5901 * @to: prior buffer 5902 * @from: buffer to add 5903 * @fragstolen: pointer to boolean 5904 * @delta_truesize: how much more was allocated than was requested 5905 */ 5906bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, 5907 bool *fragstolen, int *delta_truesize) 5908{ 5909 struct skb_shared_info *to_shinfo, *from_shinfo; 5910 int i, delta, len = from->len; 5911 5912 *fragstolen = false; 5913 5914 if (skb_cloned(to)) 5915 return false; 5916 5917 /* In general, avoid mixing page_pool and non-page_pool allocated 5918 * pages within the same SKB. In theory we could take full 5919 * references if @from is cloned and !@to->pp_recycle but its 5920 * tricky (due to potential race with the clone disappearing) and 5921 * rare, so not worth dealing with. 5922 */ 5923 if (to->pp_recycle != from->pp_recycle) 5924 return false; 5925 5926 if (len <= skb_tailroom(to)) { 5927 if (len) 5928 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len)); 5929 *delta_truesize = 0; 5930 return true; 5931 } 5932 5933 to_shinfo = skb_shinfo(to); 5934 from_shinfo = skb_shinfo(from); 5935 if (to_shinfo->frag_list || from_shinfo->frag_list) 5936 return false; 5937 if (skb_zcopy(to) || skb_zcopy(from)) 5938 return false; 5939 5940 if (skb_headlen(from) != 0) { 5941 struct page *page; 5942 unsigned int offset; 5943 5944 if (to_shinfo->nr_frags + 5945 from_shinfo->nr_frags >= MAX_SKB_FRAGS) 5946 return false; 5947 5948 if (skb_head_is_locked(from)) 5949 return false; 5950 5951 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); 5952 5953 page = virt_to_head_page(from->head); 5954 offset = from->data - (unsigned char *)page_address(page); 5955 5956 skb_fill_page_desc(to, to_shinfo->nr_frags, 5957 page, offset, skb_headlen(from)); 5958 *fragstolen = true; 5959 } else { 5960 if (to_shinfo->nr_frags + 5961 from_shinfo->nr_frags > MAX_SKB_FRAGS) 5962 return false; 5963 5964 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from)); 5965 } 5966 5967 WARN_ON_ONCE(delta < len); 5968 5969 memcpy(to_shinfo->frags + to_shinfo->nr_frags, 5970 from_shinfo->frags, 5971 from_shinfo->nr_frags * sizeof(skb_frag_t)); 5972 to_shinfo->nr_frags += from_shinfo->nr_frags; 5973 5974 if (!skb_cloned(from)) 5975 from_shinfo->nr_frags = 0; 5976 5977 /* if the skb is not cloned this does nothing 5978 * since we set nr_frags to 0. 5979 */ 5980 if (skb_pp_frag_ref(from)) { 5981 for (i = 0; i < from_shinfo->nr_frags; i++) 5982 __skb_frag_ref(&from_shinfo->frags[i]); 5983 } 5984 5985 to->truesize += delta; 5986 to->len += len; 5987 to->data_len += len; 5988 5989 *delta_truesize = delta; 5990 return true; 5991} 5992EXPORT_SYMBOL(skb_try_coalesce); 5993 5994/** 5995 * skb_scrub_packet - scrub an skb 5996 * 5997 * @skb: buffer to clean 5998 * @xnet: packet is crossing netns 5999 * 6000 * skb_scrub_packet can be used after encapsulating or decapsulting a packet 6001 * into/from a tunnel. Some information have to be cleared during these 6002 * operations. 6003 * skb_scrub_packet can also be used to clean a skb before injecting it in 6004 * another namespace (@xnet == true). We have to clear all information in the 6005 * skb that could impact namespace isolation. 6006 */ 6007void skb_scrub_packet(struct sk_buff *skb, bool xnet) 6008{ 6009 skb->pkt_type = PACKET_HOST; 6010 skb->skb_iif = 0; 6011 skb->ignore_df = 0; 6012 skb_dst_drop(skb); 6013 skb_ext_reset(skb); 6014 nf_reset_ct(skb); 6015 nf_reset_trace(skb); 6016 6017#ifdef CONFIG_NET_SWITCHDEV 6018 skb->offload_fwd_mark = 0; 6019 skb->offload_l3_fwd_mark = 0; 6020#endif 6021 6022 if (!xnet) 6023 return; 6024 6025 ipvs_reset(skb); 6026 skb->mark = 0; 6027 skb_clear_tstamp(skb); 6028} 6029EXPORT_SYMBOL_GPL(skb_scrub_packet); 6030 6031static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb) 6032{ 6033 int mac_len, meta_len; 6034 void *meta; 6035 6036 if (skb_cow(skb, skb_headroom(skb)) < 0) { 6037 kfree_skb(skb); 6038 return NULL; 6039 } 6040 6041 mac_len = skb->data - skb_mac_header(skb); 6042 if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) { 6043 memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb), 6044 mac_len - VLAN_HLEN - ETH_TLEN); 6045 } 6046 6047 meta_len = skb_metadata_len(skb); 6048 if (meta_len) { 6049 meta = skb_metadata_end(skb) - meta_len; 6050 memmove(meta + VLAN_HLEN, meta, meta_len); 6051 } 6052 6053 skb->mac_header += VLAN_HLEN; 6054 return skb; 6055} 6056 6057struct sk_buff *skb_vlan_untag(struct sk_buff *skb) 6058{ 6059 struct vlan_hdr *vhdr; 6060 u16 vlan_tci; 6061 6062 if (unlikely(skb_vlan_tag_present(skb))) { 6063 /* vlan_tci is already set-up so leave this for another time */ 6064 return skb; 6065 } 6066 6067 skb = skb_share_check(skb, GFP_ATOMIC); 6068 if (unlikely(!skb)) 6069 goto err_free; 6070 /* We may access the two bytes after vlan_hdr in vlan_set_encap_proto(). */ 6071 if (unlikely(!pskb_may_pull(skb, VLAN_HLEN + sizeof(unsigned short)))) 6072 goto err_free; 6073 6074 vhdr = (struct vlan_hdr *)skb->data; 6075 vlan_tci = ntohs(vhdr->h_vlan_TCI); 6076 __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci); 6077 6078 skb_pull_rcsum(skb, VLAN_HLEN); 6079 vlan_set_encap_proto(skb, vhdr); 6080 6081 skb = skb_reorder_vlan_header(skb); 6082 if (unlikely(!skb)) 6083 goto err_free; 6084 6085 skb_reset_network_header(skb); 6086 if (!skb_transport_header_was_set(skb)) 6087 skb_reset_transport_header(skb); 6088 skb_reset_mac_len(skb); 6089 6090 return skb; 6091 6092err_free: 6093 kfree_skb(skb); 6094 return NULL; 6095} 6096EXPORT_SYMBOL(skb_vlan_untag); 6097 6098int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len) 6099{ 6100 if (!pskb_may_pull(skb, write_len)) 6101 return -ENOMEM; 6102 6103 if (!skb_cloned(skb) || skb_clone_writable(skb, write_len)) 6104 return 0; 6105 6106 return pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 6107} 6108EXPORT_SYMBOL(skb_ensure_writable); 6109 6110int skb_ensure_writable_head_tail(struct sk_buff *skb, struct net_device *dev) 6111{ 6112 int needed_headroom = dev->needed_headroom; 6113 int needed_tailroom = dev->needed_tailroom; 6114 6115 /* For tail taggers, we need to pad short frames ourselves, to ensure 6116 * that the tail tag does not fail at its role of being at the end of 6117 * the packet, once the conduit interface pads the frame. Account for 6118 * that pad length here, and pad later. 6119 */ 6120 if (unlikely(needed_tailroom && skb->len < ETH_ZLEN)) 6121 needed_tailroom += ETH_ZLEN - skb->len; 6122 /* skb_headroom() returns unsigned int... */ 6123 needed_headroom = max_t(int, needed_headroom - skb_headroom(skb), 0); 6124 needed_tailroom = max_t(int, needed_tailroom - skb_tailroom(skb), 0); 6125 6126 if (likely(!needed_headroom && !needed_tailroom && !skb_cloned(skb))) 6127 /* No reallocation needed, yay! */ 6128 return 0; 6129 6130 return pskb_expand_head(skb, needed_headroom, needed_tailroom, 6131 GFP_ATOMIC); 6132} 6133EXPORT_SYMBOL(skb_ensure_writable_head_tail); 6134 6135/* remove VLAN header from packet and update csum accordingly. 6136 * expects a non skb_vlan_tag_present skb with a vlan tag payload 6137 */ 6138int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci) 6139{ 6140 int offset = skb->data - skb_mac_header(skb); 6141 int err; 6142 6143 if (WARN_ONCE(offset, 6144 "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n", 6145 offset)) { 6146 return -EINVAL; 6147 } 6148 6149 err = skb_ensure_writable(skb, VLAN_ETH_HLEN); 6150 if (unlikely(err)) 6151 return err; 6152 6153 skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 6154 6155 vlan_remove_tag(skb, vlan_tci); 6156 6157 skb->mac_header += VLAN_HLEN; 6158 6159 if (skb_network_offset(skb) < ETH_HLEN) 6160 skb_set_network_header(skb, ETH_HLEN); 6161 6162 skb_reset_mac_len(skb); 6163 6164 return err; 6165} 6166EXPORT_SYMBOL(__skb_vlan_pop); 6167 6168/* Pop a vlan tag either from hwaccel or from payload. 6169 * Expects skb->data at mac header. 6170 */ 6171int skb_vlan_pop(struct sk_buff *skb) 6172{ 6173 u16 vlan_tci; 6174 __be16 vlan_proto; 6175 int err; 6176 6177 if (likely(skb_vlan_tag_present(skb))) { 6178 __vlan_hwaccel_clear_tag(skb); 6179 } else { 6180 if (unlikely(!eth_type_vlan(skb->protocol))) 6181 return 0; 6182 6183 err = __skb_vlan_pop(skb, &vlan_tci); 6184 if (err) 6185 return err; 6186 } 6187 /* move next vlan tag to hw accel tag */ 6188 if (likely(!eth_type_vlan(skb->protocol))) 6189 return 0; 6190 6191 vlan_proto = skb->protocol; 6192 err = __skb_vlan_pop(skb, &vlan_tci); 6193 if (unlikely(err)) 6194 return err; 6195 6196 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 6197 return 0; 6198} 6199EXPORT_SYMBOL(skb_vlan_pop); 6200 6201/* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present). 6202 * Expects skb->data at mac header. 6203 */ 6204int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci) 6205{ 6206 if (skb_vlan_tag_present(skb)) { 6207 int offset = skb->data - skb_mac_header(skb); 6208 int err; 6209 6210 if (WARN_ONCE(offset, 6211 "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n", 6212 offset)) { 6213 return -EINVAL; 6214 } 6215 6216 err = __vlan_insert_tag(skb, skb->vlan_proto, 6217 skb_vlan_tag_get(skb)); 6218 if (err) 6219 return err; 6220 6221 skb->protocol = skb->vlan_proto; 6222 skb->mac_len += VLAN_HLEN; 6223 6224 skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 6225 } 6226 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 6227 return 0; 6228} 6229EXPORT_SYMBOL(skb_vlan_push); 6230 6231/** 6232 * skb_eth_pop() - Drop the Ethernet header at the head of a packet 6233 * 6234 * @skb: Socket buffer to modify 6235 * 6236 * Drop the Ethernet header of @skb. 6237 * 6238 * Expects that skb->data points to the mac header and that no VLAN tags are 6239 * present. 6240 * 6241 * Returns 0 on success, -errno otherwise. 6242 */ 6243int skb_eth_pop(struct sk_buff *skb) 6244{ 6245 if (!pskb_may_pull(skb, ETH_HLEN) || skb_vlan_tagged(skb) || 6246 skb_network_offset(skb) < ETH_HLEN) 6247 return -EPROTO; 6248 6249 skb_pull_rcsum(skb, ETH_HLEN); 6250 skb_reset_mac_header(skb); 6251 skb_reset_mac_len(skb); 6252 6253 return 0; 6254} 6255EXPORT_SYMBOL(skb_eth_pop); 6256 6257/** 6258 * skb_eth_push() - Add a new Ethernet header at the head of a packet 6259 * 6260 * @skb: Socket buffer to modify 6261 * @dst: Destination MAC address of the new header 6262 * @src: Source MAC address of the new header 6263 * 6264 * Prepend @skb with a new Ethernet header. 6265 * 6266 * Expects that skb->data points to the mac header, which must be empty. 6267 * 6268 * Returns 0 on success, -errno otherwise. 6269 */ 6270int skb_eth_push(struct sk_buff *skb, const unsigned char *dst, 6271 const unsigned char *src) 6272{ 6273 struct ethhdr *eth; 6274 int err; 6275 6276 if (skb_network_offset(skb) || skb_vlan_tag_present(skb)) 6277 return -EPROTO; 6278 6279 err = skb_cow_head(skb, sizeof(*eth)); 6280 if (err < 0) 6281 return err; 6282 6283 skb_push(skb, sizeof(*eth)); 6284 skb_reset_mac_header(skb); 6285 skb_reset_mac_len(skb); 6286 6287 eth = eth_hdr(skb); 6288 ether_addr_copy(eth->h_dest, dst); 6289 ether_addr_copy(eth->h_source, src); 6290 eth->h_proto = skb->protocol; 6291 6292 skb_postpush_rcsum(skb, eth, sizeof(*eth)); 6293 6294 return 0; 6295} 6296EXPORT_SYMBOL(skb_eth_push); 6297 6298/* Update the ethertype of hdr and the skb csum value if required. */ 6299static void skb_mod_eth_type(struct sk_buff *skb, struct ethhdr *hdr, 6300 __be16 ethertype) 6301{ 6302 if (skb->ip_summed == CHECKSUM_COMPLETE) { 6303 __be16 diff[] = { ~hdr->h_proto, ethertype }; 6304 6305 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum); 6306 } 6307 6308 hdr->h_proto = ethertype; 6309} 6310 6311/** 6312 * skb_mpls_push() - push a new MPLS header after mac_len bytes from start of 6313 * the packet 6314 * 6315 * @skb: buffer 6316 * @mpls_lse: MPLS label stack entry to push 6317 * @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848) 6318 * @mac_len: length of the MAC header 6319 * @ethernet: flag to indicate if the resulting packet after skb_mpls_push is 6320 * ethernet 6321 * 6322 * Expects skb->data at mac header. 6323 * 6324 * Returns 0 on success, -errno otherwise. 6325 */ 6326int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto, 6327 int mac_len, bool ethernet) 6328{ 6329 struct mpls_shim_hdr *lse; 6330 int err; 6331 6332 if (unlikely(!eth_p_mpls(mpls_proto))) 6333 return -EINVAL; 6334 6335 /* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */ 6336 if (skb->encapsulation) 6337 return -EINVAL; 6338 6339 err = skb_cow_head(skb, MPLS_HLEN); 6340 if (unlikely(err)) 6341 return err; 6342 6343 if (!skb->inner_protocol) { 6344 skb_set_inner_network_header(skb, skb_network_offset(skb)); 6345 skb_set_inner_protocol(skb, skb->protocol); 6346 } 6347 6348 skb_push(skb, MPLS_HLEN); 6349 memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb), 6350 mac_len); 6351 skb_reset_mac_header(skb); 6352 skb_set_network_header(skb, mac_len); 6353 skb_reset_mac_len(skb); 6354 6355 lse = mpls_hdr(skb); 6356 lse->label_stack_entry = mpls_lse; 6357 skb_postpush_rcsum(skb, lse, MPLS_HLEN); 6358 6359 if (ethernet && mac_len >= ETH_HLEN) 6360 skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto); 6361 skb->protocol = mpls_proto; 6362 6363 return 0; 6364} 6365EXPORT_SYMBOL_GPL(skb_mpls_push); 6366 6367/** 6368 * skb_mpls_pop() - pop the outermost MPLS header 6369 * 6370 * @skb: buffer 6371 * @next_proto: ethertype of header after popped MPLS header 6372 * @mac_len: length of the MAC header 6373 * @ethernet: flag to indicate if the packet is ethernet 6374 * 6375 * Expects skb->data at mac header. 6376 * 6377 * Returns 0 on success, -errno otherwise. 6378 */ 6379int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len, 6380 bool ethernet) 6381{ 6382 int err; 6383 6384 if (unlikely(!eth_p_mpls(skb->protocol))) 6385 return 0; 6386 6387 err = skb_ensure_writable(skb, mac_len + MPLS_HLEN); 6388 if (unlikely(err)) 6389 return err; 6390 6391 skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN); 6392 memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb), 6393 mac_len); 6394 6395 __skb_pull(skb, MPLS_HLEN); 6396 skb_reset_mac_header(skb); 6397 skb_set_network_header(skb, mac_len); 6398 6399 if (ethernet && mac_len >= ETH_HLEN) { 6400 struct ethhdr *hdr; 6401 6402 /* use mpls_hdr() to get ethertype to account for VLANs. */ 6403 hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN); 6404 skb_mod_eth_type(skb, hdr, next_proto); 6405 } 6406 skb->protocol = next_proto; 6407 6408 return 0; 6409} 6410EXPORT_SYMBOL_GPL(skb_mpls_pop); 6411 6412/** 6413 * skb_mpls_update_lse() - modify outermost MPLS header and update csum 6414 * 6415 * @skb: buffer 6416 * @mpls_lse: new MPLS label stack entry to update to 6417 * 6418 * Expects skb->data at mac header. 6419 * 6420 * Returns 0 on success, -errno otherwise. 6421 */ 6422int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse) 6423{ 6424 int err; 6425 6426 if (unlikely(!eth_p_mpls(skb->protocol))) 6427 return -EINVAL; 6428 6429 err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN); 6430 if (unlikely(err)) 6431 return err; 6432 6433 if (skb->ip_summed == CHECKSUM_COMPLETE) { 6434 __be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse }; 6435 6436 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum); 6437 } 6438 6439 mpls_hdr(skb)->label_stack_entry = mpls_lse; 6440 6441 return 0; 6442} 6443EXPORT_SYMBOL_GPL(skb_mpls_update_lse); 6444 6445/** 6446 * skb_mpls_dec_ttl() - decrement the TTL of the outermost MPLS header 6447 * 6448 * @skb: buffer 6449 * 6450 * Expects skb->data at mac header. 6451 * 6452 * Returns 0 on success, -errno otherwise. 6453 */ 6454int skb_mpls_dec_ttl(struct sk_buff *skb) 6455{ 6456 u32 lse; 6457 u8 ttl; 6458 6459 if (unlikely(!eth_p_mpls(skb->protocol))) 6460 return -EINVAL; 6461 6462 if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN)) 6463 return -ENOMEM; 6464 6465 lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry); 6466 ttl = (lse & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT; 6467 if (!--ttl) 6468 return -EINVAL; 6469 6470 lse &= ~MPLS_LS_TTL_MASK; 6471 lse |= ttl << MPLS_LS_TTL_SHIFT; 6472 6473 return skb_mpls_update_lse(skb, cpu_to_be32(lse)); 6474} 6475EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl); 6476 6477/** 6478 * alloc_skb_with_frags - allocate skb with page frags 6479 * 6480 * @header_len: size of linear part 6481 * @data_len: needed length in frags 6482 * @order: max page order desired. 6483 * @errcode: pointer to error code if any 6484 * @gfp_mask: allocation mask 6485 * 6486 * This can be used to allocate a paged skb, given a maximal order for frags. 6487 */ 6488struct sk_buff *alloc_skb_with_frags(unsigned long header_len, 6489 unsigned long data_len, 6490 int order, 6491 int *errcode, 6492 gfp_t gfp_mask) 6493{ 6494 unsigned long chunk; 6495 struct sk_buff *skb; 6496 struct page *page; 6497 int nr_frags = 0; 6498 6499 *errcode = -EMSGSIZE; 6500 if (unlikely(data_len > MAX_SKB_FRAGS * (PAGE_SIZE << order))) 6501 return NULL; 6502 6503 *errcode = -ENOBUFS; 6504 skb = alloc_skb(header_len, gfp_mask); 6505 if (!skb) 6506 return NULL; 6507 6508 while (data_len) { 6509 if (nr_frags == MAX_SKB_FRAGS - 1) 6510 goto failure; 6511 while (order && PAGE_ALIGN(data_len) < (PAGE_SIZE << order)) 6512 order--; 6513 6514 if (order) { 6515 page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) | 6516 __GFP_COMP | 6517 __GFP_NOWARN, 6518 order); 6519 if (!page) { 6520 order--; 6521 continue; 6522 } 6523 } else { 6524 page = alloc_page(gfp_mask); 6525 if (!page) 6526 goto failure; 6527 } 6528 chunk = min_t(unsigned long, data_len, 6529 PAGE_SIZE << order); 6530 skb_fill_page_desc(skb, nr_frags, page, 0, chunk); 6531 nr_frags++; 6532 skb->truesize += (PAGE_SIZE << order); 6533 data_len -= chunk; 6534 } 6535 return skb; 6536 6537failure: 6538 kfree_skb(skb); 6539 return NULL; 6540} 6541EXPORT_SYMBOL(alloc_skb_with_frags); 6542 6543/* carve out the first off bytes from skb when off < headlen */ 6544static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off, 6545 const int headlen, gfp_t gfp_mask) 6546{ 6547 int i; 6548 unsigned int size = skb_end_offset(skb); 6549 int new_hlen = headlen - off; 6550 u8 *data; 6551 6552 if (skb_pfmemalloc(skb)) 6553 gfp_mask |= __GFP_MEMALLOC; 6554 6555 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL); 6556 if (!data) 6557 return -ENOMEM; 6558 size = SKB_WITH_OVERHEAD(size); 6559 6560 /* Copy real data, and all frags */ 6561 skb_copy_from_linear_data_offset(skb, off, data, new_hlen); 6562 skb->len -= off; 6563 6564 memcpy((struct skb_shared_info *)(data + size), 6565 skb_shinfo(skb), 6566 offsetof(struct skb_shared_info, 6567 frags[skb_shinfo(skb)->nr_frags])); 6568 if (skb_cloned(skb)) { 6569 /* drop the old head gracefully */ 6570 if (skb_orphan_frags(skb, gfp_mask)) { 6571 skb_kfree_head(data, size); 6572 return -ENOMEM; 6573 } 6574 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 6575 skb_frag_ref(skb, i); 6576 if (skb_has_frag_list(skb)) 6577 skb_clone_fraglist(skb); 6578 skb_release_data(skb, SKB_CONSUMED, false); 6579 } else { 6580 /* we can reuse existing recount- all we did was 6581 * relocate values 6582 */ 6583 skb_free_head(skb, false); 6584 } 6585 6586 skb->head = data; 6587 skb->data = data; 6588 skb->head_frag = 0; 6589 skb_set_end_offset(skb, size); 6590 skb_set_tail_pointer(skb, skb_headlen(skb)); 6591 skb_headers_offset_update(skb, 0); 6592 skb->cloned = 0; 6593 skb->hdr_len = 0; 6594 skb->nohdr = 0; 6595 atomic_set(&skb_shinfo(skb)->dataref, 1); 6596 6597 return 0; 6598} 6599 6600static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp); 6601 6602/* carve out the first eat bytes from skb's frag_list. May recurse into 6603 * pskb_carve() 6604 */ 6605static int pskb_carve_frag_list(struct sk_buff *skb, 6606 struct skb_shared_info *shinfo, int eat, 6607 gfp_t gfp_mask) 6608{ 6609 struct sk_buff *list = shinfo->frag_list; 6610 struct sk_buff *clone = NULL; 6611 struct sk_buff *insp = NULL; 6612 6613 do { 6614 if (!list) { 6615 pr_err("Not enough bytes to eat. Want %d\n", eat); 6616 return -EFAULT; 6617 } 6618 if (list->len <= eat) { 6619 /* Eaten as whole. */ 6620 eat -= list->len; 6621 list = list->next; 6622 insp = list; 6623 } else { 6624 /* Eaten partially. */ 6625 if (skb_shared(list)) { 6626 clone = skb_clone(list, gfp_mask); 6627 if (!clone) 6628 return -ENOMEM; 6629 insp = list->next; 6630 list = clone; 6631 } else { 6632 /* This may be pulled without problems. */ 6633 insp = list; 6634 } 6635 if (pskb_carve(list, eat, gfp_mask) < 0) { 6636 kfree_skb(clone); 6637 return -ENOMEM; 6638 } 6639 break; 6640 } 6641 } while (eat); 6642 6643 /* Free pulled out fragments. */ 6644 while ((list = shinfo->frag_list) != insp) { 6645 shinfo->frag_list = list->next; 6646 consume_skb(list); 6647 } 6648 /* And insert new clone at head. */ 6649 if (clone) { 6650 clone->next = list; 6651 shinfo->frag_list = clone; 6652 } 6653 return 0; 6654} 6655 6656/* carve off first len bytes from skb. Split line (off) is in the 6657 * non-linear part of skb 6658 */ 6659static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off, 6660 int pos, gfp_t gfp_mask) 6661{ 6662 int i, k = 0; 6663 unsigned int size = skb_end_offset(skb); 6664 u8 *data; 6665 const int nfrags = skb_shinfo(skb)->nr_frags; 6666 struct skb_shared_info *shinfo; 6667 6668 if (skb_pfmemalloc(skb)) 6669 gfp_mask |= __GFP_MEMALLOC; 6670 6671 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL); 6672 if (!data) 6673 return -ENOMEM; 6674 size = SKB_WITH_OVERHEAD(size); 6675 6676 memcpy((struct skb_shared_info *)(data + size), 6677 skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0])); 6678 if (skb_orphan_frags(skb, gfp_mask)) { 6679 skb_kfree_head(data, size); 6680 return -ENOMEM; 6681 } 6682 shinfo = (struct skb_shared_info *)(data + size); 6683 for (i = 0; i < nfrags; i++) { 6684 int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]); 6685 6686 if (pos + fsize > off) { 6687 shinfo->frags[k] = skb_shinfo(skb)->frags[i]; 6688 6689 if (pos < off) { 6690 /* Split frag. 6691 * We have two variants in this case: 6692 * 1. Move all the frag to the second 6693 * part, if it is possible. F.e. 6694 * this approach is mandatory for TUX, 6695 * where splitting is expensive. 6696 * 2. Split is accurately. We make this. 6697 */ 6698 skb_frag_off_add(&shinfo->frags[0], off - pos); 6699 skb_frag_size_sub(&shinfo->frags[0], off - pos); 6700 } 6701 skb_frag_ref(skb, i); 6702 k++; 6703 } 6704 pos += fsize; 6705 } 6706 shinfo->nr_frags = k; 6707 if (skb_has_frag_list(skb)) 6708 skb_clone_fraglist(skb); 6709 6710 /* split line is in frag list */ 6711 if (k == 0 && pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask)) { 6712 /* skb_frag_unref() is not needed here as shinfo->nr_frags = 0. */ 6713 if (skb_has_frag_list(skb)) 6714 kfree_skb_list(skb_shinfo(skb)->frag_list); 6715 skb_kfree_head(data, size); 6716 return -ENOMEM; 6717 } 6718 skb_release_data(skb, SKB_CONSUMED, false); 6719 6720 skb->head = data; 6721 skb->head_frag = 0; 6722 skb->data = data; 6723 skb_set_end_offset(skb, size); 6724 skb_reset_tail_pointer(skb); 6725 skb_headers_offset_update(skb, 0); 6726 skb->cloned = 0; 6727 skb->hdr_len = 0; 6728 skb->nohdr = 0; 6729 skb->len -= off; 6730 skb->data_len = skb->len; 6731 atomic_set(&skb_shinfo(skb)->dataref, 1); 6732 return 0; 6733} 6734 6735/* remove len bytes from the beginning of the skb */ 6736static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp) 6737{ 6738 int headlen = skb_headlen(skb); 6739 6740 if (len < headlen) 6741 return pskb_carve_inside_header(skb, len, headlen, gfp); 6742 else 6743 return pskb_carve_inside_nonlinear(skb, len, headlen, gfp); 6744} 6745 6746/* Extract to_copy bytes starting at off from skb, and return this in 6747 * a new skb 6748 */ 6749struct sk_buff *pskb_extract(struct sk_buff *skb, int off, 6750 int to_copy, gfp_t gfp) 6751{ 6752 struct sk_buff *clone = skb_clone(skb, gfp); 6753 6754 if (!clone) 6755 return NULL; 6756 6757 if (pskb_carve(clone, off, gfp) < 0 || 6758 pskb_trim(clone, to_copy)) { 6759 kfree_skb(clone); 6760 return NULL; 6761 } 6762 return clone; 6763} 6764EXPORT_SYMBOL(pskb_extract); 6765 6766/** 6767 * skb_condense - try to get rid of fragments/frag_list if possible 6768 * @skb: buffer 6769 * 6770 * Can be used to save memory before skb is added to a busy queue. 6771 * If packet has bytes in frags and enough tail room in skb->head, 6772 * pull all of them, so that we can free the frags right now and adjust 6773 * truesize. 6774 * Notes: 6775 * We do not reallocate skb->head thus can not fail. 6776 * Caller must re-evaluate skb->truesize if needed. 6777 */ 6778void skb_condense(struct sk_buff *skb) 6779{ 6780 if (skb->data_len) { 6781 if (skb->data_len > skb->end - skb->tail || 6782 skb_cloned(skb)) 6783 return; 6784 6785 /* Nice, we can free page frag(s) right now */ 6786 __pskb_pull_tail(skb, skb->data_len); 6787 } 6788 /* At this point, skb->truesize might be over estimated, 6789 * because skb had a fragment, and fragments do not tell 6790 * their truesize. 6791 * When we pulled its content into skb->head, fragment 6792 * was freed, but __pskb_pull_tail() could not possibly 6793 * adjust skb->truesize, not knowing the frag truesize. 6794 */ 6795 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 6796} 6797EXPORT_SYMBOL(skb_condense); 6798 6799#ifdef CONFIG_SKB_EXTENSIONS 6800static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id) 6801{ 6802 return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE); 6803} 6804 6805/** 6806 * __skb_ext_alloc - allocate a new skb extensions storage 6807 * 6808 * @flags: See kmalloc(). 6809 * 6810 * Returns the newly allocated pointer. The pointer can later attached to a 6811 * skb via __skb_ext_set(). 6812 * Note: caller must handle the skb_ext as an opaque data. 6813 */ 6814struct skb_ext *__skb_ext_alloc(gfp_t flags) 6815{ 6816 struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, flags); 6817 6818 if (new) { 6819 memset(new->offset, 0, sizeof(new->offset)); 6820 refcount_set(&new->refcnt, 1); 6821 } 6822 6823 return new; 6824} 6825 6826static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old, 6827 unsigned int old_active) 6828{ 6829 struct skb_ext *new; 6830 6831 if (refcount_read(&old->refcnt) == 1) 6832 return old; 6833 6834 new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC); 6835 if (!new) 6836 return NULL; 6837 6838 memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE); 6839 refcount_set(&new->refcnt, 1); 6840 6841#ifdef CONFIG_XFRM 6842 if (old_active & (1 << SKB_EXT_SEC_PATH)) { 6843 struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH); 6844 unsigned int i; 6845 6846 for (i = 0; i < sp->len; i++) 6847 xfrm_state_hold(sp->xvec[i]); 6848 } 6849#endif 6850#ifdef CONFIG_MCTP_FLOWS 6851 if (old_active & (1 << SKB_EXT_MCTP)) { 6852 struct mctp_flow *flow = skb_ext_get_ptr(old, SKB_EXT_MCTP); 6853 6854 if (flow->key) 6855 refcount_inc(&flow->key->refs); 6856 } 6857#endif 6858 __skb_ext_put(old); 6859 return new; 6860} 6861 6862/** 6863 * __skb_ext_set - attach the specified extension storage to this skb 6864 * @skb: buffer 6865 * @id: extension id 6866 * @ext: extension storage previously allocated via __skb_ext_alloc() 6867 * 6868 * Existing extensions, if any, are cleared. 6869 * 6870 * Returns the pointer to the extension. 6871 */ 6872void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id, 6873 struct skb_ext *ext) 6874{ 6875 unsigned int newlen, newoff = SKB_EXT_CHUNKSIZEOF(*ext); 6876 6877 skb_ext_put(skb); 6878 newlen = newoff + skb_ext_type_len[id]; 6879 ext->chunks = newlen; 6880 ext->offset[id] = newoff; 6881 skb->extensions = ext; 6882 skb->active_extensions = 1 << id; 6883 return skb_ext_get_ptr(ext, id); 6884} 6885 6886/** 6887 * skb_ext_add - allocate space for given extension, COW if needed 6888 * @skb: buffer 6889 * @id: extension to allocate space for 6890 * 6891 * Allocates enough space for the given extension. 6892 * If the extension is already present, a pointer to that extension 6893 * is returned. 6894 * 6895 * If the skb was cloned, COW applies and the returned memory can be 6896 * modified without changing the extension space of clones buffers. 6897 * 6898 * Returns pointer to the extension or NULL on allocation failure. 6899 */ 6900void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id) 6901{ 6902 struct skb_ext *new, *old = NULL; 6903 unsigned int newlen, newoff; 6904 6905 if (skb->active_extensions) { 6906 old = skb->extensions; 6907 6908 new = skb_ext_maybe_cow(old, skb->active_extensions); 6909 if (!new) 6910 return NULL; 6911 6912 if (__skb_ext_exist(new, id)) 6913 goto set_active; 6914 6915 newoff = new->chunks; 6916 } else { 6917 newoff = SKB_EXT_CHUNKSIZEOF(*new); 6918 6919 new = __skb_ext_alloc(GFP_ATOMIC); 6920 if (!new) 6921 return NULL; 6922 } 6923 6924 newlen = newoff + skb_ext_type_len[id]; 6925 new->chunks = newlen; 6926 new->offset[id] = newoff; 6927set_active: 6928 skb->slow_gro = 1; 6929 skb->extensions = new; 6930 skb->active_extensions |= 1 << id; 6931 return skb_ext_get_ptr(new, id); 6932} 6933EXPORT_SYMBOL(skb_ext_add); 6934 6935#ifdef CONFIG_XFRM 6936static void skb_ext_put_sp(struct sec_path *sp) 6937{ 6938 unsigned int i; 6939 6940 for (i = 0; i < sp->len; i++) 6941 xfrm_state_put(sp->xvec[i]); 6942} 6943#endif 6944 6945#ifdef CONFIG_MCTP_FLOWS 6946static void skb_ext_put_mctp(struct mctp_flow *flow) 6947{ 6948 if (flow->key) 6949 mctp_key_unref(flow->key); 6950} 6951#endif 6952 6953void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id) 6954{ 6955 struct skb_ext *ext = skb->extensions; 6956 6957 skb->active_extensions &= ~(1 << id); 6958 if (skb->active_extensions == 0) { 6959 skb->extensions = NULL; 6960 __skb_ext_put(ext); 6961#ifdef CONFIG_XFRM 6962 } else if (id == SKB_EXT_SEC_PATH && 6963 refcount_read(&ext->refcnt) == 1) { 6964 struct sec_path *sp = skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH); 6965 6966 skb_ext_put_sp(sp); 6967 sp->len = 0; 6968#endif 6969 } 6970} 6971EXPORT_SYMBOL(__skb_ext_del); 6972 6973void __skb_ext_put(struct skb_ext *ext) 6974{ 6975 /* If this is last clone, nothing can increment 6976 * it after check passes. Avoids one atomic op. 6977 */ 6978 if (refcount_read(&ext->refcnt) == 1) 6979 goto free_now; 6980 6981 if (!refcount_dec_and_test(&ext->refcnt)) 6982 return; 6983free_now: 6984#ifdef CONFIG_XFRM 6985 if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH)) 6986 skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH)); 6987#endif 6988#ifdef CONFIG_MCTP_FLOWS 6989 if (__skb_ext_exist(ext, SKB_EXT_MCTP)) 6990 skb_ext_put_mctp(skb_ext_get_ptr(ext, SKB_EXT_MCTP)); 6991#endif 6992 6993 kmem_cache_free(skbuff_ext_cache, ext); 6994} 6995EXPORT_SYMBOL(__skb_ext_put); 6996#endif /* CONFIG_SKB_EXTENSIONS */ 6997 6998/** 6999 * skb_attempt_defer_free - queue skb for remote freeing 7000 * @skb: buffer 7001 * 7002 * Put @skb in a per-cpu list, using the cpu which 7003 * allocated the skb/pages to reduce false sharing 7004 * and memory zone spinlock contention. 7005 */ 7006void skb_attempt_defer_free(struct sk_buff *skb) 7007{ 7008 int cpu = skb->alloc_cpu; 7009 struct softnet_data *sd; 7010 unsigned int defer_max; 7011 bool kick; 7012 7013 if (WARN_ON_ONCE(cpu >= nr_cpu_ids) || 7014 !cpu_online(cpu) || 7015 cpu == raw_smp_processor_id()) { 7016nodefer: __kfree_skb(skb); 7017 return; 7018 } 7019 7020 DEBUG_NET_WARN_ON_ONCE(skb_dst(skb)); 7021 DEBUG_NET_WARN_ON_ONCE(skb->destructor); 7022 7023 sd = &per_cpu(softnet_data, cpu); 7024 defer_max = READ_ONCE(sysctl_skb_defer_max); 7025 if (READ_ONCE(sd->defer_count) >= defer_max) 7026 goto nodefer; 7027 7028 spin_lock_bh(&sd->defer_lock); 7029 /* Send an IPI every time queue reaches half capacity. */ 7030 kick = sd->defer_count == (defer_max >> 1); 7031 /* Paired with the READ_ONCE() few lines above */ 7032 WRITE_ONCE(sd->defer_count, sd->defer_count + 1); 7033 7034 skb->next = sd->defer_list; 7035 /* Paired with READ_ONCE() in skb_defer_free_flush() */ 7036 WRITE_ONCE(sd->defer_list, skb); 7037 spin_unlock_bh(&sd->defer_lock); 7038 7039 /* Make sure to trigger NET_RX_SOFTIRQ on the remote CPU 7040 * if we are unlucky enough (this seems very unlikely). 7041 */ 7042 if (unlikely(kick) && !cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) 7043 smp_call_function_single_async(cpu, &sd->defer_csd); 7044} 7045 7046static void skb_splice_csum_page(struct sk_buff *skb, struct page *page, 7047 size_t offset, size_t len) 7048{ 7049 const char *kaddr; 7050 __wsum csum; 7051 7052 kaddr = kmap_local_page(page); 7053 csum = csum_partial(kaddr + offset, len, 0); 7054 kunmap_local(kaddr); 7055 skb->csum = csum_block_add(skb->csum, csum, skb->len); 7056} 7057 7058/** 7059 * skb_splice_from_iter - Splice (or copy) pages to skbuff 7060 * @skb: The buffer to add pages to 7061 * @iter: Iterator representing the pages to be added 7062 * @maxsize: Maximum amount of pages to be added 7063 * @gfp: Allocation flags 7064 * 7065 * This is a common helper function for supporting MSG_SPLICE_PAGES. It 7066 * extracts pages from an iterator and adds them to the socket buffer if 7067 * possible, copying them to fragments if not possible (such as if they're slab 7068 * pages). 7069 * 7070 * Returns the amount of data spliced/copied or -EMSGSIZE if there's 7071 * insufficient space in the buffer to transfer anything. 7072 */ 7073ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, 7074 ssize_t maxsize, gfp_t gfp) 7075{ 7076 size_t frag_limit = READ_ONCE(sysctl_max_skb_frags); 7077 struct page *pages[8], **ppages = pages; 7078 ssize_t spliced = 0, ret = 0; 7079 unsigned int i; 7080 7081 while (iter->count > 0) { 7082 ssize_t space, nr, len; 7083 size_t off; 7084 7085 ret = -EMSGSIZE; 7086 space = frag_limit - skb_shinfo(skb)->nr_frags; 7087 if (space < 0) 7088 break; 7089 7090 /* We might be able to coalesce without increasing nr_frags */ 7091 nr = clamp_t(size_t, space, 1, ARRAY_SIZE(pages)); 7092 7093 len = iov_iter_extract_pages(iter, &ppages, maxsize, nr, 0, &off); 7094 if (len <= 0) { 7095 ret = len ?: -EIO; 7096 break; 7097 } 7098 7099 i = 0; 7100 do { 7101 struct page *page = pages[i++]; 7102 size_t part = min_t(size_t, PAGE_SIZE - off, len); 7103 7104 ret = -EIO; 7105 if (WARN_ON_ONCE(!sendpage_ok(page))) 7106 goto out; 7107 7108 ret = skb_append_pagefrags(skb, page, off, part, 7109 frag_limit); 7110 if (ret < 0) { 7111 iov_iter_revert(iter, len); 7112 goto out; 7113 } 7114 7115 if (skb->ip_summed == CHECKSUM_NONE) 7116 skb_splice_csum_page(skb, page, off, part); 7117 7118 off = 0; 7119 spliced += part; 7120 maxsize -= part; 7121 len -= part; 7122 } while (len > 0); 7123 7124 if (maxsize <= 0) 7125 break; 7126 } 7127 7128out: 7129 skb_len_add(skb, spliced); 7130 return spliced ?: ret; 7131} 7132EXPORT_SYMBOL(skb_splice_from_iter); 7133 7134static __always_inline 7135size_t memcpy_from_iter_csum(void *iter_from, size_t progress, 7136 size_t len, void *to, void *priv2) 7137{ 7138 __wsum *csum = priv2; 7139 __wsum next = csum_partial_copy_nocheck(iter_from, to + progress, len); 7140 7141 *csum = csum_block_add(*csum, next, progress); 7142 return 0; 7143} 7144 7145static __always_inline 7146size_t copy_from_user_iter_csum(void __user *iter_from, size_t progress, 7147 size_t len, void *to, void *priv2) 7148{ 7149 __wsum next, *csum = priv2; 7150 7151 next = csum_and_copy_from_user(iter_from, to + progress, len); 7152 *csum = csum_block_add(*csum, next, progress); 7153 return next ? 0 : len; 7154} 7155 7156bool csum_and_copy_from_iter_full(void *addr, size_t bytes, 7157 __wsum *csum, struct iov_iter *i) 7158{ 7159 size_t copied; 7160 7161 if (WARN_ON_ONCE(!i->data_source)) 7162 return false; 7163 copied = iterate_and_advance2(i, bytes, addr, csum, 7164 copy_from_user_iter_csum, 7165 memcpy_from_iter_csum); 7166 if (likely(copied == bytes)) 7167 return true; 7168 iov_iter_revert(i, copied); 7169 return false; 7170} 7171EXPORT_SYMBOL(csum_and_copy_from_iter_full); 7172