kern_mbuf.c (295221) | kern_mbuf.c (295222) |
---|---|
1/*- 2 * Copyright (c) 2004, 2005, 3 * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 12 unchanged lines hidden (view full) --- 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2004, 2005, 3 * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 12 unchanged lines hidden (view full) --- 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> |
29__FBSDID("$FreeBSD: head/sys/kern/kern_mbuf.c 295221 2016-02-03 22:02:36Z glebius $"); | 29__FBSDID("$FreeBSD: head/sys/kern/kern_mbuf.c 295222 2016-02-03 23:30:17Z glebius $"); |
30 31#include "opt_param.h" 32 33#include <sys/param.h> 34#include <sys/malloc.h> 35#include <sys/types.h> 36#include <sys/systm.h> 37#include <sys/mbuf.h> --- 231 unchanged lines hidden (view full) --- 269uma_zone_t zone_clust; 270uma_zone_t zone_pack; 271uma_zone_t zone_jumbop; 272uma_zone_t zone_jumbo9; 273uma_zone_t zone_jumbo16; 274uma_zone_t zone_ext_refcnt; 275 276/* | 30 31#include "opt_param.h" 32 33#include <sys/param.h> 34#include <sys/malloc.h> 35#include <sys/types.h> 36#include <sys/systm.h> 37#include <sys/mbuf.h> --- 231 unchanged lines hidden (view full) --- 269uma_zone_t zone_clust; 270uma_zone_t zone_pack; 271uma_zone_t zone_jumbop; 272uma_zone_t zone_jumbo9; 273uma_zone_t zone_jumbo16; 274uma_zone_t zone_ext_refcnt; 275 276/* |
277 * Callout to assist us in freeing mbufs. 278 */ 279static struct callout mb_reclaim_callout; 280static struct mtx mb_reclaim_callout_mtx; 281 282/* | |
283 * Local prototypes. 284 */ 285static int mb_ctor_mbuf(void *, int, void *, int); 286static int mb_ctor_clust(void *, int, void *, int); 287static int mb_ctor_pack(void *, int, void *, int); 288static void mb_dtor_mbuf(void *, int, void *); 289static void mb_dtor_clust(void *, int, void *); 290static void mb_dtor_pack(void *, int, void *); 291static int mb_zinit_pack(void *, int, int); 292static void mb_zfini_pack(void *, int); 293 | 277 * Local prototypes. 278 */ 279static int mb_ctor_mbuf(void *, int, void *, int); 280static int mb_ctor_clust(void *, int, void *, int); 281static int mb_ctor_pack(void *, int, void *, int); 282static void mb_dtor_mbuf(void *, int, void *); 283static void mb_dtor_clust(void *, int, void *); 284static void mb_dtor_pack(void *, int, void *); 285static int mb_zinit_pack(void *, int, int); 286static void mb_zfini_pack(void *, int); 287 |
294static void mb_reclaim(void *); | 288static void mb_reclaim(uma_zone_t, int); |
295static void *mbuf_jumbo_alloc(uma_zone_t, vm_size_t, uint8_t *, int); | 289static void *mbuf_jumbo_alloc(uma_zone_t, vm_size_t, uint8_t *, int); |
296static void mb_maxaction(uma_zone_t); | |
297 298/* Ensure that MSIZE is a power of 2. */ 299CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE); 300 301/* 302 * Initialize FreeBSD Network buffer allocation. 303 */ 304static void --- 9 unchanged lines hidden (view full) --- 314 trash_init, trash_fini, 315#else 316 NULL, NULL, 317#endif 318 MSIZE - 1, UMA_ZONE_MAXBUCKET); 319 if (nmbufs > 0) 320 nmbufs = uma_zone_set_max(zone_mbuf, nmbufs); 321 uma_zone_set_warning(zone_mbuf, "kern.ipc.nmbufs limit reached"); | 290 291/* Ensure that MSIZE is a power of 2. */ 292CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE); 293 294/* 295 * Initialize FreeBSD Network buffer allocation. 296 */ 297static void --- 9 unchanged lines hidden (view full) --- 307 trash_init, trash_fini, 308#else 309 NULL, NULL, 310#endif 311 MSIZE - 1, UMA_ZONE_MAXBUCKET); 312 if (nmbufs > 0) 313 nmbufs = uma_zone_set_max(zone_mbuf, nmbufs); 314 uma_zone_set_warning(zone_mbuf, "kern.ipc.nmbufs limit reached"); |
322 uma_zone_set_maxaction(zone_mbuf, mb_maxaction); | 315 uma_zone_set_maxaction(zone_mbuf, mb_reclaim); |
323 324 zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES, 325 mb_ctor_clust, mb_dtor_clust, 326#ifdef INVARIANTS 327 trash_init, trash_fini, 328#else 329 NULL, NULL, 330#endif 331 UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 332 if (nmbclusters > 0) 333 nmbclusters = uma_zone_set_max(zone_clust, nmbclusters); 334 uma_zone_set_warning(zone_clust, "kern.ipc.nmbclusters limit reached"); | 316 317 zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES, 318 mb_ctor_clust, mb_dtor_clust, 319#ifdef INVARIANTS 320 trash_init, trash_fini, 321#else 322 NULL, NULL, 323#endif 324 UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 325 if (nmbclusters > 0) 326 nmbclusters = uma_zone_set_max(zone_clust, nmbclusters); 327 uma_zone_set_warning(zone_clust, "kern.ipc.nmbclusters limit reached"); |
335 uma_zone_set_maxaction(zone_clust, mb_maxaction); | 328 uma_zone_set_maxaction(zone_clust, mb_reclaim); |
336 337 zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack, 338 mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf); 339 340 /* Make jumbo frame zone too. Page size, 9k and 16k. */ 341 zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE, 342 mb_ctor_clust, mb_dtor_clust, 343#ifdef INVARIANTS 344 trash_init, trash_fini, 345#else 346 NULL, NULL, 347#endif 348 UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 349 if (nmbjumbop > 0) 350 nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop); 351 uma_zone_set_warning(zone_jumbop, "kern.ipc.nmbjumbop limit reached"); | 329 330 zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack, 331 mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf); 332 333 /* Make jumbo frame zone too. Page size, 9k and 16k. */ 334 zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE, 335 mb_ctor_clust, mb_dtor_clust, 336#ifdef INVARIANTS 337 trash_init, trash_fini, 338#else 339 NULL, NULL, 340#endif 341 UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 342 if (nmbjumbop > 0) 343 nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop); 344 uma_zone_set_warning(zone_jumbop, "kern.ipc.nmbjumbop limit reached"); |
352 uma_zone_set_maxaction(zone_jumbop, mb_maxaction); | 345 uma_zone_set_maxaction(zone_jumbop, mb_reclaim); |
353 354 zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES, 355 mb_ctor_clust, mb_dtor_clust, 356#ifdef INVARIANTS 357 trash_init, trash_fini, 358#else 359 NULL, NULL, 360#endif 361 UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 362 uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc); 363 if (nmbjumbo9 > 0) 364 nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9); 365 uma_zone_set_warning(zone_jumbo9, "kern.ipc.nmbjumbo9 limit reached"); | 346 347 zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES, 348 mb_ctor_clust, mb_dtor_clust, 349#ifdef INVARIANTS 350 trash_init, trash_fini, 351#else 352 NULL, NULL, 353#endif 354 UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 355 uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc); 356 if (nmbjumbo9 > 0) 357 nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9); 358 uma_zone_set_warning(zone_jumbo9, "kern.ipc.nmbjumbo9 limit reached"); |
366 uma_zone_set_maxaction(zone_jumbo9, mb_maxaction); | 359 uma_zone_set_maxaction(zone_jumbo9, mb_reclaim); |
367 368 zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES, 369 mb_ctor_clust, mb_dtor_clust, 370#ifdef INVARIANTS 371 trash_init, trash_fini, 372#else 373 NULL, NULL, 374#endif 375 UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 376 uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc); 377 if (nmbjumbo16 > 0) 378 nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16); 379 uma_zone_set_warning(zone_jumbo16, "kern.ipc.nmbjumbo16 limit reached"); | 360 361 zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES, 362 mb_ctor_clust, mb_dtor_clust, 363#ifdef INVARIANTS 364 trash_init, trash_fini, 365#else 366 NULL, NULL, 367#endif 368 UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 369 uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc); 370 if (nmbjumbo16 > 0) 371 nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16); 372 uma_zone_set_warning(zone_jumbo16, "kern.ipc.nmbjumbo16 limit reached"); |
380 uma_zone_set_maxaction(zone_jumbo16, mb_maxaction); | 373 uma_zone_set_maxaction(zone_jumbo16, mb_reclaim); |
381 382 zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int), 383 NULL, NULL, 384 NULL, NULL, 385 UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 386 | 374 375 zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int), 376 NULL, NULL, 377 NULL, NULL, 378 UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 379 |
387 /* uma_prealloc() goes here... */ 388 389 /* Initialize the mb_reclaim() callout. */ 390 mtx_init(&mb_reclaim_callout_mtx, "mb_reclaim_callout_mtx", NULL, 391 MTX_DEF); 392 callout_init(&mb_reclaim_callout, 1); 393 | |
394 /* 395 * Hook event handler for low-memory situation, used to 396 * drain protocols and push data back to the caches (UMA 397 * later pushes it back to VM). 398 */ 399 EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL, 400 EVENTHANDLER_PRI_FIRST); 401} --- 270 unchanged lines hidden (view full) --- 672 if (error) 673 return (error); 674#endif 675 676 return (0); 677} 678 679/* | 380 /* 381 * Hook event handler for low-memory situation, used to 382 * drain protocols and push data back to the caches (UMA 383 * later pushes it back to VM). 384 */ 385 EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL, 386 EVENTHANDLER_PRI_FIRST); 387} --- 270 unchanged lines hidden (view full) --- 658 if (error) 659 return (error); 660#endif 661 662 return (0); 663} 664 665/* |
680 * This is the protocol drain routine. | 666 * This is the protocol drain routine. Called by UMA whenever any of the 667 * mbuf zones is closed to its limit. |
681 * 682 * No locks should be held when this is called. The drain routines have to 683 * presently acquire some locks which raises the possibility of lock order 684 * reversal. 685 */ 686static void | 668 * 669 * No locks should be held when this is called. The drain routines have to 670 * presently acquire some locks which raises the possibility of lock order 671 * reversal. 672 */ 673static void |
687mb_reclaim(void *junk) | 674mb_reclaim(uma_zone_t zone __unused, int pending __unused) |
688{ 689 struct domain *dp; 690 struct protosw *pr; 691 | 675{ 676 struct domain *dp; 677 struct protosw *pr; 678 |
692 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL, 693 "mb_reclaim()"); | 679 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL, __func__); |
694 695 for (dp = domains; dp != NULL; dp = dp->dom_next) 696 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) 697 if (pr->pr_drain != NULL) 698 (*pr->pr_drain)(); 699} | 680 681 for (dp = domains; dp != NULL; dp = dp->dom_next) 682 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) 683 if (pr->pr_drain != NULL) 684 (*pr->pr_drain)(); 685} |
700 701/* 702 * This is the function called by the mb_reclaim_callout, which is 703 * used when we hit the maximum for a zone. 704 * 705 * (See mb_maxaction() below.) 706 */ 707static void 708mb_reclaim_timer(void *junk __unused) 709{ 710 711 mtx_lock(&mb_reclaim_callout_mtx); 712 713 /* 714 * Avoid running this function extra times by skipping this invocation 715 * if the callout has already been rescheduled. 716 */ 717 if (callout_pending(&mb_reclaim_callout) || 718 !callout_active(&mb_reclaim_callout)) { 719 mtx_unlock(&mb_reclaim_callout_mtx); 720 return; 721 } 722 mtx_unlock(&mb_reclaim_callout_mtx); 723 724 mb_reclaim(NULL); 725 726 mtx_lock(&mb_reclaim_callout_mtx); 727 callout_deactivate(&mb_reclaim_callout); 728 mtx_unlock(&mb_reclaim_callout_mtx); 729} 730 731/* 732 * This function is called when we hit the maximum for a zone. 733 * 734 * At that point, we want to call the protocol drain routine to free up some 735 * mbufs. However, we will use the callout routines to schedule this to 736 * occur in another thread. (The thread calling this function holds the 737 * zone lock.) 738 */ 739static void 740mb_maxaction(uma_zone_t zone __unused) 741{ 742 743 /* 744 * If we can't immediately obtain the lock, either the callout 745 * is currently running, or another thread is scheduling the 746 * callout. 747 */ 748 if (!mtx_trylock(&mb_reclaim_callout_mtx)) 749 return; 750 751 /* If not already scheduled/running, schedule the callout. */ 752 if (!callout_active(&mb_reclaim_callout)) { 753 callout_reset(&mb_reclaim_callout, 1, mb_reclaim_timer, NULL); 754 } 755 756 mtx_unlock(&mb_reclaim_callout_mtx); 757} | |