Deleted Added
sdiff udiff text old ( 295221 ) new ( 295222 )
full compact
kern_mbuf.c (295221) kern_mbuf.c (295222)
1/*-
2 * Copyright (c) 2004, 2005,
3 * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2004, 2005,
3 * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: head/sys/kern/kern_mbuf.c 295221 2016-02-03 22:02:36Z glebius $");
29__FBSDID("$FreeBSD: head/sys/kern/kern_mbuf.c 295222 2016-02-03 23:30:17Z glebius $");
30
31#include "opt_param.h"
32
33#include <sys/param.h>
34#include <sys/malloc.h>
35#include <sys/types.h>
36#include <sys/systm.h>
37#include <sys/mbuf.h>

--- 231 unchanged lines hidden (view full) ---

269uma_zone_t zone_clust;
270uma_zone_t zone_pack;
271uma_zone_t zone_jumbop;
272uma_zone_t zone_jumbo9;
273uma_zone_t zone_jumbo16;
274uma_zone_t zone_ext_refcnt;
275
276/*
30
31#include "opt_param.h"
32
33#include <sys/param.h>
34#include <sys/malloc.h>
35#include <sys/types.h>
36#include <sys/systm.h>
37#include <sys/mbuf.h>

--- 231 unchanged lines hidden (view full) ---

269uma_zone_t zone_clust;
270uma_zone_t zone_pack;
271uma_zone_t zone_jumbop;
272uma_zone_t zone_jumbo9;
273uma_zone_t zone_jumbo16;
274uma_zone_t zone_ext_refcnt;
275
276/*
277 * Callout to assist us in freeing mbufs.
278 */
279static struct callout mb_reclaim_callout;
280static struct mtx mb_reclaim_callout_mtx;
281
282/*
283 * Local prototypes.
284 */
285static int mb_ctor_mbuf(void *, int, void *, int);
286static int mb_ctor_clust(void *, int, void *, int);
287static int mb_ctor_pack(void *, int, void *, int);
288static void mb_dtor_mbuf(void *, int, void *);
289static void mb_dtor_clust(void *, int, void *);
290static void mb_dtor_pack(void *, int, void *);
291static int mb_zinit_pack(void *, int, int);
292static void mb_zfini_pack(void *, int);
293
277 * Local prototypes.
278 */
279static int mb_ctor_mbuf(void *, int, void *, int);
280static int mb_ctor_clust(void *, int, void *, int);
281static int mb_ctor_pack(void *, int, void *, int);
282static void mb_dtor_mbuf(void *, int, void *);
283static void mb_dtor_clust(void *, int, void *);
284static void mb_dtor_pack(void *, int, void *);
285static int mb_zinit_pack(void *, int, int);
286static void mb_zfini_pack(void *, int);
287
294static void mb_reclaim(void *);
288static void mb_reclaim(uma_zone_t, int);
295static void *mbuf_jumbo_alloc(uma_zone_t, vm_size_t, uint8_t *, int);
289static void *mbuf_jumbo_alloc(uma_zone_t, vm_size_t, uint8_t *, int);
296static void mb_maxaction(uma_zone_t);
297
298/* Ensure that MSIZE is a power of 2. */
299CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
300
301/*
302 * Initialize FreeBSD Network buffer allocation.
303 */
304static void

--- 9 unchanged lines hidden (view full) ---

314 trash_init, trash_fini,
315#else
316 NULL, NULL,
317#endif
318 MSIZE - 1, UMA_ZONE_MAXBUCKET);
319 if (nmbufs > 0)
320 nmbufs = uma_zone_set_max(zone_mbuf, nmbufs);
321 uma_zone_set_warning(zone_mbuf, "kern.ipc.nmbufs limit reached");
290
291/* Ensure that MSIZE is a power of 2. */
292CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
293
294/*
295 * Initialize FreeBSD Network buffer allocation.
296 */
297static void

--- 9 unchanged lines hidden (view full) ---

307 trash_init, trash_fini,
308#else
309 NULL, NULL,
310#endif
311 MSIZE - 1, UMA_ZONE_MAXBUCKET);
312 if (nmbufs > 0)
313 nmbufs = uma_zone_set_max(zone_mbuf, nmbufs);
314 uma_zone_set_warning(zone_mbuf, "kern.ipc.nmbufs limit reached");
322 uma_zone_set_maxaction(zone_mbuf, mb_maxaction);
315 uma_zone_set_maxaction(zone_mbuf, mb_reclaim);
323
324 zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
325 mb_ctor_clust, mb_dtor_clust,
326#ifdef INVARIANTS
327 trash_init, trash_fini,
328#else
329 NULL, NULL,
330#endif
331 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
332 if (nmbclusters > 0)
333 nmbclusters = uma_zone_set_max(zone_clust, nmbclusters);
334 uma_zone_set_warning(zone_clust, "kern.ipc.nmbclusters limit reached");
316
317 zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
318 mb_ctor_clust, mb_dtor_clust,
319#ifdef INVARIANTS
320 trash_init, trash_fini,
321#else
322 NULL, NULL,
323#endif
324 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
325 if (nmbclusters > 0)
326 nmbclusters = uma_zone_set_max(zone_clust, nmbclusters);
327 uma_zone_set_warning(zone_clust, "kern.ipc.nmbclusters limit reached");
335 uma_zone_set_maxaction(zone_clust, mb_maxaction);
328 uma_zone_set_maxaction(zone_clust, mb_reclaim);
336
337 zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
338 mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);
339
340 /* Make jumbo frame zone too. Page size, 9k and 16k. */
341 zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE,
342 mb_ctor_clust, mb_dtor_clust,
343#ifdef INVARIANTS
344 trash_init, trash_fini,
345#else
346 NULL, NULL,
347#endif
348 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
349 if (nmbjumbop > 0)
350 nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop);
351 uma_zone_set_warning(zone_jumbop, "kern.ipc.nmbjumbop limit reached");
329
330 zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
331 mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);
332
333 /* Make jumbo frame zone too. Page size, 9k and 16k. */
334 zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE,
335 mb_ctor_clust, mb_dtor_clust,
336#ifdef INVARIANTS
337 trash_init, trash_fini,
338#else
339 NULL, NULL,
340#endif
341 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
342 if (nmbjumbop > 0)
343 nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop);
344 uma_zone_set_warning(zone_jumbop, "kern.ipc.nmbjumbop limit reached");
352 uma_zone_set_maxaction(zone_jumbop, mb_maxaction);
345 uma_zone_set_maxaction(zone_jumbop, mb_reclaim);
353
354 zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
355 mb_ctor_clust, mb_dtor_clust,
356#ifdef INVARIANTS
357 trash_init, trash_fini,
358#else
359 NULL, NULL,
360#endif
361 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
362 uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc);
363 if (nmbjumbo9 > 0)
364 nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9);
365 uma_zone_set_warning(zone_jumbo9, "kern.ipc.nmbjumbo9 limit reached");
346
347 zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
348 mb_ctor_clust, mb_dtor_clust,
349#ifdef INVARIANTS
350 trash_init, trash_fini,
351#else
352 NULL, NULL,
353#endif
354 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
355 uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc);
356 if (nmbjumbo9 > 0)
357 nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9);
358 uma_zone_set_warning(zone_jumbo9, "kern.ipc.nmbjumbo9 limit reached");
366 uma_zone_set_maxaction(zone_jumbo9, mb_maxaction);
359 uma_zone_set_maxaction(zone_jumbo9, mb_reclaim);
367
368 zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,
369 mb_ctor_clust, mb_dtor_clust,
370#ifdef INVARIANTS
371 trash_init, trash_fini,
372#else
373 NULL, NULL,
374#endif
375 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
376 uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc);
377 if (nmbjumbo16 > 0)
378 nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16);
379 uma_zone_set_warning(zone_jumbo16, "kern.ipc.nmbjumbo16 limit reached");
360
361 zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,
362 mb_ctor_clust, mb_dtor_clust,
363#ifdef INVARIANTS
364 trash_init, trash_fini,
365#else
366 NULL, NULL,
367#endif
368 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
369 uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc);
370 if (nmbjumbo16 > 0)
371 nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16);
372 uma_zone_set_warning(zone_jumbo16, "kern.ipc.nmbjumbo16 limit reached");
380 uma_zone_set_maxaction(zone_jumbo16, mb_maxaction);
373 uma_zone_set_maxaction(zone_jumbo16, mb_reclaim);
381
382 zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int),
383 NULL, NULL,
384 NULL, NULL,
385 UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
386
374
375 zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int),
376 NULL, NULL,
377 NULL, NULL,
378 UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
379
387 /* uma_prealloc() goes here... */
388
389 /* Initialize the mb_reclaim() callout. */
390 mtx_init(&mb_reclaim_callout_mtx, "mb_reclaim_callout_mtx", NULL,
391 MTX_DEF);
392 callout_init(&mb_reclaim_callout, 1);
393
394 /*
395 * Hook event handler for low-memory situation, used to
396 * drain protocols and push data back to the caches (UMA
397 * later pushes it back to VM).
398 */
399 EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,
400 EVENTHANDLER_PRI_FIRST);
401}

--- 270 unchanged lines hidden (view full) ---

672 if (error)
673 return (error);
674#endif
675
676 return (0);
677}
678
679/*
380 /*
381 * Hook event handler for low-memory situation, used to
382 * drain protocols and push data back to the caches (UMA
383 * later pushes it back to VM).
384 */
385 EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,
386 EVENTHANDLER_PRI_FIRST);
387}

--- 270 unchanged lines hidden (view full) ---

658 if (error)
659 return (error);
660#endif
661
662 return (0);
663}
664
665/*
680 * This is the protocol drain routine.
666 * This is the protocol drain routine. Called by UMA whenever any of the
667 * mbuf zones is closed to its limit.
681 *
682 * No locks should be held when this is called. The drain routines have to
683 * presently acquire some locks which raises the possibility of lock order
684 * reversal.
685 */
686static void
668 *
669 * No locks should be held when this is called. The drain routines have to
670 * presently acquire some locks which raises the possibility of lock order
671 * reversal.
672 */
673static void
687mb_reclaim(void *junk)
674mb_reclaim(uma_zone_t zone __unused, int pending __unused)
688{
689 struct domain *dp;
690 struct protosw *pr;
691
675{
676 struct domain *dp;
677 struct protosw *pr;
678
692 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL,
693 "mb_reclaim()");
679 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL, __func__);
694
695 for (dp = domains; dp != NULL; dp = dp->dom_next)
696 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
697 if (pr->pr_drain != NULL)
698 (*pr->pr_drain)();
699}
680
681 for (dp = domains; dp != NULL; dp = dp->dom_next)
682 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
683 if (pr->pr_drain != NULL)
684 (*pr->pr_drain)();
685}
700
701/*
702 * This is the function called by the mb_reclaim_callout, which is
703 * used when we hit the maximum for a zone.
704 *
705 * (See mb_maxaction() below.)
706 */
707static void
708mb_reclaim_timer(void *junk __unused)
709{
710
711 mtx_lock(&mb_reclaim_callout_mtx);
712
713 /*
714 * Avoid running this function extra times by skipping this invocation
715 * if the callout has already been rescheduled.
716 */
717 if (callout_pending(&mb_reclaim_callout) ||
718 !callout_active(&mb_reclaim_callout)) {
719 mtx_unlock(&mb_reclaim_callout_mtx);
720 return;
721 }
722 mtx_unlock(&mb_reclaim_callout_mtx);
723
724 mb_reclaim(NULL);
725
726 mtx_lock(&mb_reclaim_callout_mtx);
727 callout_deactivate(&mb_reclaim_callout);
728 mtx_unlock(&mb_reclaim_callout_mtx);
729}
730
731/*
732 * This function is called when we hit the maximum for a zone.
733 *
734 * At that point, we want to call the protocol drain routine to free up some
735 * mbufs. However, we will use the callout routines to schedule this to
736 * occur in another thread. (The thread calling this function holds the
737 * zone lock.)
738 */
739static void
740mb_maxaction(uma_zone_t zone __unused)
741{
742
743 /*
744 * If we can't immediately obtain the lock, either the callout
745 * is currently running, or another thread is scheduling the
746 * callout.
747 */
748 if (!mtx_trylock(&mb_reclaim_callout_mtx))
749 return;
750
751 /* If not already scheduled/running, schedule the callout. */
752 if (!callout_active(&mb_reclaim_callout)) {
753 callout_reset(&mb_reclaim_callout, 1, mb_reclaim_timer, NULL);
754 }
755
756 mtx_unlock(&mb_reclaim_callout_mtx);
757}