1/*-
2 * Copyright (c) 2004, 2005,
3 * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: head/sys/kern/kern_mbuf.c 243996 2012-12-07 22:22:04Z pjd $");
30
31#include "opt_param.h"
32
33#include <sys/param.h>
34#include <sys/malloc.h>
35#include <sys/systm.h>
36#include <sys/mbuf.h>
37#include <sys/domain.h>
38#include <sys/eventhandler.h>
39#include <sys/kernel.h>
40#include <sys/protosw.h>
41#include <sys/smp.h>
42#include <sys/sysctl.h>
43
44#include <security/mac/mac_framework.h>
45
46#include <vm/vm.h>
47#include <vm/vm_extern.h>
48#include <vm/vm_kern.h>
49#include <vm/vm_page.h>
50#include <vm/uma.h>
51#include <vm/uma_int.h>
52#include <vm/uma_dbg.h>
53
54/*
55 * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA
56 * Zones.
57 *
58 * Mbuf Clusters (2K, contiguous) are allocated from the Cluster
59 * Zone. The Zone can be capped at kern.ipc.nmbclusters, if the
60 * administrator so desires.
61 *
62 * Mbufs are allocated from a UMA Master Zone called the Mbuf
63 * Zone.
64 *
65 * Additionally, FreeBSD provides a Packet Zone, which it
66 * configures as a Secondary Zone to the Mbuf Master Zone,
67 * thus sharing backend Slab kegs with the Mbuf Master Zone.
68 *
69 * Thus common-case allocations and locking are simplified:
70 *
71 * m_clget() m_getcl()
72 * | |
73 * | .------------>[(Packet Cache)] m_get(), m_gethdr()
74 * | | [ Packet ] |
75 * [(Cluster Cache)] [ Secondary ] [ (Mbuf Cache) ]
76 * [ Cluster Zone ] [ Zone ] [ Mbuf Master Zone ]
77 * | \________ |
78 * [ Cluster Keg ] \ /
79 * | [ Mbuf Keg ]
80 * [ Cluster Slabs ] |
81 * | [ Mbuf Slabs ]
82 * \____________(VM)_________________/
83 *
84 *
85 * Whenever an object is allocated with uma_zalloc() out of
86 * one of the Zones its _ctor_ function is executed. The same
87 * for any deallocation through uma_zfree() the _dtor_ function
88 * is executed.
89 *
90 * Caches are per-CPU and are filled from the Master Zone.
91 *
92 * Whenever an object is allocated from the underlying global
93 * memory pool it gets pre-initialized with the _zinit_ functions.
94 * When the Keg's are overfull objects get decomissioned with
95 * _zfini_ functions and free'd back to the global memory pool.
96 *
97 */
98
99int nmbufs; /* limits number of mbufs */
100int nmbclusters; /* limits number of mbuf clusters */
101int nmbjumbop; /* limits number of page size jumbo clusters */
102int nmbjumbo9; /* limits number of 9k jumbo clusters */
103int nmbjumbo16; /* limits number of 16k jumbo clusters */
104struct mbstat mbstat;
105
106/*
107 * tunable_mbinit() has to be run before init_maxsockets() thus
108 * the SYSINIT order below is SI_ORDER_MIDDLE while init_maxsockets()
109 * runs at SI_ORDER_ANY.
110 *
111 * NB: This has to be done before VM init.
112 */
113static void
114tunable_mbinit(void *dummy)
115{
116
117 TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters);
118 if (nmbclusters == 0)
119 nmbclusters = maxmbufmem / MCLBYTES / 4;
120
121 TUNABLE_INT_FETCH("kern.ipc.nmbjumbop", &nmbjumbop);
122 if (nmbjumbop == 0)
123 nmbjumbop = maxmbufmem / MJUMPAGESIZE / 4;
124
125 TUNABLE_INT_FETCH("kern.ipc.nmbjumbo9", &nmbjumbo9);
126 if (nmbjumbo9 == 0)
127 nmbjumbo9 = maxmbufmem / MJUM9BYTES / 6;
128
129 TUNABLE_INT_FETCH("kern.ipc.nmbjumbo16", &nmbjumbo16);
130 if (nmbjumbo16 == 0)
131 nmbjumbo16 = maxmbufmem / MJUM16BYTES / 6;
132
133 /*
134 * We need at least as many mbufs as we have clusters of
135 * the various types added together.
136 */
137 TUNABLE_INT_FETCH("kern.ipc.nmbufs", &nmbufs);
138 if (nmbufs < nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16)
139 nmbufs = lmax(maxmbufmem / MSIZE / 5,
140 nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16);
141}
142SYSINIT(tunable_mbinit, SI_SUB_TUNABLES, SI_ORDER_MIDDLE, tunable_mbinit, NULL);
143
144static int
145sysctl_nmbclusters(SYSCTL_HANDLER_ARGS)
146{
147 int error, newnmbclusters;
148
149 newnmbclusters = nmbclusters;
150 error = sysctl_handle_int(oidp, &newnmbclusters, 0, req);
151 if (error == 0 && req->newptr) {
152 if (newnmbclusters > nmbclusters &&
153 nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
154 nmbclusters = newnmbclusters;
155 uma_zone_set_max(zone_clust, nmbclusters);
156 nmbclusters = uma_zone_get_max(zone_clust);
157 EVENTHANDLER_INVOKE(nmbclusters_change);
158 } else
159 error = EINVAL;
160 }
161 return (error);
162}
163SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbclusters, CTLTYPE_INT|CTLFLAG_RW,
164&nmbclusters, 0, sysctl_nmbclusters, "IU",
165 "Maximum number of mbuf clusters allowed");
166
167static int
168sysctl_nmbjumbop(SYSCTL_HANDLER_ARGS)
169{
170 int error, newnmbjumbop;
171
172 newnmbjumbop = nmbjumbop;
173 error = sysctl_handle_int(oidp, &newnmbjumbop, 0, req);
174 if (error == 0 && req->newptr) {
175 if (newnmbjumbop > nmbjumbop &&
176 nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
177 nmbjumbop = newnmbjumbop;
178 uma_zone_set_max(zone_jumbop, nmbjumbop);
179 nmbjumbop = uma_zone_get_max(zone_jumbop);
180 } else
181 error = EINVAL;
182 }
183 return (error);
184}
185SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbop, CTLTYPE_INT|CTLFLAG_RW,
186&nmbjumbop, 0, sysctl_nmbjumbop, "IU",
187 "Maximum number of mbuf page size jumbo clusters allowed");
188
189static int
190sysctl_nmbjumbo9(SYSCTL_HANDLER_ARGS)
191{
192 int error, newnmbjumbo9;
193
194 newnmbjumbo9 = nmbjumbo9;
195 error = sysctl_handle_int(oidp, &newnmbjumbo9, 0, req);
196 if (error == 0 && req->newptr) {
197 if (newnmbjumbo9 > nmbjumbo9&&
198 nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
199 nmbjumbo9 = newnmbjumbo9;
200 uma_zone_set_max(zone_jumbo9, nmbjumbo9);
201 nmbjumbo9 = uma_zone_get_max(zone_jumbo9);
202 } else
203 error = EINVAL;
204 }
205 return (error);
206}
207SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo9, CTLTYPE_INT|CTLFLAG_RW,
208&nmbjumbo9, 0, sysctl_nmbjumbo9, "IU",
209 "Maximum number of mbuf 9k jumbo clusters allowed");
210
211static int
212sysctl_nmbjumbo16(SYSCTL_HANDLER_ARGS)
213{
214 int error, newnmbjumbo16;
215
216 newnmbjumbo16 = nmbjumbo16;
217 error = sysctl_handle_int(oidp, &newnmbjumbo16, 0, req);
218 if (error == 0 && req->newptr) {
219 if (newnmbjumbo16 > nmbjumbo16 &&
220 nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
221 nmbjumbo16 = newnmbjumbo16;
222 uma_zone_set_max(zone_jumbo16, nmbjumbo16);
223 nmbjumbo16 = uma_zone_get_max(zone_jumbo16);
224 } else
225 error = EINVAL;
226 }
227 return (error);
228}
229SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo16, CTLTYPE_INT|CTLFLAG_RW,
230&nmbjumbo16, 0, sysctl_nmbjumbo16, "IU",
231 "Maximum number of mbuf 16k jumbo clusters allowed");
232
233static int
234sysctl_nmbufs(SYSCTL_HANDLER_ARGS)
235{
236 int error, newnmbufs;
237
238 newnmbufs = nmbufs;
239 error = sysctl_handle_int(oidp, &newnmbufs, 0, req);
240 if (error == 0 && req->newptr) {
241 if (newnmbufs > nmbufs) {
242 nmbufs = newnmbufs;
243 uma_zone_set_max(zone_mbuf, nmbufs);
244 nmbufs = uma_zone_get_max(zone_mbuf);
245 EVENTHANDLER_INVOKE(nmbufs_change);
246 } else
247 error = EINVAL;
248 }
249 return (error);
250}
251SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbuf, CTLTYPE_INT|CTLFLAG_RW,
252&nmbufs, 0, sysctl_nmbufs, "IU",
253 "Maximum number of mbufs allowed");
254
255SYSCTL_STRUCT(_kern_ipc, OID_AUTO, mbstat, CTLFLAG_RD, &mbstat, mbstat,
256 "Mbuf general information and statistics");
257
258/*
259 * Zones from which we allocate.
260 */
261uma_zone_t zone_mbuf;
262uma_zone_t zone_clust;
263uma_zone_t zone_pack;
264uma_zone_t zone_jumbop;
265uma_zone_t zone_jumbo9;
266uma_zone_t zone_jumbo16;
267uma_zone_t zone_ext_refcnt;
268
269/*
270 * Local prototypes.
271 */
272static int mb_ctor_mbuf(void *, int, void *, int);
273static int mb_ctor_clust(void *, int, void *, int);
274static int mb_ctor_pack(void *, int, void *, int);
275static void mb_dtor_mbuf(void *, int, void *);
276static void mb_dtor_clust(void *, int, void *);
277static void mb_dtor_pack(void *, int, void *);
278static int mb_zinit_pack(void *, int, int);
279static void mb_zfini_pack(void *, int);
280
281static void mb_reclaim(void *);
282static void mbuf_init(void *);
283static void *mbuf_jumbo_alloc(uma_zone_t, int, uint8_t *, int);
284
285/* Ensure that MSIZE must be a power of 2. */
286CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
287
288/*
289 * Initialize FreeBSD Network buffer allocation.
290 */
291SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL);
292static void
293mbuf_init(void *dummy)
294{
295
296 /*
297 * Configure UMA zones for Mbufs, Clusters, and Packets.
298 */
299 zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE,
300 mb_ctor_mbuf, mb_dtor_mbuf,
301#ifdef INVARIANTS
302 trash_init, trash_fini,
303#else
304 NULL, NULL,
305#endif
306 MSIZE - 1, UMA_ZONE_MAXBUCKET);
307 if (nmbufs > 0) {
308 uma_zone_set_max(zone_mbuf, nmbufs);
309 nmbufs = uma_zone_get_max(zone_mbuf);
310 }
311
312 zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
313 mb_ctor_clust, mb_dtor_clust,
314#ifdef INVARIANTS
315 trash_init, trash_fini,
316#else
317 NULL, NULL,
318#endif
319 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
320 if (nmbclusters > 0) {
321 uma_zone_set_max(zone_clust, nmbclusters);
322 nmbclusters = uma_zone_get_max(zone_clust);
323 }
324
325 zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
326 mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);
327
328 /* Make jumbo frame zone too. Page size, 9k and 16k. */
329 zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE,
330 mb_ctor_clust, mb_dtor_clust,
331#ifdef INVARIANTS
332 trash_init, trash_fini,
333#else
334 NULL, NULL,
335#endif
336 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
337 if (nmbjumbop > 0) {
338 uma_zone_set_max(zone_jumbop, nmbjumbop);
339 nmbjumbop = uma_zone_get_max(zone_jumbop);
340 }
341
342 zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
343 mb_ctor_clust, mb_dtor_clust,
344#ifdef INVARIANTS
345 trash_init, trash_fini,
346#else
347 NULL, NULL,
348#endif
349 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
350 uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc);
351 if (nmbjumbo9 > 0) {
352 uma_zone_set_max(zone_jumbo9, nmbjumbo9);
353 nmbjumbo9 = uma_zone_get_max(zone_jumbo9);
354 }
355
356 zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,
357 mb_ctor_clust, mb_dtor_clust,
358#ifdef INVARIANTS
359 trash_init, trash_fini,
360#else
361 NULL, NULL,
362#endif
363 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
364 uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc);
365 if (nmbjumbo16 > 0) {
366 uma_zone_set_max(zone_jumbo16, nmbjumbo16);
367 nmbjumbo16 = uma_zone_get_max(zone_jumbo16);
368 }
369
370 zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int),
371 NULL, NULL,
372 NULL, NULL,
373 UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
374
375 /* uma_prealloc() goes here... */
376
377 /*
378 * Hook event handler for low-memory situation, used to
379 * drain protocols and push data back to the caches (UMA
380 * later pushes it back to VM).
381 */
382 EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,
383 EVENTHANDLER_PRI_FIRST);
384
385 /*
386 * [Re]set counters and local statistics knobs.
387 * XXX Some of these should go and be replaced, but UMA stat
388 * gathering needs to be revised.
389 */
390 mbstat.m_mbufs = 0;
391 mbstat.m_mclusts = 0;
392 mbstat.m_drain = 0;
393 mbstat.m_msize = MSIZE;
394 mbstat.m_mclbytes = MCLBYTES;
395 mbstat.m_minclsize = MINCLSIZE;
396 mbstat.m_mlen = MLEN;
397 mbstat.m_mhlen = MHLEN;
398 mbstat.m_numtypes = MT_NTYPES;
399
400 mbstat.m_mcfail = mbstat.m_mpfail = 0;
401 mbstat.sf_iocnt = 0;
402 mbstat.sf_allocwait = mbstat.sf_allocfail = 0;
403}
404
405/*
406 * UMA backend page allocator for the jumbo frame zones.
407 *
408 * Allocates kernel virtual memory that is backed by contiguous physical
409 * pages.
410 */
411static void *
412mbuf_jumbo_alloc(uma_zone_t zone, int bytes, uint8_t *flags, int wait)
413{
414
415 /* Inform UMA that this allocator uses kernel_map/object. */
416 *flags = UMA_SLAB_KERNEL;
417 return ((void *)kmem_alloc_contig(kernel_map, bytes, wait,
418 (vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0, VM_MEMATTR_DEFAULT));
419}
420
421/*
422 * Constructor for Mbuf master zone.
423 *
424 * The 'arg' pointer points to a mb_args structure which
425 * contains call-specific information required to support the
426 * mbuf allocation API. See mbuf.h.
427 */
428static int
429mb_ctor_mbuf(void *mem, int size, void *arg, int how)
430{
431 struct mbuf *m;
432 struct mb_args *args;
433#ifdef MAC
434 int error;
435#endif
436 int flags;
437 short type;
438
439#ifdef INVARIANTS
440 trash_ctor(mem, size, arg, how);
441#endif
442 m = (struct mbuf *)mem;
443 args = (struct mb_args *)arg;
444 flags = args->flags;
445 type = args->type;
446
447 /*
448 * The mbuf is initialized later. The caller has the
449 * responsibility to set up any MAC labels too.
450 */
451 if (type == MT_NOINIT)
452 return (0);
453
454 m->m_next = NULL;
455 m->m_nextpkt = NULL;
456 m->m_len = 0;
457 m->m_flags = flags;
458 m->m_type = type;
459 if (flags & M_PKTHDR) {
460 m->m_data = m->m_pktdat;
461 m->m_pkthdr.rcvif = NULL;
462 m->m_pkthdr.header = NULL;
463 m->m_pkthdr.len = 0;
464 m->m_pkthdr.csum_flags = 0;
465 m->m_pkthdr.csum_data = 0;
466 m->m_pkthdr.tso_segsz = 0;
467 m->m_pkthdr.ether_vtag = 0;
468 m->m_pkthdr.flowid = 0;
469 SLIST_INIT(&m->m_pkthdr.tags);
470#ifdef MAC
471 /* If the label init fails, fail the alloc */
472 error = mac_mbuf_init(m, how);
473 if (error)
474 return (error);
475#endif
476 } else
477 m->m_data = m->m_dat;
478 return (0);
479}
480
481/*
482 * The Mbuf master zone destructor.
483 */
484static void
485mb_dtor_mbuf(void *mem, int size, void *arg)
486{
487 struct mbuf *m;
488 unsigned long flags;
489
490 m = (struct mbuf *)mem;
491 flags = (unsigned long)arg;
492
493 if ((flags & MB_NOTAGS) == 0 && (m->m_flags & M_PKTHDR) != 0)
494 m_tag_delete_chain(m, NULL);
495 KASSERT((m->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__));
496 KASSERT((m->m_flags & M_NOFREE) == 0, ("%s: M_NOFREE set", __func__));
497#ifdef INVARIANTS
498 trash_dtor(mem, size, arg);
499#endif
500}
501
502/*
503 * The Mbuf Packet zone destructor.
504 */
505static void
506mb_dtor_pack(void *mem, int size, void *arg)
507{
508 struct mbuf *m;
509
510 m = (struct mbuf *)mem;
511 if ((m->m_flags & M_PKTHDR) != 0)
512 m_tag_delete_chain(m, NULL);
513
514 /* Make sure we've got a clean cluster back. */
515 KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
516 KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__));
517 KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__));
518 KASSERT(m->m_ext.ext_arg1 == NULL, ("%s: ext_arg1 != NULL", __func__));
519 KASSERT(m->m_ext.ext_arg2 == NULL, ("%s: ext_arg2 != NULL", __func__));
520 KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__));
521 KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__));
522 KASSERT(*m->m_ext.ref_cnt == 1, ("%s: ref_cnt != 1", __func__));
523#ifdef INVARIANTS
524 trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg);
525#endif
526 /*
527 * If there are processes blocked on zone_clust, waiting for pages
528 * to be freed up, * cause them to be woken up by draining the
529 * packet zone. We are exposed to a race here * (in the check for
530 * the UMA_ZFLAG_FULL) where we might miss the flag set, but that
531 * is deliberate. We don't want to acquire the zone lock for every
532 * mbuf free.
533 */
534 if (uma_zone_exhausted_nolock(zone_clust))
535 zone_drain(zone_pack);
536}
537
538/*
539 * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor.
540 *
541 * Here the 'arg' pointer points to the Mbuf which we
542 * are configuring cluster storage for. If 'arg' is
543 * empty we allocate just the cluster without setting
544 * the mbuf to it. See mbuf.h.
545 */
546static int
547mb_ctor_clust(void *mem, int size, void *arg, int how)
548{
549 struct mbuf *m;
550 u_int *refcnt;
551 int type;
552 uma_zone_t zone;
553
554#ifdef INVARIANTS
555 trash_ctor(mem, size, arg, how);
556#endif
557 switch (size) {
558 case MCLBYTES:
559 type = EXT_CLUSTER;
560 zone = zone_clust;
561 break;
562#if MJUMPAGESIZE != MCLBYTES
563 case MJUMPAGESIZE:
564 type = EXT_JUMBOP;
565 zone = zone_jumbop;
566 break;
567#endif
568 case MJUM9BYTES:
569 type = EXT_JUMBO9;
570 zone = zone_jumbo9;
571 break;
572 case MJUM16BYTES:
573 type = EXT_JUMBO16;
574 zone = zone_jumbo16;
575 break;
576 default:
577 panic("unknown cluster size");
578 break;
579 }
580
581 m = (struct mbuf *)arg;
582 refcnt = uma_find_refcnt(zone, mem);
583 *refcnt = 1;
584 if (m != NULL) {
585 m->m_ext.ext_buf = (caddr_t)mem;
586 m->m_data = m->m_ext.ext_buf;
587 m->m_flags |= M_EXT;
588 m->m_ext.ext_free = NULL;
589 m->m_ext.ext_arg1 = NULL;
590 m->m_ext.ext_arg2 = NULL;
591 m->m_ext.ext_size = size;
592 m->m_ext.ext_type = type;
593 m->m_ext.ref_cnt = refcnt;
594 }
595
596 return (0);
597}
598
599/*
600 * The Mbuf Cluster zone destructor.
601 */
602static void
603mb_dtor_clust(void *mem, int size, void *arg)
604{
605#ifdef INVARIANTS
606 uma_zone_t zone;
607
608 zone = m_getzone(size);
609 KASSERT(*(uma_find_refcnt(zone, mem)) <= 1,
610 ("%s: refcnt incorrect %u", __func__,
611 *(uma_find_refcnt(zone, mem))) );
612
613 trash_dtor(mem, size, arg);
614#endif
615}
616
617/*
618 * The Packet secondary zone's init routine, executed on the
619 * object's transition from mbuf keg slab to zone cache.
620 */
621static int
622mb_zinit_pack(void *mem, int size, int how)
623{
624 struct mbuf *m;
625
626 m = (struct mbuf *)mem; /* m is virgin. */
627 if (uma_zalloc_arg(zone_clust, m, how) == NULL ||
628 m->m_ext.ext_buf == NULL)
629 return (ENOMEM);
630 m->m_ext.ext_type = EXT_PACKET; /* Override. */
631#ifdef INVARIANTS
632 trash_init(m->m_ext.ext_buf, MCLBYTES, how);
633#endif
634 return (0);
635}
636
637/*
638 * The Packet secondary zone's fini routine, executed on the
639 * object's transition from zone cache to keg slab.
640 */
641static void
642mb_zfini_pack(void *mem, int size)
643{
644 struct mbuf *m;
645
646 m = (struct mbuf *)mem;
647#ifdef INVARIANTS
648 trash_fini(m->m_ext.ext_buf, MCLBYTES);
649#endif
650 uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL);
651#ifdef INVARIANTS
652 trash_dtor(mem, size, NULL);
653#endif
654}
655
656/*
657 * The "packet" keg constructor.
658 */
659static int
660mb_ctor_pack(void *mem, int size, void *arg, int how)
661{
662 struct mbuf *m;
663 struct mb_args *args;
664#ifdef MAC
665 int error;
666#endif
667 int flags;
668 short type;
669
670 m = (struct mbuf *)mem;
671 args = (struct mb_args *)arg;
672 flags = args->flags;
673 type = args->type;
674
675#ifdef INVARIANTS
676 trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how);
677#endif
678 m->m_next = NULL;
679 m->m_nextpkt = NULL;
680 m->m_data = m->m_ext.ext_buf;
681 m->m_len = 0;
682 m->m_flags = (flags | M_EXT);
683 m->m_type = type;
684
685 if (flags & M_PKTHDR) {
686 m->m_pkthdr.rcvif = NULL;
687 m->m_pkthdr.len = 0;
688 m->m_pkthdr.header = NULL;
689 m->m_pkthdr.csum_flags = 0;
690 m->m_pkthdr.csum_data = 0;
691 m->m_pkthdr.tso_segsz = 0;
692 m->m_pkthdr.ether_vtag = 0;
693 m->m_pkthdr.flowid = 0;
694 SLIST_INIT(&m->m_pkthdr.tags);
695#ifdef MAC
696 /* If the label init fails, fail the alloc */
697 error = mac_mbuf_init(m, how);
698 if (error)
699 return (error);
700#endif
701 }
702 /* m_ext is already initialized. */
703
704 return (0);
705}
706
707int
708m_pkthdr_init(struct mbuf *m, int how)
709{
710#ifdef MAC
711 int error;
712#endif
713 m->m_data = m->m_pktdat;
714 SLIST_INIT(&m->m_pkthdr.tags);
715 m->m_pkthdr.rcvif = NULL;
716 m->m_pkthdr.header = NULL;
717 m->m_pkthdr.len = 0;
718 m->m_pkthdr.flowid = 0;
719 m->m_pkthdr.csum_flags = 0;
720 m->m_pkthdr.csum_data = 0;
721 m->m_pkthdr.tso_segsz = 0;
722 m->m_pkthdr.ether_vtag = 0;
723#ifdef MAC
724 /* If the label init fails, fail the alloc */
725 error = mac_mbuf_init(m, how);
726 if (error)
727 return (error);
728#endif
729
730 return (0);
731}
732
733/*
734 * This is the protocol drain routine.
735 *
736 * No locks should be held when this is called. The drain routines have to
737 * presently acquire some locks which raises the possibility of lock order
738 * reversal.
739 */
740static void
741mb_reclaim(void *junk)
742{
743 struct domain *dp;
744 struct protosw *pr;
745
746 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL,
747 "mb_reclaim()");
748
749 for (dp = domains; dp != NULL; dp = dp->dom_next)
750 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
751 if (pr->pr_drain != NULL)
752 (*pr->pr_drain)();
753}
2 * Copyright (c) 2004, 2005,
3 * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: head/sys/kern/kern_mbuf.c 243996 2012-12-07 22:22:04Z pjd $");
30
31#include "opt_param.h"
32
33#include <sys/param.h>
34#include <sys/malloc.h>
35#include <sys/systm.h>
36#include <sys/mbuf.h>
37#include <sys/domain.h>
38#include <sys/eventhandler.h>
39#include <sys/kernel.h>
40#include <sys/protosw.h>
41#include <sys/smp.h>
42#include <sys/sysctl.h>
43
44#include <security/mac/mac_framework.h>
45
46#include <vm/vm.h>
47#include <vm/vm_extern.h>
48#include <vm/vm_kern.h>
49#include <vm/vm_page.h>
50#include <vm/uma.h>
51#include <vm/uma_int.h>
52#include <vm/uma_dbg.h>
53
54/*
55 * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA
56 * Zones.
57 *
58 * Mbuf Clusters (2K, contiguous) are allocated from the Cluster
59 * Zone. The Zone can be capped at kern.ipc.nmbclusters, if the
60 * administrator so desires.
61 *
62 * Mbufs are allocated from a UMA Master Zone called the Mbuf
63 * Zone.
64 *
65 * Additionally, FreeBSD provides a Packet Zone, which it
66 * configures as a Secondary Zone to the Mbuf Master Zone,
67 * thus sharing backend Slab kegs with the Mbuf Master Zone.
68 *
69 * Thus common-case allocations and locking are simplified:
70 *
71 * m_clget() m_getcl()
72 * | |
73 * | .------------>[(Packet Cache)] m_get(), m_gethdr()
74 * | | [ Packet ] |
75 * [(Cluster Cache)] [ Secondary ] [ (Mbuf Cache) ]
76 * [ Cluster Zone ] [ Zone ] [ Mbuf Master Zone ]
77 * | \________ |
78 * [ Cluster Keg ] \ /
79 * | [ Mbuf Keg ]
80 * [ Cluster Slabs ] |
81 * | [ Mbuf Slabs ]
82 * \____________(VM)_________________/
83 *
84 *
85 * Whenever an object is allocated with uma_zalloc() out of
86 * one of the Zones its _ctor_ function is executed. The same
87 * for any deallocation through uma_zfree() the _dtor_ function
88 * is executed.
89 *
90 * Caches are per-CPU and are filled from the Master Zone.
91 *
92 * Whenever an object is allocated from the underlying global
93 * memory pool it gets pre-initialized with the _zinit_ functions.
94 * When the Keg's are overfull objects get decomissioned with
95 * _zfini_ functions and free'd back to the global memory pool.
96 *
97 */
98
99int nmbufs; /* limits number of mbufs */
100int nmbclusters; /* limits number of mbuf clusters */
101int nmbjumbop; /* limits number of page size jumbo clusters */
102int nmbjumbo9; /* limits number of 9k jumbo clusters */
103int nmbjumbo16; /* limits number of 16k jumbo clusters */
104struct mbstat mbstat;
105
106/*
107 * tunable_mbinit() has to be run before init_maxsockets() thus
108 * the SYSINIT order below is SI_ORDER_MIDDLE while init_maxsockets()
109 * runs at SI_ORDER_ANY.
110 *
111 * NB: This has to be done before VM init.
112 */
113static void
114tunable_mbinit(void *dummy)
115{
116
117 TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters);
118 if (nmbclusters == 0)
119 nmbclusters = maxmbufmem / MCLBYTES / 4;
120
121 TUNABLE_INT_FETCH("kern.ipc.nmbjumbop", &nmbjumbop);
122 if (nmbjumbop == 0)
123 nmbjumbop = maxmbufmem / MJUMPAGESIZE / 4;
124
125 TUNABLE_INT_FETCH("kern.ipc.nmbjumbo9", &nmbjumbo9);
126 if (nmbjumbo9 == 0)
127 nmbjumbo9 = maxmbufmem / MJUM9BYTES / 6;
128
129 TUNABLE_INT_FETCH("kern.ipc.nmbjumbo16", &nmbjumbo16);
130 if (nmbjumbo16 == 0)
131 nmbjumbo16 = maxmbufmem / MJUM16BYTES / 6;
132
133 /*
134 * We need at least as many mbufs as we have clusters of
135 * the various types added together.
136 */
137 TUNABLE_INT_FETCH("kern.ipc.nmbufs", &nmbufs);
138 if (nmbufs < nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16)
139 nmbufs = lmax(maxmbufmem / MSIZE / 5,
140 nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16);
141}
142SYSINIT(tunable_mbinit, SI_SUB_TUNABLES, SI_ORDER_MIDDLE, tunable_mbinit, NULL);
143
144static int
145sysctl_nmbclusters(SYSCTL_HANDLER_ARGS)
146{
147 int error, newnmbclusters;
148
149 newnmbclusters = nmbclusters;
150 error = sysctl_handle_int(oidp, &newnmbclusters, 0, req);
151 if (error == 0 && req->newptr) {
152 if (newnmbclusters > nmbclusters &&
153 nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
154 nmbclusters = newnmbclusters;
155 uma_zone_set_max(zone_clust, nmbclusters);
156 nmbclusters = uma_zone_get_max(zone_clust);
157 EVENTHANDLER_INVOKE(nmbclusters_change);
158 } else
159 error = EINVAL;
160 }
161 return (error);
162}
163SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbclusters, CTLTYPE_INT|CTLFLAG_RW,
164&nmbclusters, 0, sysctl_nmbclusters, "IU",
165 "Maximum number of mbuf clusters allowed");
166
167static int
168sysctl_nmbjumbop(SYSCTL_HANDLER_ARGS)
169{
170 int error, newnmbjumbop;
171
172 newnmbjumbop = nmbjumbop;
173 error = sysctl_handle_int(oidp, &newnmbjumbop, 0, req);
174 if (error == 0 && req->newptr) {
175 if (newnmbjumbop > nmbjumbop &&
176 nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
177 nmbjumbop = newnmbjumbop;
178 uma_zone_set_max(zone_jumbop, nmbjumbop);
179 nmbjumbop = uma_zone_get_max(zone_jumbop);
180 } else
181 error = EINVAL;
182 }
183 return (error);
184}
185SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbop, CTLTYPE_INT|CTLFLAG_RW,
186&nmbjumbop, 0, sysctl_nmbjumbop, "IU",
187 "Maximum number of mbuf page size jumbo clusters allowed");
188
189static int
190sysctl_nmbjumbo9(SYSCTL_HANDLER_ARGS)
191{
192 int error, newnmbjumbo9;
193
194 newnmbjumbo9 = nmbjumbo9;
195 error = sysctl_handle_int(oidp, &newnmbjumbo9, 0, req);
196 if (error == 0 && req->newptr) {
197 if (newnmbjumbo9 > nmbjumbo9&&
198 nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
199 nmbjumbo9 = newnmbjumbo9;
200 uma_zone_set_max(zone_jumbo9, nmbjumbo9);
201 nmbjumbo9 = uma_zone_get_max(zone_jumbo9);
202 } else
203 error = EINVAL;
204 }
205 return (error);
206}
207SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo9, CTLTYPE_INT|CTLFLAG_RW,
208&nmbjumbo9, 0, sysctl_nmbjumbo9, "IU",
209 "Maximum number of mbuf 9k jumbo clusters allowed");
210
211static int
212sysctl_nmbjumbo16(SYSCTL_HANDLER_ARGS)
213{
214 int error, newnmbjumbo16;
215
216 newnmbjumbo16 = nmbjumbo16;
217 error = sysctl_handle_int(oidp, &newnmbjumbo16, 0, req);
218 if (error == 0 && req->newptr) {
219 if (newnmbjumbo16 > nmbjumbo16 &&
220 nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
221 nmbjumbo16 = newnmbjumbo16;
222 uma_zone_set_max(zone_jumbo16, nmbjumbo16);
223 nmbjumbo16 = uma_zone_get_max(zone_jumbo16);
224 } else
225 error = EINVAL;
226 }
227 return (error);
228}
229SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo16, CTLTYPE_INT|CTLFLAG_RW,
230&nmbjumbo16, 0, sysctl_nmbjumbo16, "IU",
231 "Maximum number of mbuf 16k jumbo clusters allowed");
232
233static int
234sysctl_nmbufs(SYSCTL_HANDLER_ARGS)
235{
236 int error, newnmbufs;
237
238 newnmbufs = nmbufs;
239 error = sysctl_handle_int(oidp, &newnmbufs, 0, req);
240 if (error == 0 && req->newptr) {
241 if (newnmbufs > nmbufs) {
242 nmbufs = newnmbufs;
243 uma_zone_set_max(zone_mbuf, nmbufs);
244 nmbufs = uma_zone_get_max(zone_mbuf);
245 EVENTHANDLER_INVOKE(nmbufs_change);
246 } else
247 error = EINVAL;
248 }
249 return (error);
250}
251SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbuf, CTLTYPE_INT|CTLFLAG_RW,
252&nmbufs, 0, sysctl_nmbufs, "IU",
253 "Maximum number of mbufs allowed");
254
255SYSCTL_STRUCT(_kern_ipc, OID_AUTO, mbstat, CTLFLAG_RD, &mbstat, mbstat,
256 "Mbuf general information and statistics");
257
258/*
259 * Zones from which we allocate.
260 */
261uma_zone_t zone_mbuf;
262uma_zone_t zone_clust;
263uma_zone_t zone_pack;
264uma_zone_t zone_jumbop;
265uma_zone_t zone_jumbo9;
266uma_zone_t zone_jumbo16;
267uma_zone_t zone_ext_refcnt;
268
269/*
270 * Local prototypes.
271 */
272static int mb_ctor_mbuf(void *, int, void *, int);
273static int mb_ctor_clust(void *, int, void *, int);
274static int mb_ctor_pack(void *, int, void *, int);
275static void mb_dtor_mbuf(void *, int, void *);
276static void mb_dtor_clust(void *, int, void *);
277static void mb_dtor_pack(void *, int, void *);
278static int mb_zinit_pack(void *, int, int);
279static void mb_zfini_pack(void *, int);
280
281static void mb_reclaim(void *);
282static void mbuf_init(void *);
283static void *mbuf_jumbo_alloc(uma_zone_t, int, uint8_t *, int);
284
285/* Ensure that MSIZE must be a power of 2. */
286CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
287
288/*
289 * Initialize FreeBSD Network buffer allocation.
290 */
291SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL);
292static void
293mbuf_init(void *dummy)
294{
295
296 /*
297 * Configure UMA zones for Mbufs, Clusters, and Packets.
298 */
299 zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE,
300 mb_ctor_mbuf, mb_dtor_mbuf,
301#ifdef INVARIANTS
302 trash_init, trash_fini,
303#else
304 NULL, NULL,
305#endif
306 MSIZE - 1, UMA_ZONE_MAXBUCKET);
307 if (nmbufs > 0) {
308 uma_zone_set_max(zone_mbuf, nmbufs);
309 nmbufs = uma_zone_get_max(zone_mbuf);
310 }
311
312 zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
313 mb_ctor_clust, mb_dtor_clust,
314#ifdef INVARIANTS
315 trash_init, trash_fini,
316#else
317 NULL, NULL,
318#endif
319 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
320 if (nmbclusters > 0) {
321 uma_zone_set_max(zone_clust, nmbclusters);
322 nmbclusters = uma_zone_get_max(zone_clust);
323 }
324
325 zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
326 mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);
327
328 /* Make jumbo frame zone too. Page size, 9k and 16k. */
329 zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE,
330 mb_ctor_clust, mb_dtor_clust,
331#ifdef INVARIANTS
332 trash_init, trash_fini,
333#else
334 NULL, NULL,
335#endif
336 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
337 if (nmbjumbop > 0) {
338 uma_zone_set_max(zone_jumbop, nmbjumbop);
339 nmbjumbop = uma_zone_get_max(zone_jumbop);
340 }
341
342 zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
343 mb_ctor_clust, mb_dtor_clust,
344#ifdef INVARIANTS
345 trash_init, trash_fini,
346#else
347 NULL, NULL,
348#endif
349 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
350 uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc);
351 if (nmbjumbo9 > 0) {
352 uma_zone_set_max(zone_jumbo9, nmbjumbo9);
353 nmbjumbo9 = uma_zone_get_max(zone_jumbo9);
354 }
355
356 zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,
357 mb_ctor_clust, mb_dtor_clust,
358#ifdef INVARIANTS
359 trash_init, trash_fini,
360#else
361 NULL, NULL,
362#endif
363 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
364 uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc);
365 if (nmbjumbo16 > 0) {
366 uma_zone_set_max(zone_jumbo16, nmbjumbo16);
367 nmbjumbo16 = uma_zone_get_max(zone_jumbo16);
368 }
369
370 zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int),
371 NULL, NULL,
372 NULL, NULL,
373 UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
374
375 /* uma_prealloc() goes here... */
376
377 /*
378 * Hook event handler for low-memory situation, used to
379 * drain protocols and push data back to the caches (UMA
380 * later pushes it back to VM).
381 */
382 EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,
383 EVENTHANDLER_PRI_FIRST);
384
385 /*
386 * [Re]set counters and local statistics knobs.
387 * XXX Some of these should go and be replaced, but UMA stat
388 * gathering needs to be revised.
389 */
390 mbstat.m_mbufs = 0;
391 mbstat.m_mclusts = 0;
392 mbstat.m_drain = 0;
393 mbstat.m_msize = MSIZE;
394 mbstat.m_mclbytes = MCLBYTES;
395 mbstat.m_minclsize = MINCLSIZE;
396 mbstat.m_mlen = MLEN;
397 mbstat.m_mhlen = MHLEN;
398 mbstat.m_numtypes = MT_NTYPES;
399
400 mbstat.m_mcfail = mbstat.m_mpfail = 0;
401 mbstat.sf_iocnt = 0;
402 mbstat.sf_allocwait = mbstat.sf_allocfail = 0;
403}
404
405/*
406 * UMA backend page allocator for the jumbo frame zones.
407 *
408 * Allocates kernel virtual memory that is backed by contiguous physical
409 * pages.
410 */
411static void *
412mbuf_jumbo_alloc(uma_zone_t zone, int bytes, uint8_t *flags, int wait)
413{
414
415 /* Inform UMA that this allocator uses kernel_map/object. */
416 *flags = UMA_SLAB_KERNEL;
417 return ((void *)kmem_alloc_contig(kernel_map, bytes, wait,
418 (vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0, VM_MEMATTR_DEFAULT));
419}
420
421/*
422 * Constructor for Mbuf master zone.
423 *
424 * The 'arg' pointer points to a mb_args structure which
425 * contains call-specific information required to support the
426 * mbuf allocation API. See mbuf.h.
427 */
428static int
429mb_ctor_mbuf(void *mem, int size, void *arg, int how)
430{
431 struct mbuf *m;
432 struct mb_args *args;
433#ifdef MAC
434 int error;
435#endif
436 int flags;
437 short type;
438
439#ifdef INVARIANTS
440 trash_ctor(mem, size, arg, how);
441#endif
442 m = (struct mbuf *)mem;
443 args = (struct mb_args *)arg;
444 flags = args->flags;
445 type = args->type;
446
447 /*
448 * The mbuf is initialized later. The caller has the
449 * responsibility to set up any MAC labels too.
450 */
451 if (type == MT_NOINIT)
452 return (0);
453
454 m->m_next = NULL;
455 m->m_nextpkt = NULL;
456 m->m_len = 0;
457 m->m_flags = flags;
458 m->m_type = type;
459 if (flags & M_PKTHDR) {
460 m->m_data = m->m_pktdat;
461 m->m_pkthdr.rcvif = NULL;
462 m->m_pkthdr.header = NULL;
463 m->m_pkthdr.len = 0;
464 m->m_pkthdr.csum_flags = 0;
465 m->m_pkthdr.csum_data = 0;
466 m->m_pkthdr.tso_segsz = 0;
467 m->m_pkthdr.ether_vtag = 0;
468 m->m_pkthdr.flowid = 0;
469 SLIST_INIT(&m->m_pkthdr.tags);
470#ifdef MAC
471 /* If the label init fails, fail the alloc */
472 error = mac_mbuf_init(m, how);
473 if (error)
474 return (error);
475#endif
476 } else
477 m->m_data = m->m_dat;
478 return (0);
479}
480
481/*
482 * The Mbuf master zone destructor.
483 */
484static void
485mb_dtor_mbuf(void *mem, int size, void *arg)
486{
487 struct mbuf *m;
488 unsigned long flags;
489
490 m = (struct mbuf *)mem;
491 flags = (unsigned long)arg;
492
493 if ((flags & MB_NOTAGS) == 0 && (m->m_flags & M_PKTHDR) != 0)
494 m_tag_delete_chain(m, NULL);
495 KASSERT((m->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__));
496 KASSERT((m->m_flags & M_NOFREE) == 0, ("%s: M_NOFREE set", __func__));
497#ifdef INVARIANTS
498 trash_dtor(mem, size, arg);
499#endif
500}
501
502/*
503 * The Mbuf Packet zone destructor.
504 */
505static void
506mb_dtor_pack(void *mem, int size, void *arg)
507{
508 struct mbuf *m;
509
510 m = (struct mbuf *)mem;
511 if ((m->m_flags & M_PKTHDR) != 0)
512 m_tag_delete_chain(m, NULL);
513
514 /* Make sure we've got a clean cluster back. */
515 KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
516 KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__));
517 KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__));
518 KASSERT(m->m_ext.ext_arg1 == NULL, ("%s: ext_arg1 != NULL", __func__));
519 KASSERT(m->m_ext.ext_arg2 == NULL, ("%s: ext_arg2 != NULL", __func__));
520 KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__));
521 KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__));
522 KASSERT(*m->m_ext.ref_cnt == 1, ("%s: ref_cnt != 1", __func__));
523#ifdef INVARIANTS
524 trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg);
525#endif
526 /*
527 * If there are processes blocked on zone_clust, waiting for pages
528 * to be freed up, * cause them to be woken up by draining the
529 * packet zone. We are exposed to a race here * (in the check for
530 * the UMA_ZFLAG_FULL) where we might miss the flag set, but that
531 * is deliberate. We don't want to acquire the zone lock for every
532 * mbuf free.
533 */
534 if (uma_zone_exhausted_nolock(zone_clust))
535 zone_drain(zone_pack);
536}
537
538/*
539 * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor.
540 *
541 * Here the 'arg' pointer points to the Mbuf which we
542 * are configuring cluster storage for. If 'arg' is
543 * empty we allocate just the cluster without setting
544 * the mbuf to it. See mbuf.h.
545 */
546static int
547mb_ctor_clust(void *mem, int size, void *arg, int how)
548{
549 struct mbuf *m;
550 u_int *refcnt;
551 int type;
552 uma_zone_t zone;
553
554#ifdef INVARIANTS
555 trash_ctor(mem, size, arg, how);
556#endif
557 switch (size) {
558 case MCLBYTES:
559 type = EXT_CLUSTER;
560 zone = zone_clust;
561 break;
562#if MJUMPAGESIZE != MCLBYTES
563 case MJUMPAGESIZE:
564 type = EXT_JUMBOP;
565 zone = zone_jumbop;
566 break;
567#endif
568 case MJUM9BYTES:
569 type = EXT_JUMBO9;
570 zone = zone_jumbo9;
571 break;
572 case MJUM16BYTES:
573 type = EXT_JUMBO16;
574 zone = zone_jumbo16;
575 break;
576 default:
577 panic("unknown cluster size");
578 break;
579 }
580
581 m = (struct mbuf *)arg;
582 refcnt = uma_find_refcnt(zone, mem);
583 *refcnt = 1;
584 if (m != NULL) {
585 m->m_ext.ext_buf = (caddr_t)mem;
586 m->m_data = m->m_ext.ext_buf;
587 m->m_flags |= M_EXT;
588 m->m_ext.ext_free = NULL;
589 m->m_ext.ext_arg1 = NULL;
590 m->m_ext.ext_arg2 = NULL;
591 m->m_ext.ext_size = size;
592 m->m_ext.ext_type = type;
593 m->m_ext.ref_cnt = refcnt;
594 }
595
596 return (0);
597}
598
599/*
600 * The Mbuf Cluster zone destructor.
601 */
602static void
603mb_dtor_clust(void *mem, int size, void *arg)
604{
605#ifdef INVARIANTS
606 uma_zone_t zone;
607
608 zone = m_getzone(size);
609 KASSERT(*(uma_find_refcnt(zone, mem)) <= 1,
610 ("%s: refcnt incorrect %u", __func__,
611 *(uma_find_refcnt(zone, mem))) );
612
613 trash_dtor(mem, size, arg);
614#endif
615}
616
617/*
618 * The Packet secondary zone's init routine, executed on the
619 * object's transition from mbuf keg slab to zone cache.
620 */
621static int
622mb_zinit_pack(void *mem, int size, int how)
623{
624 struct mbuf *m;
625
626 m = (struct mbuf *)mem; /* m is virgin. */
627 if (uma_zalloc_arg(zone_clust, m, how) == NULL ||
628 m->m_ext.ext_buf == NULL)
629 return (ENOMEM);
630 m->m_ext.ext_type = EXT_PACKET; /* Override. */
631#ifdef INVARIANTS
632 trash_init(m->m_ext.ext_buf, MCLBYTES, how);
633#endif
634 return (0);
635}
636
637/*
638 * The Packet secondary zone's fini routine, executed on the
639 * object's transition from zone cache to keg slab.
640 */
641static void
642mb_zfini_pack(void *mem, int size)
643{
644 struct mbuf *m;
645
646 m = (struct mbuf *)mem;
647#ifdef INVARIANTS
648 trash_fini(m->m_ext.ext_buf, MCLBYTES);
649#endif
650 uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL);
651#ifdef INVARIANTS
652 trash_dtor(mem, size, NULL);
653#endif
654}
655
656/*
657 * The "packet" keg constructor.
658 */
659static int
660mb_ctor_pack(void *mem, int size, void *arg, int how)
661{
662 struct mbuf *m;
663 struct mb_args *args;
664#ifdef MAC
665 int error;
666#endif
667 int flags;
668 short type;
669
670 m = (struct mbuf *)mem;
671 args = (struct mb_args *)arg;
672 flags = args->flags;
673 type = args->type;
674
675#ifdef INVARIANTS
676 trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how);
677#endif
678 m->m_next = NULL;
679 m->m_nextpkt = NULL;
680 m->m_data = m->m_ext.ext_buf;
681 m->m_len = 0;
682 m->m_flags = (flags | M_EXT);
683 m->m_type = type;
684
685 if (flags & M_PKTHDR) {
686 m->m_pkthdr.rcvif = NULL;
687 m->m_pkthdr.len = 0;
688 m->m_pkthdr.header = NULL;
689 m->m_pkthdr.csum_flags = 0;
690 m->m_pkthdr.csum_data = 0;
691 m->m_pkthdr.tso_segsz = 0;
692 m->m_pkthdr.ether_vtag = 0;
693 m->m_pkthdr.flowid = 0;
694 SLIST_INIT(&m->m_pkthdr.tags);
695#ifdef MAC
696 /* If the label init fails, fail the alloc */
697 error = mac_mbuf_init(m, how);
698 if (error)
699 return (error);
700#endif
701 }
702 /* m_ext is already initialized. */
703
704 return (0);
705}
706
707int
708m_pkthdr_init(struct mbuf *m, int how)
709{
710#ifdef MAC
711 int error;
712#endif
713 m->m_data = m->m_pktdat;
714 SLIST_INIT(&m->m_pkthdr.tags);
715 m->m_pkthdr.rcvif = NULL;
716 m->m_pkthdr.header = NULL;
717 m->m_pkthdr.len = 0;
718 m->m_pkthdr.flowid = 0;
719 m->m_pkthdr.csum_flags = 0;
720 m->m_pkthdr.csum_data = 0;
721 m->m_pkthdr.tso_segsz = 0;
722 m->m_pkthdr.ether_vtag = 0;
723#ifdef MAC
724 /* If the label init fails, fail the alloc */
725 error = mac_mbuf_init(m, how);
726 if (error)
727 return (error);
728#endif
729
730 return (0);
731}
732
733/*
734 * This is the protocol drain routine.
735 *
736 * No locks should be held when this is called. The drain routines have to
737 * presently acquire some locks which raises the possibility of lock order
738 * reversal.
739 */
740static void
741mb_reclaim(void *junk)
742{
743 struct domain *dp;
744 struct protosw *pr;
745
746 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL,
747 "mb_reclaim()");
748
749 for (dp = domains; dp != NULL; dp = dp->dom_next)
750 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
751 if (pr->pr_drain != NULL)
752 (*pr->pr_drain)();
753}