1/*-
2 * Copyright (c) 2004, 2005,
3 * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
| 1/*-
2 * Copyright (c) 2004, 2005,
3 * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
|
29__FBSDID("$FreeBSD: head/sys/kern/kern_mbuf.c 159481 2006-06-10 14:34:07Z rwatson $");
| 29__FBSDID("$FreeBSD: head/sys/kern/kern_mbuf.c 162377 2006-09-17 13:44:32Z andre $");
|
30
31#include "opt_mac.h"
32#include "opt_param.h"
33
34#include <sys/param.h>
35#include <sys/mac.h>
36#include <sys/malloc.h>
37#include <sys/systm.h>
38#include <sys/mbuf.h>
39#include <sys/domain.h>
40#include <sys/eventhandler.h>
41#include <sys/kernel.h>
42#include <sys/protosw.h>
43#include <sys/smp.h>
44#include <sys/sysctl.h>
45
46#include <vm/vm.h>
47#include <vm/vm_page.h>
48#include <vm/uma.h>
49#include <vm/uma_int.h>
50#include <vm/uma_dbg.h>
51
52/*
53 * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA
54 * Zones.
55 *
56 * Mbuf Clusters (2K, contiguous) are allocated from the Cluster
57 * Zone. The Zone can be capped at kern.ipc.nmbclusters, if the
58 * administrator so desires.
59 *
60 * Mbufs are allocated from a UMA Master Zone called the Mbuf
61 * Zone.
62 *
63 * Additionally, FreeBSD provides a Packet Zone, which it
64 * configures as a Secondary Zone to the Mbuf Master Zone,
65 * thus sharing backend Slab kegs with the Mbuf Master Zone.
66 *
67 * Thus common-case allocations and locking are simplified:
68 *
69 * m_clget() m_getcl()
70 * | |
71 * | .------------>[(Packet Cache)] m_get(), m_gethdr()
72 * | | [ Packet ] |
73 * [(Cluster Cache)] [ Secondary ] [ (Mbuf Cache) ]
74 * [ Cluster Zone ] [ Zone ] [ Mbuf Master Zone ]
75 * | \________ |
76 * [ Cluster Keg ] \ /
77 * | [ Mbuf Keg ]
78 * [ Cluster Slabs ] |
79 * | [ Mbuf Slabs ]
80 * \____________(VM)_________________/
81 *
82 *
83 * Whenever an object is allocated with uma_zalloc() out of
84 * one of the Zones its _ctor_ function is executed. The same
85 * for any deallocation through uma_zfree() the _dtor_ function
86 * is executed.
87 *
88 * Caches are per-CPU and are filled from the Master Zone.
89 *
90 * Whenever an object is allocated from the underlying global
91 * memory pool it gets pre-initialized with the _zinit_ functions.
92 * When the Keg's are overfull objects get decomissioned with
93 * _zfini_ functions and free'd back to the global memory pool.
94 *
95 */
96
97int nmbclusters; /* limits number of mbuf clusters */
98int nmbjumbop; /* limits number of page size jumbo clusters */
99int nmbjumbo9; /* limits number of 9k jumbo clusters */
100int nmbjumbo16; /* limits number of 16k jumbo clusters */
101struct mbstat mbstat;
102
103static void
104tunable_mbinit(void *dummy)
105{
106
107 /* This has to be done before VM init. */
108 nmbclusters = 1024 + maxusers * 64;
109 TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters);
110}
111SYSINIT(tunable_mbinit, SI_SUB_TUNABLES, SI_ORDER_ANY, tunable_mbinit, NULL);
112
113/* XXX: These should be tuneables. Can't change UMA limits on the fly. */
114static int
115sysctl_nmbclusters(SYSCTL_HANDLER_ARGS)
116{
117 int error, newnmbclusters;
118
119 newnmbclusters = nmbclusters;
120 error = sysctl_handle_int(oidp, &newnmbclusters, sizeof(int), req);
121 if (error == 0 && req->newptr) {
122 if (newnmbclusters > nmbclusters) {
123 nmbclusters = newnmbclusters;
124 uma_zone_set_max(zone_clust, nmbclusters);
125 EVENTHANDLER_INVOKE(nmbclusters_change);
126 } else
127 error = EINVAL;
128 }
129 return (error);
130}
131SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbclusters, CTLTYPE_INT|CTLFLAG_RW,
132&nmbclusters, 0, sysctl_nmbclusters, "IU",
133 "Maximum number of mbuf clusters allowed");
134SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbop, CTLFLAG_RW, &nmbjumbop, 0,
135 "Maximum number of mbuf page size jumbo clusters allowed");
136SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbo9, CTLFLAG_RW, &nmbjumbo9, 0,
137 "Maximum number of mbuf 9k jumbo clusters allowed");
138SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbo16, CTLFLAG_RW, &nmbjumbo16, 0,
139 "Maximum number of mbuf 16k jumbo clusters allowed");
140SYSCTL_STRUCT(_kern_ipc, OID_AUTO, mbstat, CTLFLAG_RD, &mbstat, mbstat,
141 "Mbuf general information and statistics");
142
143/*
144 * Zones from which we allocate.
145 */
146uma_zone_t zone_mbuf;
147uma_zone_t zone_clust;
148uma_zone_t zone_pack;
149uma_zone_t zone_jumbop;
150uma_zone_t zone_jumbo9;
151uma_zone_t zone_jumbo16;
152uma_zone_t zone_ext_refcnt;
| 30
31#include "opt_mac.h"
32#include "opt_param.h"
33
34#include <sys/param.h>
35#include <sys/mac.h>
36#include <sys/malloc.h>
37#include <sys/systm.h>
38#include <sys/mbuf.h>
39#include <sys/domain.h>
40#include <sys/eventhandler.h>
41#include <sys/kernel.h>
42#include <sys/protosw.h>
43#include <sys/smp.h>
44#include <sys/sysctl.h>
45
46#include <vm/vm.h>
47#include <vm/vm_page.h>
48#include <vm/uma.h>
49#include <vm/uma_int.h>
50#include <vm/uma_dbg.h>
51
52/*
53 * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA
54 * Zones.
55 *
56 * Mbuf Clusters (2K, contiguous) are allocated from the Cluster
57 * Zone. The Zone can be capped at kern.ipc.nmbclusters, if the
58 * administrator so desires.
59 *
60 * Mbufs are allocated from a UMA Master Zone called the Mbuf
61 * Zone.
62 *
63 * Additionally, FreeBSD provides a Packet Zone, which it
64 * configures as a Secondary Zone to the Mbuf Master Zone,
65 * thus sharing backend Slab kegs with the Mbuf Master Zone.
66 *
67 * Thus common-case allocations and locking are simplified:
68 *
69 * m_clget() m_getcl()
70 * | |
71 * | .------------>[(Packet Cache)] m_get(), m_gethdr()
72 * | | [ Packet ] |
73 * [(Cluster Cache)] [ Secondary ] [ (Mbuf Cache) ]
74 * [ Cluster Zone ] [ Zone ] [ Mbuf Master Zone ]
75 * | \________ |
76 * [ Cluster Keg ] \ /
77 * | [ Mbuf Keg ]
78 * [ Cluster Slabs ] |
79 * | [ Mbuf Slabs ]
80 * \____________(VM)_________________/
81 *
82 *
83 * Whenever an object is allocated with uma_zalloc() out of
84 * one of the Zones its _ctor_ function is executed. The same
85 * for any deallocation through uma_zfree() the _dtor_ function
86 * is executed.
87 *
88 * Caches are per-CPU and are filled from the Master Zone.
89 *
90 * Whenever an object is allocated from the underlying global
91 * memory pool it gets pre-initialized with the _zinit_ functions.
92 * When the Keg's are overfull objects get decomissioned with
93 * _zfini_ functions and free'd back to the global memory pool.
94 *
95 */
96
97int nmbclusters; /* limits number of mbuf clusters */
98int nmbjumbop; /* limits number of page size jumbo clusters */
99int nmbjumbo9; /* limits number of 9k jumbo clusters */
100int nmbjumbo16; /* limits number of 16k jumbo clusters */
101struct mbstat mbstat;
102
103static void
104tunable_mbinit(void *dummy)
105{
106
107 /* This has to be done before VM init. */
108 nmbclusters = 1024 + maxusers * 64;
109 TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters);
110}
111SYSINIT(tunable_mbinit, SI_SUB_TUNABLES, SI_ORDER_ANY, tunable_mbinit, NULL);
112
113/* XXX: These should be tuneables. Can't change UMA limits on the fly. */
114static int
115sysctl_nmbclusters(SYSCTL_HANDLER_ARGS)
116{
117 int error, newnmbclusters;
118
119 newnmbclusters = nmbclusters;
120 error = sysctl_handle_int(oidp, &newnmbclusters, sizeof(int), req);
121 if (error == 0 && req->newptr) {
122 if (newnmbclusters > nmbclusters) {
123 nmbclusters = newnmbclusters;
124 uma_zone_set_max(zone_clust, nmbclusters);
125 EVENTHANDLER_INVOKE(nmbclusters_change);
126 } else
127 error = EINVAL;
128 }
129 return (error);
130}
131SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbclusters, CTLTYPE_INT|CTLFLAG_RW,
132&nmbclusters, 0, sysctl_nmbclusters, "IU",
133 "Maximum number of mbuf clusters allowed");
134SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbop, CTLFLAG_RW, &nmbjumbop, 0,
135 "Maximum number of mbuf page size jumbo clusters allowed");
136SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbo9, CTLFLAG_RW, &nmbjumbo9, 0,
137 "Maximum number of mbuf 9k jumbo clusters allowed");
138SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbo16, CTLFLAG_RW, &nmbjumbo16, 0,
139 "Maximum number of mbuf 16k jumbo clusters allowed");
140SYSCTL_STRUCT(_kern_ipc, OID_AUTO, mbstat, CTLFLAG_RD, &mbstat, mbstat,
141 "Mbuf general information and statistics");
142
143/*
144 * Zones from which we allocate.
145 */
146uma_zone_t zone_mbuf;
147uma_zone_t zone_clust;
148uma_zone_t zone_pack;
149uma_zone_t zone_jumbop;
150uma_zone_t zone_jumbo9;
151uma_zone_t zone_jumbo16;
152uma_zone_t zone_ext_refcnt;
|
153uma_zone_t zone_mtag_vlan;
| |
154
155/*
156 * Local prototypes.
157 */
158static int mb_ctor_mbuf(void *, int, void *, int);
159static int mb_ctor_clust(void *, int, void *, int);
160static int mb_ctor_pack(void *, int, void *, int);
161static void mb_dtor_mbuf(void *, int, void *);
162static void mb_dtor_clust(void *, int, void *);
163static void mb_dtor_pack(void *, int, void *);
164static int mb_zinit_pack(void *, int, int);
165static void mb_zfini_pack(void *, int);
| 153
154/*
155 * Local prototypes.
156 */
157static int mb_ctor_mbuf(void *, int, void *, int);
158static int mb_ctor_clust(void *, int, void *, int);
159static int mb_ctor_pack(void *, int, void *, int);
160static void mb_dtor_mbuf(void *, int, void *);
161static void mb_dtor_clust(void *, int, void *);
162static void mb_dtor_pack(void *, int, void *);
163static int mb_zinit_pack(void *, int, int);
164static void mb_zfini_pack(void *, int);
|
166static int mt_zinit_vlan(void *, int, int);
| |
167
168static void mb_reclaim(void *);
169static void mbuf_init(void *);
170
171/* Ensure that MSIZE doesn't break dtom() - it must be a power of 2 */
172CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
173
174/*
175 * Initialize FreeBSD Network buffer allocation.
176 */
177SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL)
178static void
179mbuf_init(void *dummy)
180{
181
182 /*
183 * Configure UMA zones for Mbufs, Clusters, and Packets.
184 */
185 zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE,
186 mb_ctor_mbuf, mb_dtor_mbuf,
187#ifdef INVARIANTS
188 trash_init, trash_fini,
189#else
190 NULL, NULL,
191#endif
192 MSIZE - 1, UMA_ZONE_MAXBUCKET);
193
194 zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
195 mb_ctor_clust, mb_dtor_clust,
196#ifdef INVARIANTS
197 trash_init, trash_fini,
198#else
199 NULL, NULL,
200#endif
201 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
202 if (nmbclusters > 0)
203 uma_zone_set_max(zone_clust, nmbclusters);
204
205 zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
206 mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);
207
208 /* Make jumbo frame zone too. Page size, 9k and 16k. */
209 zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE,
210 mb_ctor_clust, mb_dtor_clust,
211#ifdef INVARIANTS
212 trash_init, trash_fini,
213#else
214 NULL, NULL,
215#endif
216 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
217 if (nmbjumbop > 0)
218 uma_zone_set_max(zone_jumbop, nmbjumbop);
219
220 zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
221 mb_ctor_clust, mb_dtor_clust,
222#ifdef INVARIANTS
223 trash_init, trash_fini,
224#else
225 NULL, NULL,
226#endif
227 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
228 if (nmbjumbo9 > 0)
229 uma_zone_set_max(zone_jumbo9, nmbjumbo9);
230
231 zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,
232 mb_ctor_clust, mb_dtor_clust,
233#ifdef INVARIANTS
234 trash_init, trash_fini,
235#else
236 NULL, NULL,
237#endif
238 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
239 if (nmbjumbo16 > 0)
240 uma_zone_set_max(zone_jumbo16, nmbjumbo16);
241
242 zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int),
243 NULL, NULL,
244 NULL, NULL,
245 UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
246
| 165
166static void mb_reclaim(void *);
167static void mbuf_init(void *);
168
169/* Ensure that MSIZE doesn't break dtom() - it must be a power of 2 */
170CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
171
172/*
173 * Initialize FreeBSD Network buffer allocation.
174 */
175SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL)
176static void
177mbuf_init(void *dummy)
178{
179
180 /*
181 * Configure UMA zones for Mbufs, Clusters, and Packets.
182 */
183 zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE,
184 mb_ctor_mbuf, mb_dtor_mbuf,
185#ifdef INVARIANTS
186 trash_init, trash_fini,
187#else
188 NULL, NULL,
189#endif
190 MSIZE - 1, UMA_ZONE_MAXBUCKET);
191
192 zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
193 mb_ctor_clust, mb_dtor_clust,
194#ifdef INVARIANTS
195 trash_init, trash_fini,
196#else
197 NULL, NULL,
198#endif
199 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
200 if (nmbclusters > 0)
201 uma_zone_set_max(zone_clust, nmbclusters);
202
203 zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
204 mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);
205
206 /* Make jumbo frame zone too. Page size, 9k and 16k. */
207 zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE,
208 mb_ctor_clust, mb_dtor_clust,
209#ifdef INVARIANTS
210 trash_init, trash_fini,
211#else
212 NULL, NULL,
213#endif
214 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
215 if (nmbjumbop > 0)
216 uma_zone_set_max(zone_jumbop, nmbjumbop);
217
218 zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
219 mb_ctor_clust, mb_dtor_clust,
220#ifdef INVARIANTS
221 trash_init, trash_fini,
222#else
223 NULL, NULL,
224#endif
225 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
226 if (nmbjumbo9 > 0)
227 uma_zone_set_max(zone_jumbo9, nmbjumbo9);
228
229 zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,
230 mb_ctor_clust, mb_dtor_clust,
231#ifdef INVARIANTS
232 trash_init, trash_fini,
233#else
234 NULL, NULL,
235#endif
236 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
237 if (nmbjumbo16 > 0)
238 uma_zone_set_max(zone_jumbo16, nmbjumbo16);
239
240 zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int),
241 NULL, NULL,
242 NULL, NULL,
243 UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
244
|
247 zone_mtag_vlan = uma_zcreate("mtag_vlan",
248 sizeof(struct m_tag) + sizeof(u_int),
249 NULL, NULL,
250 mt_zinit_vlan, NULL,
251 UMA_ALIGN_INT, 0);
252
| |
253 /* uma_prealloc() goes here... */
254
255 /*
256 * Hook event handler for low-memory situation, used to
257 * drain protocols and push data back to the caches (UMA
258 * later pushes it back to VM).
259 */
260 EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,
261 EVENTHANDLER_PRI_FIRST);
262
263 /*
264 * [Re]set counters and local statistics knobs.
265 * XXX Some of these should go and be replaced, but UMA stat
266 * gathering needs to be revised.
267 */
268 mbstat.m_mbufs = 0;
269 mbstat.m_mclusts = 0;
270 mbstat.m_drain = 0;
271 mbstat.m_msize = MSIZE;
272 mbstat.m_mclbytes = MCLBYTES;
273 mbstat.m_minclsize = MINCLSIZE;
274 mbstat.m_mlen = MLEN;
275 mbstat.m_mhlen = MHLEN;
276 mbstat.m_numtypes = MT_NTYPES;
277
278 mbstat.m_mcfail = mbstat.m_mpfail = 0;
279 mbstat.sf_iocnt = 0;
280 mbstat.sf_allocwait = mbstat.sf_allocfail = 0;
281}
282
283/*
284 * Constructor for Mbuf master zone.
285 *
286 * The 'arg' pointer points to a mb_args structure which
287 * contains call-specific information required to support the
288 * mbuf allocation API. See mbuf.h.
289 */
290static int
291mb_ctor_mbuf(void *mem, int size, void *arg, int how)
292{
293 struct mbuf *m;
294 struct mb_args *args;
295#ifdef MAC
296 int error;
297#endif
298 int flags;
299 short type;
300
301#ifdef INVARIANTS
302 trash_ctor(mem, size, arg, how);
303#endif
304 m = (struct mbuf *)mem;
305 args = (struct mb_args *)arg;
306 flags = args->flags;
307 type = args->type;
308
309 /*
310 * The mbuf is initialized later. The caller has the
311 * responsibility to set up any MAC labels too.
312 */
313 if (type == MT_NOINIT)
314 return (0);
315
316 m->m_next = NULL;
317 m->m_nextpkt = NULL;
318 m->m_len = 0;
319 m->m_flags = flags;
320 m->m_type = type;
321 if (flags & M_PKTHDR) {
322 m->m_data = m->m_pktdat;
323 m->m_pkthdr.rcvif = NULL;
324 m->m_pkthdr.len = 0;
325 m->m_pkthdr.header = NULL;
326 m->m_pkthdr.csum_flags = 0;
327 m->m_pkthdr.csum_data = 0;
| 245 /* uma_prealloc() goes here... */
246
247 /*
248 * Hook event handler for low-memory situation, used to
249 * drain protocols and push data back to the caches (UMA
250 * later pushes it back to VM).
251 */
252 EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,
253 EVENTHANDLER_PRI_FIRST);
254
255 /*
256 * [Re]set counters and local statistics knobs.
257 * XXX Some of these should go and be replaced, but UMA stat
258 * gathering needs to be revised.
259 */
260 mbstat.m_mbufs = 0;
261 mbstat.m_mclusts = 0;
262 mbstat.m_drain = 0;
263 mbstat.m_msize = MSIZE;
264 mbstat.m_mclbytes = MCLBYTES;
265 mbstat.m_minclsize = MINCLSIZE;
266 mbstat.m_mlen = MLEN;
267 mbstat.m_mhlen = MHLEN;
268 mbstat.m_numtypes = MT_NTYPES;
269
270 mbstat.m_mcfail = mbstat.m_mpfail = 0;
271 mbstat.sf_iocnt = 0;
272 mbstat.sf_allocwait = mbstat.sf_allocfail = 0;
273}
274
275/*
276 * Constructor for Mbuf master zone.
277 *
278 * The 'arg' pointer points to a mb_args structure which
279 * contains call-specific information required to support the
280 * mbuf allocation API. See mbuf.h.
281 */
282static int
283mb_ctor_mbuf(void *mem, int size, void *arg, int how)
284{
285 struct mbuf *m;
286 struct mb_args *args;
287#ifdef MAC
288 int error;
289#endif
290 int flags;
291 short type;
292
293#ifdef INVARIANTS
294 trash_ctor(mem, size, arg, how);
295#endif
296 m = (struct mbuf *)mem;
297 args = (struct mb_args *)arg;
298 flags = args->flags;
299 type = args->type;
300
301 /*
302 * The mbuf is initialized later. The caller has the
303 * responsibility to set up any MAC labels too.
304 */
305 if (type == MT_NOINIT)
306 return (0);
307
308 m->m_next = NULL;
309 m->m_nextpkt = NULL;
310 m->m_len = 0;
311 m->m_flags = flags;
312 m->m_type = type;
313 if (flags & M_PKTHDR) {
314 m->m_data = m->m_pktdat;
315 m->m_pkthdr.rcvif = NULL;
316 m->m_pkthdr.len = 0;
317 m->m_pkthdr.header = NULL;
318 m->m_pkthdr.csum_flags = 0;
319 m->m_pkthdr.csum_data = 0;
|
| 320 m->m_pkthdr.tso_segsz = 0;
321 m->m_pkthdr.ether_vtag = 0;
|
328 SLIST_INIT(&m->m_pkthdr.tags);
329#ifdef MAC
330 /* If the label init fails, fail the alloc */
331 error = mac_init_mbuf(m, how);
332 if (error)
333 return (error);
334#endif
335 } else
336 m->m_data = m->m_dat;
337 return (0);
338}
339
340/*
341 * The Mbuf master zone destructor.
342 */
343static void
344mb_dtor_mbuf(void *mem, int size, void *arg)
345{
346 struct mbuf *m;
347
348 m = (struct mbuf *)mem;
349 if ((m->m_flags & M_PKTHDR) != 0)
350 m_tag_delete_chain(m, NULL);
351 KASSERT((m->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__));
352#ifdef INVARIANTS
353 trash_dtor(mem, size, arg);
354#endif
355}
356
357/*
358 * The Mbuf Packet zone destructor.
359 */
360static void
361mb_dtor_pack(void *mem, int size, void *arg)
362{
363 struct mbuf *m;
364
365 m = (struct mbuf *)mem;
366 if ((m->m_flags & M_PKTHDR) != 0)
367 m_tag_delete_chain(m, NULL);
368
369 /* Make sure we've got a clean cluster back. */
370 KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
371 KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__));
372 KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__));
373 KASSERT(m->m_ext.ext_args == NULL, ("%s: ext_args != NULL", __func__));
374 KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__));
375 KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__));
376 KASSERT(*m->m_ext.ref_cnt == 1, ("%s: ref_cnt != 1", __func__));
377#ifdef INVARIANTS
378 trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg);
379#endif
380}
381
382/*
383 * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor.
384 *
385 * Here the 'arg' pointer points to the Mbuf which we
386 * are configuring cluster storage for. If 'arg' is
387 * empty we allocate just the cluster without setting
388 * the mbuf to it. See mbuf.h.
389 */
390static int
391mb_ctor_clust(void *mem, int size, void *arg, int how)
392{
393 struct mbuf *m;
394 u_int *refcnt;
395 int type = 0;
396
397#ifdef INVARIANTS
398 trash_ctor(mem, size, arg, how);
399#endif
400 m = (struct mbuf *)arg;
401 if (m != NULL) {
402 switch (size) {
403 case MCLBYTES:
404 type = EXT_CLUSTER;
405 break;
406#if MJUMPAGESIZE != MCLBYTES
407 case MJUMPAGESIZE:
408 type = EXT_JUMBOP;
409 break;
410#endif
411 case MJUM9BYTES:
412 type = EXT_JUMBO9;
413 break;
414 case MJUM16BYTES:
415 type = EXT_JUMBO16;
416 break;
417 default:
418 panic("unknown cluster size");
419 break;
420 }
421 m->m_ext.ext_buf = (caddr_t)mem;
422 m->m_data = m->m_ext.ext_buf;
423 m->m_flags |= M_EXT;
424 m->m_ext.ext_free = NULL;
425 m->m_ext.ext_args = NULL;
426 m->m_ext.ext_size = size;
427 m->m_ext.ext_type = type;
428 m->m_ext.ref_cnt = uma_find_refcnt(zone_clust, mem);
429 *m->m_ext.ref_cnt = 1;
430 } else {
431 refcnt = uma_find_refcnt(zone_clust, mem);
432 *refcnt = 1;
433 }
434 return (0);
435}
436
437/*
438 * The Mbuf Cluster zone destructor.
439 */
440static void
441mb_dtor_clust(void *mem, int size, void *arg)
442{
443
444 KASSERT(*(uma_find_refcnt(zone_clust, mem)) <= 1,
445 ("%s: refcnt incorrect %u", __func__,
446 *(uma_find_refcnt(zone_clust, mem))) );
447#ifdef INVARIANTS
448 trash_dtor(mem, size, arg);
449#endif
450}
451
452/*
453 * The Packet secondary zone's init routine, executed on the
454 * object's transition from mbuf keg slab to zone cache.
455 */
456static int
457mb_zinit_pack(void *mem, int size, int how)
458{
459 struct mbuf *m;
460
461 m = (struct mbuf *)mem; /* m is virgin. */
462 if (uma_zalloc_arg(zone_clust, m, how) == NULL ||
463 m->m_ext.ext_buf == NULL)
464 return (ENOMEM);
465 m->m_ext.ext_type = EXT_PACKET; /* Override. */
466#ifdef INVARIANTS
467 trash_init(m->m_ext.ext_buf, MCLBYTES, how);
468#endif
469 return (0);
470}
471
472/*
473 * The Packet secondary zone's fini routine, executed on the
474 * object's transition from zone cache to keg slab.
475 */
476static void
477mb_zfini_pack(void *mem, int size)
478{
479 struct mbuf *m;
480
481 m = (struct mbuf *)mem;
482#ifdef INVARIANTS
483 trash_fini(m->m_ext.ext_buf, MCLBYTES);
484#endif
485 uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL);
486#ifdef INVARIANTS
487 trash_dtor(mem, size, NULL);
488#endif
489}
490
491/*
492 * The "packet" keg constructor.
493 */
494static int
495mb_ctor_pack(void *mem, int size, void *arg, int how)
496{
497 struct mbuf *m;
498 struct mb_args *args;
499#ifdef MAC
500 int error;
501#endif
502 int flags;
503 short type;
504
505 m = (struct mbuf *)mem;
506 args = (struct mb_args *)arg;
507 flags = args->flags;
508 type = args->type;
509
510#ifdef INVARIANTS
511 trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how);
512#endif
513 m->m_next = NULL;
514 m->m_nextpkt = NULL;
515 m->m_data = m->m_ext.ext_buf;
516 m->m_len = 0;
517 m->m_flags = (flags | M_EXT);
518 m->m_type = type;
519
520 if (flags & M_PKTHDR) {
521 m->m_pkthdr.rcvif = NULL;
522 m->m_pkthdr.len = 0;
523 m->m_pkthdr.header = NULL;
524 m->m_pkthdr.csum_flags = 0;
525 m->m_pkthdr.csum_data = 0;
| 322 SLIST_INIT(&m->m_pkthdr.tags);
323#ifdef MAC
324 /* If the label init fails, fail the alloc */
325 error = mac_init_mbuf(m, how);
326 if (error)
327 return (error);
328#endif
329 } else
330 m->m_data = m->m_dat;
331 return (0);
332}
333
334/*
335 * The Mbuf master zone destructor.
336 */
337static void
338mb_dtor_mbuf(void *mem, int size, void *arg)
339{
340 struct mbuf *m;
341
342 m = (struct mbuf *)mem;
343 if ((m->m_flags & M_PKTHDR) != 0)
344 m_tag_delete_chain(m, NULL);
345 KASSERT((m->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__));
346#ifdef INVARIANTS
347 trash_dtor(mem, size, arg);
348#endif
349}
350
351/*
352 * The Mbuf Packet zone destructor.
353 */
354static void
355mb_dtor_pack(void *mem, int size, void *arg)
356{
357 struct mbuf *m;
358
359 m = (struct mbuf *)mem;
360 if ((m->m_flags & M_PKTHDR) != 0)
361 m_tag_delete_chain(m, NULL);
362
363 /* Make sure we've got a clean cluster back. */
364 KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
365 KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__));
366 KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__));
367 KASSERT(m->m_ext.ext_args == NULL, ("%s: ext_args != NULL", __func__));
368 KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__));
369 KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__));
370 KASSERT(*m->m_ext.ref_cnt == 1, ("%s: ref_cnt != 1", __func__));
371#ifdef INVARIANTS
372 trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg);
373#endif
374}
375
376/*
377 * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor.
378 *
379 * Here the 'arg' pointer points to the Mbuf which we
380 * are configuring cluster storage for. If 'arg' is
381 * empty we allocate just the cluster without setting
382 * the mbuf to it. See mbuf.h.
383 */
384static int
385mb_ctor_clust(void *mem, int size, void *arg, int how)
386{
387 struct mbuf *m;
388 u_int *refcnt;
389 int type = 0;
390
391#ifdef INVARIANTS
392 trash_ctor(mem, size, arg, how);
393#endif
394 m = (struct mbuf *)arg;
395 if (m != NULL) {
396 switch (size) {
397 case MCLBYTES:
398 type = EXT_CLUSTER;
399 break;
400#if MJUMPAGESIZE != MCLBYTES
401 case MJUMPAGESIZE:
402 type = EXT_JUMBOP;
403 break;
404#endif
405 case MJUM9BYTES:
406 type = EXT_JUMBO9;
407 break;
408 case MJUM16BYTES:
409 type = EXT_JUMBO16;
410 break;
411 default:
412 panic("unknown cluster size");
413 break;
414 }
415 m->m_ext.ext_buf = (caddr_t)mem;
416 m->m_data = m->m_ext.ext_buf;
417 m->m_flags |= M_EXT;
418 m->m_ext.ext_free = NULL;
419 m->m_ext.ext_args = NULL;
420 m->m_ext.ext_size = size;
421 m->m_ext.ext_type = type;
422 m->m_ext.ref_cnt = uma_find_refcnt(zone_clust, mem);
423 *m->m_ext.ref_cnt = 1;
424 } else {
425 refcnt = uma_find_refcnt(zone_clust, mem);
426 *refcnt = 1;
427 }
428 return (0);
429}
430
431/*
432 * The Mbuf Cluster zone destructor.
433 */
434static void
435mb_dtor_clust(void *mem, int size, void *arg)
436{
437
438 KASSERT(*(uma_find_refcnt(zone_clust, mem)) <= 1,
439 ("%s: refcnt incorrect %u", __func__,
440 *(uma_find_refcnt(zone_clust, mem))) );
441#ifdef INVARIANTS
442 trash_dtor(mem, size, arg);
443#endif
444}
445
446/*
447 * The Packet secondary zone's init routine, executed on the
448 * object's transition from mbuf keg slab to zone cache.
449 */
450static int
451mb_zinit_pack(void *mem, int size, int how)
452{
453 struct mbuf *m;
454
455 m = (struct mbuf *)mem; /* m is virgin. */
456 if (uma_zalloc_arg(zone_clust, m, how) == NULL ||
457 m->m_ext.ext_buf == NULL)
458 return (ENOMEM);
459 m->m_ext.ext_type = EXT_PACKET; /* Override. */
460#ifdef INVARIANTS
461 trash_init(m->m_ext.ext_buf, MCLBYTES, how);
462#endif
463 return (0);
464}
465
466/*
467 * The Packet secondary zone's fini routine, executed on the
468 * object's transition from zone cache to keg slab.
469 */
470static void
471mb_zfini_pack(void *mem, int size)
472{
473 struct mbuf *m;
474
475 m = (struct mbuf *)mem;
476#ifdef INVARIANTS
477 trash_fini(m->m_ext.ext_buf, MCLBYTES);
478#endif
479 uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL);
480#ifdef INVARIANTS
481 trash_dtor(mem, size, NULL);
482#endif
483}
484
485/*
486 * The "packet" keg constructor.
487 */
488static int
489mb_ctor_pack(void *mem, int size, void *arg, int how)
490{
491 struct mbuf *m;
492 struct mb_args *args;
493#ifdef MAC
494 int error;
495#endif
496 int flags;
497 short type;
498
499 m = (struct mbuf *)mem;
500 args = (struct mb_args *)arg;
501 flags = args->flags;
502 type = args->type;
503
504#ifdef INVARIANTS
505 trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how);
506#endif
507 m->m_next = NULL;
508 m->m_nextpkt = NULL;
509 m->m_data = m->m_ext.ext_buf;
510 m->m_len = 0;
511 m->m_flags = (flags | M_EXT);
512 m->m_type = type;
513
514 if (flags & M_PKTHDR) {
515 m->m_pkthdr.rcvif = NULL;
516 m->m_pkthdr.len = 0;
517 m->m_pkthdr.header = NULL;
518 m->m_pkthdr.csum_flags = 0;
519 m->m_pkthdr.csum_data = 0;
|
| 520 m->m_pkthdr.tso_segsz = 0;
521 m->m_pkthdr.ether_vtag = 0;
|
526 SLIST_INIT(&m->m_pkthdr.tags);
527#ifdef MAC
528 /* If the label init fails, fail the alloc */
529 error = mac_init_mbuf(m, how);
530 if (error)
531 return (error);
532#endif
533 }
534 /* m_ext is already initialized. */
535
536 return (0);
537}
538
| 522 SLIST_INIT(&m->m_pkthdr.tags);
523#ifdef MAC
524 /* If the label init fails, fail the alloc */
525 error = mac_init_mbuf(m, how);
526 if (error)
527 return (error);
528#endif
529 }
530 /* m_ext is already initialized. */
531
532 return (0);
533}
534
|
539static void
540mt_vlan_free(struct m_tag *mtag)
541{
542 uma_zfree(zone_mtag_vlan, mtag);
543}
544
545static int
546mt_zinit_vlan(void *mem, int size, int how)
547{
548 struct m_tag *mtag = (struct m_tag *)mem;
549
550 m_tag_setup(mtag, MTAG_VLAN, MTAG_VLAN_TAG, sizeof(u_int));
551 mtag->m_tag_free = mt_vlan_free;
552
553 return (0);
554}
555
| |
556/*
557 * This is the protocol drain routine.
558 *
559 * No locks should be held when this is called. The drain routines have to
560 * presently acquire some locks which raises the possibility of lock order
561 * reversal.
562 */
563static void
564mb_reclaim(void *junk)
565{
566 struct domain *dp;
567 struct protosw *pr;
568
569 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL,
570 "mb_reclaim()");
571
572 for (dp = domains; dp != NULL; dp = dp->dom_next)
573 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
574 if (pr->pr_drain != NULL)
575 (*pr->pr_drain)();
576}
| 535/*
536 * This is the protocol drain routine.
537 *
538 * No locks should be held when this is called. The drain routines have to
539 * presently acquire some locks which raises the possibility of lock order
540 * reversal.
541 */
542static void
543mb_reclaim(void *junk)
544{
545 struct domain *dp;
546 struct protosw *pr;
547
548 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL,
549 "mb_reclaim()");
550
551 for (dp = domains; dp != NULL; dp = dp->dom_next)
552 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
553 if (pr->pr_drain != NULL)
554 (*pr->pr_drain)();
555}
|