140}
141SYSINIT(tunable_mbinit, SI_SUB_TUNABLES, SI_ORDER_MIDDLE, tunable_mbinit, NULL);
142
143static int
144sysctl_nmbclusters(SYSCTL_HANDLER_ARGS)
145{
146 int error, newnmbclusters;
147
148 newnmbclusters = nmbclusters;
149 error = sysctl_handle_int(oidp, &newnmbclusters, 0, req);
150 if (error == 0 && req->newptr) {
151 if (newnmbclusters > nmbclusters &&
152 nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
153 nmbclusters = newnmbclusters;
154 uma_zone_set_max(zone_clust, nmbclusters);
155 nmbclusters = uma_zone_get_max(zone_clust);
156 EVENTHANDLER_INVOKE(nmbclusters_change);
157 } else
158 error = EINVAL;
159 }
160 return (error);
161}
162SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbclusters, CTLTYPE_INT|CTLFLAG_RW,
163&nmbclusters, 0, sysctl_nmbclusters, "IU",
164 "Maximum number of mbuf clusters allowed");
165
166static int
167sysctl_nmbjumbop(SYSCTL_HANDLER_ARGS)
168{
169 int error, newnmbjumbop;
170
171 newnmbjumbop = nmbjumbop;
172 error = sysctl_handle_int(oidp, &newnmbjumbop, 0, req);
173 if (error == 0 && req->newptr) {
174 if (newnmbjumbop > nmbjumbop &&
175 nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
176 nmbjumbop = newnmbjumbop;
177 uma_zone_set_max(zone_jumbop, nmbjumbop);
178 nmbjumbop = uma_zone_get_max(zone_jumbop);
179 } else
180 error = EINVAL;
181 }
182 return (error);
183}
184SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbop, CTLTYPE_INT|CTLFLAG_RW,
185&nmbjumbop, 0, sysctl_nmbjumbop, "IU",
186 "Maximum number of mbuf page size jumbo clusters allowed");
187
188
189static int
190sysctl_nmbjumbo9(SYSCTL_HANDLER_ARGS)
191{
192 int error, newnmbjumbo9;
193
194 newnmbjumbo9 = nmbjumbo9;
195 error = sysctl_handle_int(oidp, &newnmbjumbo9, 0, req);
196 if (error == 0 && req->newptr) {
197 if (newnmbjumbo9 > nmbjumbo9&&
198 nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
199 nmbjumbo9 = newnmbjumbo9;
200 uma_zone_set_max(zone_jumbo9, nmbjumbo9);
201 nmbjumbo9 = uma_zone_get_max(zone_jumbo9);
202 } else
203 error = EINVAL;
204 }
205 return (error);
206}
207SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo9, CTLTYPE_INT|CTLFLAG_RW,
208&nmbjumbo9, 0, sysctl_nmbjumbo9, "IU",
209 "Maximum number of mbuf 9k jumbo clusters allowed");
210
211static int
212sysctl_nmbjumbo16(SYSCTL_HANDLER_ARGS)
213{
214 int error, newnmbjumbo16;
215
216 newnmbjumbo16 = nmbjumbo16;
217 error = sysctl_handle_int(oidp, &newnmbjumbo16, 0, req);
218 if (error == 0 && req->newptr) {
219 if (newnmbjumbo16 > nmbjumbo16 &&
220 nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
221 nmbjumbo16 = newnmbjumbo16;
222 uma_zone_set_max(zone_jumbo16, nmbjumbo16);
223 nmbjumbo16 = uma_zone_get_max(zone_jumbo16);
224 } else
225 error = EINVAL;
226 }
227 return (error);
228}
229SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo16, CTLTYPE_INT|CTLFLAG_RW,
230&nmbjumbo16, 0, sysctl_nmbjumbo16, "IU",
231 "Maximum number of mbuf 16k jumbo clusters allowed");
232
233static int
234sysctl_nmbufs(SYSCTL_HANDLER_ARGS)
235{
236 int error, newnmbufs;
237
238 newnmbufs = nmbufs;
239 error = sysctl_handle_int(oidp, &newnmbufs, 0, req);
240 if (error == 0 && req->newptr) {
241 if (newnmbufs > nmbufs) {
242 nmbufs = newnmbufs;
243 uma_zone_set_max(zone_mbuf, nmbufs);
| 142}
143SYSINIT(tunable_mbinit, SI_SUB_TUNABLES, SI_ORDER_MIDDLE, tunable_mbinit, NULL);
144
145static int
146sysctl_nmbclusters(SYSCTL_HANDLER_ARGS)
147{
148 int error, newnmbclusters;
149
150 newnmbclusters = nmbclusters;
151 error = sysctl_handle_int(oidp, &newnmbclusters, 0, req);
152 if (error == 0 && req->newptr) {
153 if (newnmbclusters > nmbclusters &&
154 nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
155 nmbclusters = newnmbclusters;
156 uma_zone_set_max(zone_clust, nmbclusters);
157 nmbclusters = uma_zone_get_max(zone_clust);
158 EVENTHANDLER_INVOKE(nmbclusters_change);
159 } else
160 error = EINVAL;
161 }
162 return (error);
163}
164SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbclusters, CTLTYPE_INT|CTLFLAG_RW,
165&nmbclusters, 0, sysctl_nmbclusters, "IU",
166 "Maximum number of mbuf clusters allowed");
167
168static int
169sysctl_nmbjumbop(SYSCTL_HANDLER_ARGS)
170{
171 int error, newnmbjumbop;
172
173 newnmbjumbop = nmbjumbop;
174 error = sysctl_handle_int(oidp, &newnmbjumbop, 0, req);
175 if (error == 0 && req->newptr) {
176 if (newnmbjumbop > nmbjumbop &&
177 nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
178 nmbjumbop = newnmbjumbop;
179 uma_zone_set_max(zone_jumbop, nmbjumbop);
180 nmbjumbop = uma_zone_get_max(zone_jumbop);
181 } else
182 error = EINVAL;
183 }
184 return (error);
185}
186SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbop, CTLTYPE_INT|CTLFLAG_RW,
187&nmbjumbop, 0, sysctl_nmbjumbop, "IU",
188 "Maximum number of mbuf page size jumbo clusters allowed");
189
190
191static int
192sysctl_nmbjumbo9(SYSCTL_HANDLER_ARGS)
193{
194 int error, newnmbjumbo9;
195
196 newnmbjumbo9 = nmbjumbo9;
197 error = sysctl_handle_int(oidp, &newnmbjumbo9, 0, req);
198 if (error == 0 && req->newptr) {
199 if (newnmbjumbo9 > nmbjumbo9&&
200 nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
201 nmbjumbo9 = newnmbjumbo9;
202 uma_zone_set_max(zone_jumbo9, nmbjumbo9);
203 nmbjumbo9 = uma_zone_get_max(zone_jumbo9);
204 } else
205 error = EINVAL;
206 }
207 return (error);
208}
209SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo9, CTLTYPE_INT|CTLFLAG_RW,
210&nmbjumbo9, 0, sysctl_nmbjumbo9, "IU",
211 "Maximum number of mbuf 9k jumbo clusters allowed");
212
213static int
214sysctl_nmbjumbo16(SYSCTL_HANDLER_ARGS)
215{
216 int error, newnmbjumbo16;
217
218 newnmbjumbo16 = nmbjumbo16;
219 error = sysctl_handle_int(oidp, &newnmbjumbo16, 0, req);
220 if (error == 0 && req->newptr) {
221 if (newnmbjumbo16 > nmbjumbo16 &&
222 nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
223 nmbjumbo16 = newnmbjumbo16;
224 uma_zone_set_max(zone_jumbo16, nmbjumbo16);
225 nmbjumbo16 = uma_zone_get_max(zone_jumbo16);
226 } else
227 error = EINVAL;
228 }
229 return (error);
230}
231SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo16, CTLTYPE_INT|CTLFLAG_RW,
232&nmbjumbo16, 0, sysctl_nmbjumbo16, "IU",
233 "Maximum number of mbuf 16k jumbo clusters allowed");
234
235static int
236sysctl_nmbufs(SYSCTL_HANDLER_ARGS)
237{
238 int error, newnmbufs;
239
240 newnmbufs = nmbufs;
241 error = sysctl_handle_int(oidp, &newnmbufs, 0, req);
242 if (error == 0 && req->newptr) {
243 if (newnmbufs > nmbufs) {
244 nmbufs = newnmbufs;
245 uma_zone_set_max(zone_mbuf, nmbufs);
|
245 EVENTHANDLER_INVOKE(nmbufs_change);
246 } else
247 error = EINVAL;
248 }
249 return (error);
250}
251SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbuf, CTLTYPE_INT|CTLFLAG_RW,
252&nmbufs, 0, sysctl_nmbufs, "IU",
253 "Maximum number of mbufs allowed");
254
255
256SYSCTL_STRUCT(_kern_ipc, OID_AUTO, mbstat, CTLFLAG_RD, &mbstat, mbstat,
257 "Mbuf general information and statistics");
258
259/*
260 * Zones from which we allocate.
261 */
262uma_zone_t zone_mbuf;
263uma_zone_t zone_clust;
264uma_zone_t zone_pack;
265uma_zone_t zone_jumbop;
266uma_zone_t zone_jumbo9;
267uma_zone_t zone_jumbo16;
268uma_zone_t zone_ext_refcnt;
269
270/*
271 * Local prototypes.
272 */
273static int mb_ctor_mbuf(void *, int, void *, int);
274static int mb_ctor_clust(void *, int, void *, int);
275static int mb_ctor_pack(void *, int, void *, int);
276static void mb_dtor_mbuf(void *, int, void *);
277static void mb_dtor_clust(void *, int, void *);
278static void mb_dtor_pack(void *, int, void *);
279static int mb_zinit_pack(void *, int, int);
280static void mb_zfini_pack(void *, int);
281
282static void mb_reclaim(void *);
283static void mbuf_init(void *);
284static void *mbuf_jumbo_alloc(uma_zone_t, int, uint8_t *, int);
285
286/* Ensure that MSIZE must be a power of 2. */
287CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
288
289/*
290 * Initialize FreeBSD Network buffer allocation.
291 */
292SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL);
293static void
294mbuf_init(void *dummy)
295{
296
297 /*
298 * Configure UMA zones for Mbufs, Clusters, and Packets.
299 */
300 zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE,
301 mb_ctor_mbuf, mb_dtor_mbuf,
302#ifdef INVARIANTS
303 trash_init, trash_fini,
304#else
305 NULL, NULL,
306#endif
307 MSIZE - 1, UMA_ZONE_MAXBUCKET);
308 if (nmbufs > 0) {
309 uma_zone_set_max(zone_mbuf, nmbufs);
310 nmbufs = uma_zone_get_max(zone_mbuf);
311 }
312
313 zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
314 mb_ctor_clust, mb_dtor_clust,
315#ifdef INVARIANTS
316 trash_init, trash_fini,
317#else
318 NULL, NULL,
319#endif
320 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
321 if (nmbclusters > 0) {
322 uma_zone_set_max(zone_clust, nmbclusters);
323 nmbclusters = uma_zone_get_max(zone_clust);
324 }
325
326 zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
327 mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);
328
329 /* Make jumbo frame zone too. Page size, 9k and 16k. */
330 zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE,
331 mb_ctor_clust, mb_dtor_clust,
332#ifdef INVARIANTS
333 trash_init, trash_fini,
334#else
335 NULL, NULL,
336#endif
337 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
338 if (nmbjumbop > 0) {
339 uma_zone_set_max(zone_jumbop, nmbjumbop);
340 nmbjumbop = uma_zone_get_max(zone_jumbop);
341 }
342
343 zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
344 mb_ctor_clust, mb_dtor_clust,
345#ifdef INVARIANTS
346 trash_init, trash_fini,
347#else
348 NULL, NULL,
349#endif
350 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
351 uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc);
352 if (nmbjumbo9 > 0) {
353 uma_zone_set_max(zone_jumbo9, nmbjumbo9);
354 nmbjumbo9 = uma_zone_get_max(zone_jumbo9);
355 }
356
357 zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,
358 mb_ctor_clust, mb_dtor_clust,
359#ifdef INVARIANTS
360 trash_init, trash_fini,
361#else
362 NULL, NULL,
363#endif
364 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
365 uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc);
366 if (nmbjumbo16 > 0) {
367 uma_zone_set_max(zone_jumbo16, nmbjumbo16);
368 nmbjumbo16 = uma_zone_get_max(zone_jumbo16);
369 }
370
371 zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int),
372 NULL, NULL,
373 NULL, NULL,
374 UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
375
376 /* uma_prealloc() goes here... */
377
378 /*
379 * Hook event handler for low-memory situation, used to
380 * drain protocols and push data back to the caches (UMA
381 * later pushes it back to VM).
382 */
383 EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,
384 EVENTHANDLER_PRI_FIRST);
385
386 /*
387 * [Re]set counters and local statistics knobs.
388 * XXX Some of these should go and be replaced, but UMA stat
389 * gathering needs to be revised.
390 */
391 mbstat.m_mbufs = 0;
392 mbstat.m_mclusts = 0;
393 mbstat.m_drain = 0;
394 mbstat.m_msize = MSIZE;
395 mbstat.m_mclbytes = MCLBYTES;
396 mbstat.m_minclsize = MINCLSIZE;
397 mbstat.m_mlen = MLEN;
398 mbstat.m_mhlen = MHLEN;
399 mbstat.m_numtypes = MT_NTYPES;
400
401 mbstat.m_mcfail = mbstat.m_mpfail = 0;
402 mbstat.sf_iocnt = 0;
403 mbstat.sf_allocwait = mbstat.sf_allocfail = 0;
404}
405
406/*
407 * UMA backend page allocator for the jumbo frame zones.
408 *
409 * Allocates kernel virtual memory that is backed by contiguous physical
410 * pages.
411 */
412static void *
413mbuf_jumbo_alloc(uma_zone_t zone, int bytes, uint8_t *flags, int wait)
414{
415
416 /* Inform UMA that this allocator uses kernel_map/object. */
417 *flags = UMA_SLAB_KERNEL;
418 return ((void *)kmem_alloc_contig(kernel_map, bytes, wait,
419 (vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0, VM_MEMATTR_DEFAULT));
420}
421
422/*
423 * Constructor for Mbuf master zone.
424 *
425 * The 'arg' pointer points to a mb_args structure which
426 * contains call-specific information required to support the
427 * mbuf allocation API. See mbuf.h.
428 */
429static int
430mb_ctor_mbuf(void *mem, int size, void *arg, int how)
431{
432 struct mbuf *m;
433 struct mb_args *args;
434#ifdef MAC
435 int error;
436#endif
437 int flags;
438 short type;
439
440#ifdef INVARIANTS
441 trash_ctor(mem, size, arg, how);
442#endif
443 m = (struct mbuf *)mem;
444 args = (struct mb_args *)arg;
445 flags = args->flags;
446 type = args->type;
447
448 /*
449 * The mbuf is initialized later. The caller has the
450 * responsibility to set up any MAC labels too.
451 */
452 if (type == MT_NOINIT)
453 return (0);
454
455 m->m_next = NULL;
456 m->m_nextpkt = NULL;
457 m->m_len = 0;
458 m->m_flags = flags;
459 m->m_type = type;
460 if (flags & M_PKTHDR) {
461 m->m_data = m->m_pktdat;
462 m->m_pkthdr.rcvif = NULL;
463 m->m_pkthdr.header = NULL;
464 m->m_pkthdr.len = 0;
465 m->m_pkthdr.csum_flags = 0;
466 m->m_pkthdr.csum_data = 0;
467 m->m_pkthdr.tso_segsz = 0;
468 m->m_pkthdr.ether_vtag = 0;
469 m->m_pkthdr.flowid = 0;
470 SLIST_INIT(&m->m_pkthdr.tags);
471#ifdef MAC
472 /* If the label init fails, fail the alloc */
473 error = mac_mbuf_init(m, how);
474 if (error)
475 return (error);
476#endif
477 } else
478 m->m_data = m->m_dat;
479 return (0);
480}
481
482/*
483 * The Mbuf master zone destructor.
484 */
485static void
486mb_dtor_mbuf(void *mem, int size, void *arg)
487{
488 struct mbuf *m;
489 unsigned long flags;
490
491 m = (struct mbuf *)mem;
492 flags = (unsigned long)arg;
493
494 if ((flags & MB_NOTAGS) == 0 && (m->m_flags & M_PKTHDR) != 0)
495 m_tag_delete_chain(m, NULL);
496 KASSERT((m->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__));
497 KASSERT((m->m_flags & M_NOFREE) == 0, ("%s: M_NOFREE set", __func__));
498#ifdef INVARIANTS
499 trash_dtor(mem, size, arg);
500#endif
501}
502
503/*
504 * The Mbuf Packet zone destructor.
505 */
506static void
507mb_dtor_pack(void *mem, int size, void *arg)
508{
509 struct mbuf *m;
510
511 m = (struct mbuf *)mem;
512 if ((m->m_flags & M_PKTHDR) != 0)
513 m_tag_delete_chain(m, NULL);
514
515 /* Make sure we've got a clean cluster back. */
516 KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
517 KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__));
518 KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__));
519 KASSERT(m->m_ext.ext_arg1 == NULL, ("%s: ext_arg1 != NULL", __func__));
520 KASSERT(m->m_ext.ext_arg2 == NULL, ("%s: ext_arg2 != NULL", __func__));
521 KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__));
522 KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__));
523 KASSERT(*m->m_ext.ref_cnt == 1, ("%s: ref_cnt != 1", __func__));
524#ifdef INVARIANTS
525 trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg);
526#endif
527 /*
528 * If there are processes blocked on zone_clust, waiting for pages
529 * to be freed up, * cause them to be woken up by draining the
530 * packet zone. We are exposed to a race here * (in the check for
531 * the UMA_ZFLAG_FULL) where we might miss the flag set, but that
532 * is deliberate. We don't want to acquire the zone lock for every
533 * mbuf free.
534 */
535 if (uma_zone_exhausted_nolock(zone_clust))
536 zone_drain(zone_pack);
537}
538
539/*
540 * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor.
541 *
542 * Here the 'arg' pointer points to the Mbuf which we
543 * are configuring cluster storage for. If 'arg' is
544 * empty we allocate just the cluster without setting
545 * the mbuf to it. See mbuf.h.
546 */
547static int
548mb_ctor_clust(void *mem, int size, void *arg, int how)
549{
550 struct mbuf *m;
551 u_int *refcnt;
552 int type;
553 uma_zone_t zone;
554
555#ifdef INVARIANTS
556 trash_ctor(mem, size, arg, how);
557#endif
558 switch (size) {
559 case MCLBYTES:
560 type = EXT_CLUSTER;
561 zone = zone_clust;
562 break;
563#if MJUMPAGESIZE != MCLBYTES
564 case MJUMPAGESIZE:
565 type = EXT_JUMBOP;
566 zone = zone_jumbop;
567 break;
568#endif
569 case MJUM9BYTES:
570 type = EXT_JUMBO9;
571 zone = zone_jumbo9;
572 break;
573 case MJUM16BYTES:
574 type = EXT_JUMBO16;
575 zone = zone_jumbo16;
576 break;
577 default:
578 panic("unknown cluster size");
579 break;
580 }
581
582 m = (struct mbuf *)arg;
583 refcnt = uma_find_refcnt(zone, mem);
584 *refcnt = 1;
585 if (m != NULL) {
586 m->m_ext.ext_buf = (caddr_t)mem;
587 m->m_data = m->m_ext.ext_buf;
588 m->m_flags |= M_EXT;
589 m->m_ext.ext_free = NULL;
590 m->m_ext.ext_arg1 = NULL;
591 m->m_ext.ext_arg2 = NULL;
592 m->m_ext.ext_size = size;
593 m->m_ext.ext_type = type;
594 m->m_ext.ref_cnt = refcnt;
595 }
596
597 return (0);
598}
599
600/*
601 * The Mbuf Cluster zone destructor.
602 */
603static void
604mb_dtor_clust(void *mem, int size, void *arg)
605{
606#ifdef INVARIANTS
607 uma_zone_t zone;
608
609 zone = m_getzone(size);
610 KASSERT(*(uma_find_refcnt(zone, mem)) <= 1,
611 ("%s: refcnt incorrect %u", __func__,
612 *(uma_find_refcnt(zone, mem))) );
613
614 trash_dtor(mem, size, arg);
615#endif
616}
617
618/*
619 * The Packet secondary zone's init routine, executed on the
620 * object's transition from mbuf keg slab to zone cache.
621 */
622static int
623mb_zinit_pack(void *mem, int size, int how)
624{
625 struct mbuf *m;
626
627 m = (struct mbuf *)mem; /* m is virgin. */
628 if (uma_zalloc_arg(zone_clust, m, how) == NULL ||
629 m->m_ext.ext_buf == NULL)
630 return (ENOMEM);
631 m->m_ext.ext_type = EXT_PACKET; /* Override. */
632#ifdef INVARIANTS
633 trash_init(m->m_ext.ext_buf, MCLBYTES, how);
634#endif
635 return (0);
636}
637
638/*
639 * The Packet secondary zone's fini routine, executed on the
640 * object's transition from zone cache to keg slab.
641 */
642static void
643mb_zfini_pack(void *mem, int size)
644{
645 struct mbuf *m;
646
647 m = (struct mbuf *)mem;
648#ifdef INVARIANTS
649 trash_fini(m->m_ext.ext_buf, MCLBYTES);
650#endif
651 uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL);
652#ifdef INVARIANTS
653 trash_dtor(mem, size, NULL);
654#endif
655}
656
657/*
658 * The "packet" keg constructor.
659 */
660static int
661mb_ctor_pack(void *mem, int size, void *arg, int how)
662{
663 struct mbuf *m;
664 struct mb_args *args;
665#ifdef MAC
666 int error;
667#endif
668 int flags;
669 short type;
670
671 m = (struct mbuf *)mem;
672 args = (struct mb_args *)arg;
673 flags = args->flags;
674 type = args->type;
675
676#ifdef INVARIANTS
677 trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how);
678#endif
679 m->m_next = NULL;
680 m->m_nextpkt = NULL;
681 m->m_data = m->m_ext.ext_buf;
682 m->m_len = 0;
683 m->m_flags = (flags | M_EXT);
684 m->m_type = type;
685
686 if (flags & M_PKTHDR) {
687 m->m_pkthdr.rcvif = NULL;
688 m->m_pkthdr.len = 0;
689 m->m_pkthdr.header = NULL;
690 m->m_pkthdr.csum_flags = 0;
691 m->m_pkthdr.csum_data = 0;
692 m->m_pkthdr.tso_segsz = 0;
693 m->m_pkthdr.ether_vtag = 0;
694 m->m_pkthdr.flowid = 0;
695 SLIST_INIT(&m->m_pkthdr.tags);
696#ifdef MAC
697 /* If the label init fails, fail the alloc */
698 error = mac_mbuf_init(m, how);
699 if (error)
700 return (error);
701#endif
702 }
703 /* m_ext is already initialized. */
704
705 return (0);
706}
707
708int
709m_pkthdr_init(struct mbuf *m, int how)
710{
711#ifdef MAC
712 int error;
713#endif
714 m->m_data = m->m_pktdat;
715 SLIST_INIT(&m->m_pkthdr.tags);
716 m->m_pkthdr.rcvif = NULL;
717 m->m_pkthdr.header = NULL;
718 m->m_pkthdr.len = 0;
719 m->m_pkthdr.flowid = 0;
720 m->m_pkthdr.csum_flags = 0;
721 m->m_pkthdr.csum_data = 0;
722 m->m_pkthdr.tso_segsz = 0;
723 m->m_pkthdr.ether_vtag = 0;
724#ifdef MAC
725 /* If the label init fails, fail the alloc */
726 error = mac_mbuf_init(m, how);
727 if (error)
728 return (error);
729#endif
730
731 return (0);
732}
733
734/*
735 * This is the protocol drain routine.
736 *
737 * No locks should be held when this is called. The drain routines have to
738 * presently acquire some locks which raises the possibility of lock order
739 * reversal.
740 */
741static void
742mb_reclaim(void *junk)
743{
744 struct domain *dp;
745 struct protosw *pr;
746
747 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL,
748 "mb_reclaim()");
749
750 for (dp = domains; dp != NULL; dp = dp->dom_next)
751 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
752 if (pr->pr_drain != NULL)
753 (*pr->pr_drain)();
754}
| 247 EVENTHANDLER_INVOKE(nmbufs_change);
248 } else
249 error = EINVAL;
250 }
251 return (error);
252}
253SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbuf, CTLTYPE_INT|CTLFLAG_RW,
254&nmbufs, 0, sysctl_nmbufs, "IU",
255 "Maximum number of mbufs allowed");
256
257
258SYSCTL_STRUCT(_kern_ipc, OID_AUTO, mbstat, CTLFLAG_RD, &mbstat, mbstat,
259 "Mbuf general information and statistics");
260
261/*
262 * Zones from which we allocate.
263 */
264uma_zone_t zone_mbuf;
265uma_zone_t zone_clust;
266uma_zone_t zone_pack;
267uma_zone_t zone_jumbop;
268uma_zone_t zone_jumbo9;
269uma_zone_t zone_jumbo16;
270uma_zone_t zone_ext_refcnt;
271
272/*
273 * Local prototypes.
274 */
275static int mb_ctor_mbuf(void *, int, void *, int);
276static int mb_ctor_clust(void *, int, void *, int);
277static int mb_ctor_pack(void *, int, void *, int);
278static void mb_dtor_mbuf(void *, int, void *);
279static void mb_dtor_clust(void *, int, void *);
280static void mb_dtor_pack(void *, int, void *);
281static int mb_zinit_pack(void *, int, int);
282static void mb_zfini_pack(void *, int);
283
284static void mb_reclaim(void *);
285static void mbuf_init(void *);
286static void *mbuf_jumbo_alloc(uma_zone_t, int, uint8_t *, int);
287
288/* Ensure that MSIZE must be a power of 2. */
289CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
290
291/*
292 * Initialize FreeBSD Network buffer allocation.
293 */
294SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL);
295static void
296mbuf_init(void *dummy)
297{
298
299 /*
300 * Configure UMA zones for Mbufs, Clusters, and Packets.
301 */
302 zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE,
303 mb_ctor_mbuf, mb_dtor_mbuf,
304#ifdef INVARIANTS
305 trash_init, trash_fini,
306#else
307 NULL, NULL,
308#endif
309 MSIZE - 1, UMA_ZONE_MAXBUCKET);
310 if (nmbufs > 0) {
311 uma_zone_set_max(zone_mbuf, nmbufs);
312 nmbufs = uma_zone_get_max(zone_mbuf);
313 }
314
315 zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
316 mb_ctor_clust, mb_dtor_clust,
317#ifdef INVARIANTS
318 trash_init, trash_fini,
319#else
320 NULL, NULL,
321#endif
322 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
323 if (nmbclusters > 0) {
324 uma_zone_set_max(zone_clust, nmbclusters);
325 nmbclusters = uma_zone_get_max(zone_clust);
326 }
327
328 zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
329 mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);
330
331 /* Make jumbo frame zone too. Page size, 9k and 16k. */
332 zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE,
333 mb_ctor_clust, mb_dtor_clust,
334#ifdef INVARIANTS
335 trash_init, trash_fini,
336#else
337 NULL, NULL,
338#endif
339 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
340 if (nmbjumbop > 0) {
341 uma_zone_set_max(zone_jumbop, nmbjumbop);
342 nmbjumbop = uma_zone_get_max(zone_jumbop);
343 }
344
345 zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
346 mb_ctor_clust, mb_dtor_clust,
347#ifdef INVARIANTS
348 trash_init, trash_fini,
349#else
350 NULL, NULL,
351#endif
352 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
353 uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc);
354 if (nmbjumbo9 > 0) {
355 uma_zone_set_max(zone_jumbo9, nmbjumbo9);
356 nmbjumbo9 = uma_zone_get_max(zone_jumbo9);
357 }
358
359 zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,
360 mb_ctor_clust, mb_dtor_clust,
361#ifdef INVARIANTS
362 trash_init, trash_fini,
363#else
364 NULL, NULL,
365#endif
366 UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
367 uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc);
368 if (nmbjumbo16 > 0) {
369 uma_zone_set_max(zone_jumbo16, nmbjumbo16);
370 nmbjumbo16 = uma_zone_get_max(zone_jumbo16);
371 }
372
373 zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int),
374 NULL, NULL,
375 NULL, NULL,
376 UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
377
378 /* uma_prealloc() goes here... */
379
380 /*
381 * Hook event handler for low-memory situation, used to
382 * drain protocols and push data back to the caches (UMA
383 * later pushes it back to VM).
384 */
385 EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,
386 EVENTHANDLER_PRI_FIRST);
387
388 /*
389 * [Re]set counters and local statistics knobs.
390 * XXX Some of these should go and be replaced, but UMA stat
391 * gathering needs to be revised.
392 */
393 mbstat.m_mbufs = 0;
394 mbstat.m_mclusts = 0;
395 mbstat.m_drain = 0;
396 mbstat.m_msize = MSIZE;
397 mbstat.m_mclbytes = MCLBYTES;
398 mbstat.m_minclsize = MINCLSIZE;
399 mbstat.m_mlen = MLEN;
400 mbstat.m_mhlen = MHLEN;
401 mbstat.m_numtypes = MT_NTYPES;
402
403 mbstat.m_mcfail = mbstat.m_mpfail = 0;
404 mbstat.sf_iocnt = 0;
405 mbstat.sf_allocwait = mbstat.sf_allocfail = 0;
406}
407
408/*
409 * UMA backend page allocator for the jumbo frame zones.
410 *
411 * Allocates kernel virtual memory that is backed by contiguous physical
412 * pages.
413 */
414static void *
415mbuf_jumbo_alloc(uma_zone_t zone, int bytes, uint8_t *flags, int wait)
416{
417
418 /* Inform UMA that this allocator uses kernel_map/object. */
419 *flags = UMA_SLAB_KERNEL;
420 return ((void *)kmem_alloc_contig(kernel_map, bytes, wait,
421 (vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0, VM_MEMATTR_DEFAULT));
422}
423
424/*
425 * Constructor for Mbuf master zone.
426 *
427 * The 'arg' pointer points to a mb_args structure which
428 * contains call-specific information required to support the
429 * mbuf allocation API. See mbuf.h.
430 */
431static int
432mb_ctor_mbuf(void *mem, int size, void *arg, int how)
433{
434 struct mbuf *m;
435 struct mb_args *args;
436#ifdef MAC
437 int error;
438#endif
439 int flags;
440 short type;
441
442#ifdef INVARIANTS
443 trash_ctor(mem, size, arg, how);
444#endif
445 m = (struct mbuf *)mem;
446 args = (struct mb_args *)arg;
447 flags = args->flags;
448 type = args->type;
449
450 /*
451 * The mbuf is initialized later. The caller has the
452 * responsibility to set up any MAC labels too.
453 */
454 if (type == MT_NOINIT)
455 return (0);
456
457 m->m_next = NULL;
458 m->m_nextpkt = NULL;
459 m->m_len = 0;
460 m->m_flags = flags;
461 m->m_type = type;
462 if (flags & M_PKTHDR) {
463 m->m_data = m->m_pktdat;
464 m->m_pkthdr.rcvif = NULL;
465 m->m_pkthdr.header = NULL;
466 m->m_pkthdr.len = 0;
467 m->m_pkthdr.csum_flags = 0;
468 m->m_pkthdr.csum_data = 0;
469 m->m_pkthdr.tso_segsz = 0;
470 m->m_pkthdr.ether_vtag = 0;
471 m->m_pkthdr.flowid = 0;
472 SLIST_INIT(&m->m_pkthdr.tags);
473#ifdef MAC
474 /* If the label init fails, fail the alloc */
475 error = mac_mbuf_init(m, how);
476 if (error)
477 return (error);
478#endif
479 } else
480 m->m_data = m->m_dat;
481 return (0);
482}
483
484/*
485 * The Mbuf master zone destructor.
486 */
487static void
488mb_dtor_mbuf(void *mem, int size, void *arg)
489{
490 struct mbuf *m;
491 unsigned long flags;
492
493 m = (struct mbuf *)mem;
494 flags = (unsigned long)arg;
495
496 if ((flags & MB_NOTAGS) == 0 && (m->m_flags & M_PKTHDR) != 0)
497 m_tag_delete_chain(m, NULL);
498 KASSERT((m->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__));
499 KASSERT((m->m_flags & M_NOFREE) == 0, ("%s: M_NOFREE set", __func__));
500#ifdef INVARIANTS
501 trash_dtor(mem, size, arg);
502#endif
503}
504
505/*
506 * The Mbuf Packet zone destructor.
507 */
508static void
509mb_dtor_pack(void *mem, int size, void *arg)
510{
511 struct mbuf *m;
512
513 m = (struct mbuf *)mem;
514 if ((m->m_flags & M_PKTHDR) != 0)
515 m_tag_delete_chain(m, NULL);
516
517 /* Make sure we've got a clean cluster back. */
518 KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
519 KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__));
520 KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__));
521 KASSERT(m->m_ext.ext_arg1 == NULL, ("%s: ext_arg1 != NULL", __func__));
522 KASSERT(m->m_ext.ext_arg2 == NULL, ("%s: ext_arg2 != NULL", __func__));
523 KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__));
524 KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__));
525 KASSERT(*m->m_ext.ref_cnt == 1, ("%s: ref_cnt != 1", __func__));
526#ifdef INVARIANTS
527 trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg);
528#endif
529 /*
530 * If there are processes blocked on zone_clust, waiting for pages
531 * to be freed up, * cause them to be woken up by draining the
532 * packet zone. We are exposed to a race here * (in the check for
533 * the UMA_ZFLAG_FULL) where we might miss the flag set, but that
534 * is deliberate. We don't want to acquire the zone lock for every
535 * mbuf free.
536 */
537 if (uma_zone_exhausted_nolock(zone_clust))
538 zone_drain(zone_pack);
539}
540
541/*
542 * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor.
543 *
544 * Here the 'arg' pointer points to the Mbuf which we
545 * are configuring cluster storage for. If 'arg' is
546 * empty we allocate just the cluster without setting
547 * the mbuf to it. See mbuf.h.
548 */
549static int
550mb_ctor_clust(void *mem, int size, void *arg, int how)
551{
552 struct mbuf *m;
553 u_int *refcnt;
554 int type;
555 uma_zone_t zone;
556
557#ifdef INVARIANTS
558 trash_ctor(mem, size, arg, how);
559#endif
560 switch (size) {
561 case MCLBYTES:
562 type = EXT_CLUSTER;
563 zone = zone_clust;
564 break;
565#if MJUMPAGESIZE != MCLBYTES
566 case MJUMPAGESIZE:
567 type = EXT_JUMBOP;
568 zone = zone_jumbop;
569 break;
570#endif
571 case MJUM9BYTES:
572 type = EXT_JUMBO9;
573 zone = zone_jumbo9;
574 break;
575 case MJUM16BYTES:
576 type = EXT_JUMBO16;
577 zone = zone_jumbo16;
578 break;
579 default:
580 panic("unknown cluster size");
581 break;
582 }
583
584 m = (struct mbuf *)arg;
585 refcnt = uma_find_refcnt(zone, mem);
586 *refcnt = 1;
587 if (m != NULL) {
588 m->m_ext.ext_buf = (caddr_t)mem;
589 m->m_data = m->m_ext.ext_buf;
590 m->m_flags |= M_EXT;
591 m->m_ext.ext_free = NULL;
592 m->m_ext.ext_arg1 = NULL;
593 m->m_ext.ext_arg2 = NULL;
594 m->m_ext.ext_size = size;
595 m->m_ext.ext_type = type;
596 m->m_ext.ref_cnt = refcnt;
597 }
598
599 return (0);
600}
601
602/*
603 * The Mbuf Cluster zone destructor.
604 */
605static void
606mb_dtor_clust(void *mem, int size, void *arg)
607{
608#ifdef INVARIANTS
609 uma_zone_t zone;
610
611 zone = m_getzone(size);
612 KASSERT(*(uma_find_refcnt(zone, mem)) <= 1,
613 ("%s: refcnt incorrect %u", __func__,
614 *(uma_find_refcnt(zone, mem))) );
615
616 trash_dtor(mem, size, arg);
617#endif
618}
619
620/*
621 * The Packet secondary zone's init routine, executed on the
622 * object's transition from mbuf keg slab to zone cache.
623 */
624static int
625mb_zinit_pack(void *mem, int size, int how)
626{
627 struct mbuf *m;
628
629 m = (struct mbuf *)mem; /* m is virgin. */
630 if (uma_zalloc_arg(zone_clust, m, how) == NULL ||
631 m->m_ext.ext_buf == NULL)
632 return (ENOMEM);
633 m->m_ext.ext_type = EXT_PACKET; /* Override. */
634#ifdef INVARIANTS
635 trash_init(m->m_ext.ext_buf, MCLBYTES, how);
636#endif
637 return (0);
638}
639
640/*
641 * The Packet secondary zone's fini routine, executed on the
642 * object's transition from zone cache to keg slab.
643 */
644static void
645mb_zfini_pack(void *mem, int size)
646{
647 struct mbuf *m;
648
649 m = (struct mbuf *)mem;
650#ifdef INVARIANTS
651 trash_fini(m->m_ext.ext_buf, MCLBYTES);
652#endif
653 uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL);
654#ifdef INVARIANTS
655 trash_dtor(mem, size, NULL);
656#endif
657}
658
659/*
660 * The "packet" keg constructor.
661 */
662static int
663mb_ctor_pack(void *mem, int size, void *arg, int how)
664{
665 struct mbuf *m;
666 struct mb_args *args;
667#ifdef MAC
668 int error;
669#endif
670 int flags;
671 short type;
672
673 m = (struct mbuf *)mem;
674 args = (struct mb_args *)arg;
675 flags = args->flags;
676 type = args->type;
677
678#ifdef INVARIANTS
679 trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how);
680#endif
681 m->m_next = NULL;
682 m->m_nextpkt = NULL;
683 m->m_data = m->m_ext.ext_buf;
684 m->m_len = 0;
685 m->m_flags = (flags | M_EXT);
686 m->m_type = type;
687
688 if (flags & M_PKTHDR) {
689 m->m_pkthdr.rcvif = NULL;
690 m->m_pkthdr.len = 0;
691 m->m_pkthdr.header = NULL;
692 m->m_pkthdr.csum_flags = 0;
693 m->m_pkthdr.csum_data = 0;
694 m->m_pkthdr.tso_segsz = 0;
695 m->m_pkthdr.ether_vtag = 0;
696 m->m_pkthdr.flowid = 0;
697 SLIST_INIT(&m->m_pkthdr.tags);
698#ifdef MAC
699 /* If the label init fails, fail the alloc */
700 error = mac_mbuf_init(m, how);
701 if (error)
702 return (error);
703#endif
704 }
705 /* m_ext is already initialized. */
706
707 return (0);
708}
709
710int
711m_pkthdr_init(struct mbuf *m, int how)
712{
713#ifdef MAC
714 int error;
715#endif
716 m->m_data = m->m_pktdat;
717 SLIST_INIT(&m->m_pkthdr.tags);
718 m->m_pkthdr.rcvif = NULL;
719 m->m_pkthdr.header = NULL;
720 m->m_pkthdr.len = 0;
721 m->m_pkthdr.flowid = 0;
722 m->m_pkthdr.csum_flags = 0;
723 m->m_pkthdr.csum_data = 0;
724 m->m_pkthdr.tso_segsz = 0;
725 m->m_pkthdr.ether_vtag = 0;
726#ifdef MAC
727 /* If the label init fails, fail the alloc */
728 error = mac_mbuf_init(m, how);
729 if (error)
730 return (error);
731#endif
732
733 return (0);
734}
735
736/*
737 * This is the protocol drain routine.
738 *
739 * No locks should be held when this is called. The drain routines have to
740 * presently acquire some locks which raises the possibility of lock order
741 * reversal.
742 */
743static void
744mb_reclaim(void *junk)
745{
746 struct domain *dp;
747 struct protosw *pr;
748
749 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL,
750 "mb_reclaim()");
751
752 for (dp = domains; dp != NULL; dp = dp->dom_next)
753 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
754 if (pr->pr_drain != NULL)
755 (*pr->pr_drain)();
756}
|