kern_mbuf.c revision 152035
1129906Sbmilekic/*- 2141991Sbmilekic * Copyright (c) 2004, 2005, 3141991Sbmilekic * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved. 4129906Sbmilekic * 5129906Sbmilekic * Redistribution and use in source and binary forms, with or without 6129906Sbmilekic * modification, are permitted provided that the following conditions 7129906Sbmilekic * are met: 8129906Sbmilekic * 1. Redistributions of source code must retain the above copyright 9129906Sbmilekic * notice unmodified, this list of conditions and the following 10129906Sbmilekic * disclaimer. 11129906Sbmilekic * 2. Redistributions in binary form must reproduce the above copyright 12129906Sbmilekic * notice, this list of conditions and the following disclaimer in the 13129906Sbmilekic * documentation and/or other materials provided with the distribution. 14129906Sbmilekic * 15129906Sbmilekic * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16129906Sbmilekic * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17129906Sbmilekic * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18129906Sbmilekic * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19129906Sbmilekic * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20129906Sbmilekic * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21129906Sbmilekic * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22129906Sbmilekic * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23129906Sbmilekic * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24129906Sbmilekic * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25129906Sbmilekic * SUCH DAMAGE. 26129906Sbmilekic */ 27129906Sbmilekic 28129906Sbmilekic#include <sys/cdefs.h> 29129906Sbmilekic__FBSDID("$FreeBSD: head/sys/kern/kern_mbuf.c 152035 2005-11-04 17:20:53Z andre $"); 30129906Sbmilekic 31129906Sbmilekic#include "opt_mac.h" 32129906Sbmilekic#include "opt_param.h" 33129906Sbmilekic 34129906Sbmilekic#include <sys/param.h> 35129906Sbmilekic#include <sys/mac.h> 36129906Sbmilekic#include <sys/malloc.h> 37129906Sbmilekic#include <sys/systm.h> 38129906Sbmilekic#include <sys/mbuf.h> 39129906Sbmilekic#include <sys/domain.h> 40129906Sbmilekic#include <sys/eventhandler.h> 41129906Sbmilekic#include <sys/kernel.h> 42129906Sbmilekic#include <sys/protosw.h> 43129906Sbmilekic#include <sys/smp.h> 44129906Sbmilekic#include <sys/sysctl.h> 45129906Sbmilekic 46129906Sbmilekic#include <vm/vm.h> 47129906Sbmilekic#include <vm/vm_page.h> 48129906Sbmilekic#include <vm/uma.h> 49147537Ssilby#include <vm/uma_int.h> 50147537Ssilby#include <vm/uma_dbg.h> 51129906Sbmilekic 52129906Sbmilekic/* 53129906Sbmilekic * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA 54129906Sbmilekic * Zones. 55129906Sbmilekic * 56129906Sbmilekic * Mbuf Clusters (2K, contiguous) are allocated from the Cluster 57129906Sbmilekic * Zone. The Zone can be capped at kern.ipc.nmbclusters, if the 58129906Sbmilekic * administrator so desires. 59129906Sbmilekic * 60129906Sbmilekic * Mbufs are allocated from a UMA Master Zone called the Mbuf 61129906Sbmilekic * Zone. 62129906Sbmilekic * 63129906Sbmilekic * Additionally, FreeBSD provides a Packet Zone, which it 64129906Sbmilekic * configures as a Secondary Zone to the Mbuf Master Zone, 65129906Sbmilekic * thus sharing backend Slab kegs with the Mbuf Master Zone. 66129906Sbmilekic * 67129906Sbmilekic * Thus common-case allocations and locking are simplified: 68129906Sbmilekic * 69129906Sbmilekic * m_clget() m_getcl() 70129906Sbmilekic * | | 71129906Sbmilekic * | .------------>[(Packet Cache)] m_get(), m_gethdr() 72129906Sbmilekic * | | [ Packet ] | 73129906Sbmilekic * [(Cluster Cache)] [ Secondary ] [ (Mbuf Cache) ] 74129906Sbmilekic * [ Cluster Zone ] [ Zone ] [ Mbuf Master Zone ] 75129906Sbmilekic * | \________ | 76129906Sbmilekic * [ Cluster Keg ] \ / 77129906Sbmilekic * | [ Mbuf Keg ] 78129906Sbmilekic * [ Cluster Slabs ] | 79129906Sbmilekic * | [ Mbuf Slabs ] 80129906Sbmilekic * \____________(VM)_________________/ 81151976Sandre * 82151976Sandre * 83151976Sandre * Whenever a object is allocated with uma_zalloc() out of the 84151976Sandre * one of the Zones its _ctor_ function is executed. The same 85151976Sandre * for any deallocation through uma_zfree() the _dror_ function 86151976Sandre * is executed. 87151976Sandre * 88151976Sandre * Caches are per-CPU and are filled from the Master Zone. 89151976Sandre * 90151976Sandre * Whenever a object is allocated from the underlying global 91151976Sandre * memory pool it gets pre-initialized with the _zinit_ functions. 92151976Sandre * When the Keg's are overfull objects get decomissioned with 93151976Sandre * _zfini_ functions and free'd back to the global memory pool. 94151976Sandre * 95129906Sbmilekic */ 96129906Sbmilekic 97151976Sandreint nmbclusters; /* limits number of mbuf clusters */ 98151976Sandreint nmbjumbo9; /* limits number of 9k jumbo clusters */ 99151976Sandreint nmbjumbo16; /* limits number of 16k jumbo clusters */ 100129906Sbmilekicstruct mbstat mbstat; 101129906Sbmilekic 102129906Sbmilekicstatic void 103129906Sbmilekictunable_mbinit(void *dummy) 104129906Sbmilekic{ 105129906Sbmilekic 106129906Sbmilekic /* This has to be done before VM init. */ 107129906Sbmilekic nmbclusters = 1024 + maxusers * 64; 108129906Sbmilekic TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters); 109129906Sbmilekic} 110129906SbmilekicSYSINIT(tunable_mbinit, SI_SUB_TUNABLES, SI_ORDER_ANY, tunable_mbinit, NULL); 111129906Sbmilekic 112129906SbmilekicSYSCTL_DECL(_kern_ipc); 113151976Sandre/* XXX: These should be tuneables. Can't change UMA limits on the fly. */ 114129906SbmilekicSYSCTL_INT(_kern_ipc, OID_AUTO, nmbclusters, CTLFLAG_RW, &nmbclusters, 0, 115129906Sbmilekic "Maximum number of mbuf clusters allowed"); 116151976SandreSYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbo9, CTLFLAG_RW, &nmbjumbo9, 0, 117151976Sandre "Maximum number of mbuf 9k jumbo clusters allowed"); 118151976SandreSYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbo16, CTLFLAG_RW, &nmbjumbo16, 0, 119151976Sandre "Maximum number of mbuf 16k jumbo clusters allowed"); 120129906SbmilekicSYSCTL_STRUCT(_kern_ipc, OID_AUTO, mbstat, CTLFLAG_RD, &mbstat, mbstat, 121129906Sbmilekic "Mbuf general information and statistics"); 122129906Sbmilekic 123129906Sbmilekic/* 124129906Sbmilekic * Zones from which we allocate. 125129906Sbmilekic */ 126129906Sbmilekicuma_zone_t zone_mbuf; 127129906Sbmilekicuma_zone_t zone_clust; 128129906Sbmilekicuma_zone_t zone_pack; 129151976Sandreuma_zone_t zone_jumbo9; 130151976Sandreuma_zone_t zone_jumbo16; 131151976Sandreuma_zone_t zone_ext_refcnt; 132129906Sbmilekic 133129906Sbmilekic/* 134129906Sbmilekic * Local prototypes. 135129906Sbmilekic */ 136132987Sgreenstatic int mb_ctor_mbuf(void *, int, void *, int); 137132987Sgreenstatic int mb_ctor_clust(void *, int, void *, int); 138132987Sgreenstatic int mb_ctor_pack(void *, int, void *, int); 139129906Sbmilekicstatic void mb_dtor_mbuf(void *, int, void *); 140151976Sandrestatic void mb_dtor_clust(void *, int, void *); 141151976Sandrestatic void mb_dtor_pack(void *, int, void *); 142151976Sandrestatic int mb_zinit_pack(void *, int, int); 143151976Sandrestatic void mb_zfini_pack(void *, int); 144129906Sbmilekic 145129906Sbmilekicstatic void mb_reclaim(void *); 146129906Sbmilekicstatic void mbuf_init(void *); 147129906Sbmilekic 148135510Sbrian/* Ensure that MSIZE doesn't break dtom() - it must be a power of 2 */ 149135510SbrianCTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE); 150135510Sbrian 151129906Sbmilekic/* 152129906Sbmilekic * Initialize FreeBSD Network buffer allocation. 153129906Sbmilekic */ 154129906SbmilekicSYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL) 155129906Sbmilekicstatic void 156129906Sbmilekicmbuf_init(void *dummy) 157129906Sbmilekic{ 158129906Sbmilekic 159129906Sbmilekic /* 160129906Sbmilekic * Configure UMA zones for Mbufs, Clusters, and Packets. 161129906Sbmilekic */ 162151976Sandre zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE, 163151976Sandre mb_ctor_mbuf, mb_dtor_mbuf, 164147537Ssilby#ifdef INVARIANTS 165151976Sandre trash_init, trash_fini, 166147537Ssilby#else 167151976Sandre NULL, NULL, 168147537Ssilby#endif 169151976Sandre MSIZE - 1, UMA_ZONE_MAXBUCKET); 170151976Sandre 171148095Srwatson zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES, 172151976Sandre mb_ctor_clust, mb_dtor_clust, 173147537Ssilby#ifdef INVARIANTS 174151976Sandre trash_init, trash_fini, 175147537Ssilby#else 176151976Sandre NULL, NULL, 177147537Ssilby#endif 178151976Sandre UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 179129906Sbmilekic if (nmbclusters > 0) 180129906Sbmilekic uma_zone_set_max(zone_clust, nmbclusters); 181151976Sandre 182148095Srwatson zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack, 183151976Sandre mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf); 184129906Sbmilekic 185151976Sandre /* Make jumbo frame zone too. 9k and 16k. */ 186151976Sandre zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES, 187151976Sandre mb_ctor_clust, mb_dtor_clust, 188151976Sandre#ifdef INVARIANTS 189151976Sandre trash_init, trash_fini, 190151976Sandre#else 191151976Sandre NULL, NULL, 192151976Sandre#endif 193151976Sandre UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 194151976Sandre if (nmbjumbo9 > 0) 195151976Sandre uma_zone_set_max(zone_jumbo9, nmbjumbo9); 196129906Sbmilekic 197151976Sandre zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES, 198151976Sandre mb_ctor_clust, mb_dtor_clust, 199151976Sandre#ifdef INVARIANTS 200151976Sandre trash_init, trash_fini, 201151976Sandre#else 202151976Sandre NULL, NULL, 203151976Sandre#endif 204151976Sandre UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 205151976Sandre if (nmbjumbo16 > 0) 206151976Sandre uma_zone_set_max(zone_jumbo16, nmbjumbo16); 207151976Sandre 208151976Sandre zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int), 209151976Sandre NULL, NULL, 210151976Sandre NULL, NULL, 211151976Sandre UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 212151976Sandre 213151976Sandre /* uma_prealloc() goes here... */ 214151976Sandre 215129906Sbmilekic /* 216129906Sbmilekic * Hook event handler for low-memory situation, used to 217129906Sbmilekic * drain protocols and push data back to the caches (UMA 218129906Sbmilekic * later pushes it back to VM). 219129906Sbmilekic */ 220129906Sbmilekic EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL, 221129906Sbmilekic EVENTHANDLER_PRI_FIRST); 222129906Sbmilekic 223129906Sbmilekic /* 224129906Sbmilekic * [Re]set counters and local statistics knobs. 225129906Sbmilekic * XXX Some of these should go and be replaced, but UMA stat 226129906Sbmilekic * gathering needs to be revised. 227129906Sbmilekic */ 228129906Sbmilekic mbstat.m_mbufs = 0; 229129906Sbmilekic mbstat.m_mclusts = 0; 230129906Sbmilekic mbstat.m_drain = 0; 231129906Sbmilekic mbstat.m_msize = MSIZE; 232129906Sbmilekic mbstat.m_mclbytes = MCLBYTES; 233129906Sbmilekic mbstat.m_minclsize = MINCLSIZE; 234129906Sbmilekic mbstat.m_mlen = MLEN; 235129906Sbmilekic mbstat.m_mhlen = MHLEN; 236129906Sbmilekic mbstat.m_numtypes = MT_NTYPES; 237129906Sbmilekic 238129906Sbmilekic mbstat.m_mcfail = mbstat.m_mpfail = 0; 239129906Sbmilekic mbstat.sf_iocnt = 0; 240129906Sbmilekic mbstat.sf_allocwait = mbstat.sf_allocfail = 0; 241129906Sbmilekic} 242129906Sbmilekic 243129906Sbmilekic/* 244129906Sbmilekic * Constructor for Mbuf master zone. 245129906Sbmilekic * 246129906Sbmilekic * The 'arg' pointer points to a mb_args structure which 247129906Sbmilekic * contains call-specific information required to support the 248151976Sandre * mbuf allocation API. See mbuf.h. 249129906Sbmilekic */ 250132987Sgreenstatic int 251132987Sgreenmb_ctor_mbuf(void *mem, int size, void *arg, int how) 252129906Sbmilekic{ 253129906Sbmilekic struct mbuf *m; 254129906Sbmilekic struct mb_args *args; 255132987Sgreen#ifdef MAC 256132987Sgreen int error; 257132987Sgreen#endif 258129906Sbmilekic int flags; 259129906Sbmilekic short type; 260129906Sbmilekic 261147537Ssilby#ifdef INVARIANTS 262147537Ssilby trash_ctor(mem, size, arg, how); 263147537Ssilby#endif 264129906Sbmilekic m = (struct mbuf *)mem; 265129906Sbmilekic args = (struct mb_args *)arg; 266129906Sbmilekic flags = args->flags; 267129906Sbmilekic type = args->type; 268129906Sbmilekic 269151976Sandre /* 270151976Sandre * The mbuf is initialized later. The caller has the 271151976Sandre * responseability to setup any MAC labels too. 272151976Sandre */ 273151976Sandre if (type == MT_NOINIT) 274151976Sandre return (0); 275151976Sandre 276129906Sbmilekic m->m_next = NULL; 277129906Sbmilekic m->m_nextpkt = NULL; 278151976Sandre m->m_len = 0; 279129947Sbmilekic m->m_flags = flags; 280151976Sandre m->m_type = type; 281129906Sbmilekic if (flags & M_PKTHDR) { 282129906Sbmilekic m->m_data = m->m_pktdat; 283129906Sbmilekic m->m_pkthdr.rcvif = NULL; 284151976Sandre m->m_pkthdr.len = 0; 285151976Sandre m->m_pkthdr.header = NULL; 286129906Sbmilekic m->m_pkthdr.csum_flags = 0; 287151976Sandre m->m_pkthdr.csum_data = 0; 288129906Sbmilekic SLIST_INIT(&m->m_pkthdr.tags); 289129906Sbmilekic#ifdef MAC 290129906Sbmilekic /* If the label init fails, fail the alloc */ 291132987Sgreen error = mac_init_mbuf(m, how); 292132987Sgreen if (error) 293132987Sgreen return (error); 294129906Sbmilekic#endif 295129947Sbmilekic } else 296129906Sbmilekic m->m_data = m->m_dat; 297132987Sgreen return (0); 298129906Sbmilekic} 299129906Sbmilekic 300129906Sbmilekic/* 301151976Sandre * The Mbuf master zone destructor. 302129906Sbmilekic */ 303129906Sbmilekicstatic void 304129906Sbmilekicmb_dtor_mbuf(void *mem, int size, void *arg) 305129906Sbmilekic{ 306129906Sbmilekic struct mbuf *m; 307129906Sbmilekic 308129906Sbmilekic m = (struct mbuf *)mem; 309129906Sbmilekic if ((m->m_flags & M_PKTHDR) != 0) 310129906Sbmilekic m_tag_delete_chain(m, NULL); 311151976Sandre KASSERT((m->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__)); 312147537Ssilby#ifdef INVARIANTS 313147537Ssilby trash_dtor(mem, size, arg); 314147537Ssilby#endif 315129906Sbmilekic} 316129906Sbmilekic 317151976Sandre/* 318151976Sandre * The Mbuf Packet zone destructor. 319151976Sandre */ 320129906Sbmilekicstatic void 321129906Sbmilekicmb_dtor_pack(void *mem, int size, void *arg) 322129906Sbmilekic{ 323129906Sbmilekic struct mbuf *m; 324129906Sbmilekic 325129906Sbmilekic m = (struct mbuf *)mem; 326129906Sbmilekic if ((m->m_flags & M_PKTHDR) != 0) 327129906Sbmilekic m_tag_delete_chain(m, NULL); 328151976Sandre 329151976Sandre /* Make sure we've got a clean cluster back. */ 330151976Sandre KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__)); 331151976Sandre KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__)); 332151976Sandre KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__)); 333151976Sandre KASSERT(m->m_ext.ext_args == NULL, ("%s: ext_args != NULL", __func__)); 334151976Sandre KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__)); 335151976Sandre KASSERT(m->m_ext.ext_type == EXT_CLUSTER, ("%s: ext_type != EXT_CLUSTER", __func__)); 336151976Sandre KASSERT(*m->m_ext.ref_cnt == 1, ("%s: ref_cnt != 1", __func__)); 337147537Ssilby#ifdef INVARIANTS 338147537Ssilby trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg); 339147537Ssilby#endif 340129906Sbmilekic} 341129906Sbmilekic 342129906Sbmilekic/* 343151976Sandre * The Cluster and Jumbo[9|16] zone constructor. 344129906Sbmilekic * 345129906Sbmilekic * Here the 'arg' pointer points to the Mbuf which we 346151976Sandre * are configuring cluster storage for. If 'arg' is 347151976Sandre * empty we allocate just the cluster without setting 348151976Sandre * the mbuf to it. See mbuf.h. 349129906Sbmilekic */ 350132987Sgreenstatic int 351132987Sgreenmb_ctor_clust(void *mem, int size, void *arg, int how) 352129906Sbmilekic{ 353129906Sbmilekic struct mbuf *m; 354151976Sandre u_int *refcnt; 355151976Sandre int type = 0; 356129906Sbmilekic 357147537Ssilby#ifdef INVARIANTS 358147537Ssilby trash_ctor(mem, size, arg, how); 359147537Ssilby#endif 360129906Sbmilekic m = (struct mbuf *)arg; 361151976Sandre if (m != NULL) { 362151976Sandre switch (size) { 363151976Sandre case MCLBYTES: 364151976Sandre type = EXT_CLUSTER; 365151976Sandre break; 366151976Sandre case MJUM9BYTES: 367151976Sandre type = EXT_JUMBO9; 368151976Sandre break; 369151976Sandre case MJUM16BYTES: 370151976Sandre type = EXT_JUMBO16; 371151976Sandre break; 372151976Sandre default: 373151976Sandre panic("unknown cluster size"); 374151976Sandre break; 375151976Sandre } 376151976Sandre m->m_ext.ext_buf = (caddr_t)mem; 377151976Sandre m->m_data = m->m_ext.ext_buf; 378151976Sandre m->m_flags |= M_EXT; 379151976Sandre m->m_ext.ext_free = NULL; 380151976Sandre m->m_ext.ext_args = NULL; 381151976Sandre m->m_ext.ext_size = size; 382151976Sandre m->m_ext.ext_type = type; 383151976Sandre m->m_ext.ref_cnt = uma_find_refcnt(zone_clust, mem); 384151976Sandre *m->m_ext.ref_cnt = 1; 385151976Sandre } else { 386151976Sandre refcnt = uma_find_refcnt(zone_clust, mem); 387151976Sandre *refcnt = 1; 388151976Sandre } 389132987Sgreen return (0); 390129906Sbmilekic} 391129906Sbmilekic 392151976Sandre/* 393151976Sandre * The Mbuf Cluster zone destructor. 394151976Sandre */ 395129906Sbmilekicstatic void 396129906Sbmilekicmb_dtor_clust(void *mem, int size, void *arg) 397129906Sbmilekic{ 398151976Sandre 399152035Sandre KASSERT(*(uma_find_refcnt(zone_clust, mem)) <= 1, 400152035Sandre ("%s: refcnt incorrect %u", __func__, 401152035Sandre *(uma_find_refcnt(zone_clust, mem))) ); 402147537Ssilby#ifdef INVARIANTS 403147537Ssilby trash_dtor(mem, size, arg); 404147537Ssilby#endif 405129906Sbmilekic} 406129906Sbmilekic 407129906Sbmilekic/* 408129906Sbmilekic * The Packet secondary zone's init routine, executed on the 409151976Sandre * object's transition from mbuf keg slab to zone cache. 410129906Sbmilekic */ 411132987Sgreenstatic int 412151976Sandremb_zinit_pack(void *mem, int size, int how) 413129906Sbmilekic{ 414129906Sbmilekic struct mbuf *m; 415129906Sbmilekic 416151976Sandre m = (struct mbuf *)mem; /* m is virgin. */ 417132987Sgreen uma_zalloc_arg(zone_clust, m, how); 418132987Sgreen if (m->m_ext.ext_buf == NULL) 419132987Sgreen return (ENOMEM); 420147537Ssilby#ifdef INVARIANTS 421147537Ssilby trash_init(m->m_ext.ext_buf, MCLBYTES, how); 422147537Ssilby#endif 423132987Sgreen return (0); 424129906Sbmilekic} 425129906Sbmilekic 426129906Sbmilekic/* 427129906Sbmilekic * The Packet secondary zone's fini routine, executed on the 428129906Sbmilekic * object's transition from zone cache to keg slab. 429129906Sbmilekic */ 430129906Sbmilekicstatic void 431151976Sandremb_zfini_pack(void *mem, int size) 432129906Sbmilekic{ 433129906Sbmilekic struct mbuf *m; 434129906Sbmilekic 435129906Sbmilekic m = (struct mbuf *)mem; 436147537Ssilby#ifdef INVARIANTS 437147537Ssilby trash_fini(m->m_ext.ext_buf, MCLBYTES); 438147537Ssilby#endif 439129906Sbmilekic uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL); 440147652Ssilby#ifdef INVARIANTS 441147652Ssilby trash_dtor(mem, size, NULL); 442147652Ssilby#endif 443129906Sbmilekic} 444129906Sbmilekic 445129906Sbmilekic/* 446129906Sbmilekic * The "packet" keg constructor. 447129906Sbmilekic */ 448132987Sgreenstatic int 449132987Sgreenmb_ctor_pack(void *mem, int size, void *arg, int how) 450129906Sbmilekic{ 451129906Sbmilekic struct mbuf *m; 452129906Sbmilekic struct mb_args *args; 453132987Sgreen#ifdef MAC 454132987Sgreen int error; 455132987Sgreen#endif 456132987Sgreen int flags; 457129906Sbmilekic short type; 458129906Sbmilekic 459129906Sbmilekic m = (struct mbuf *)mem; 460129906Sbmilekic args = (struct mb_args *)arg; 461129906Sbmilekic flags = args->flags; 462129906Sbmilekic type = args->type; 463129906Sbmilekic 464147537Ssilby#ifdef INVARIANTS 465147537Ssilby trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how); 466147537Ssilby#endif 467129906Sbmilekic m->m_next = NULL; 468129947Sbmilekic m->m_nextpkt = NULL; 469129906Sbmilekic m->m_data = m->m_ext.ext_buf; 470151976Sandre m->m_len = 0; 471151976Sandre m->m_flags = (flags | M_EXT); 472151976Sandre m->m_type = type; 473129906Sbmilekic 474129906Sbmilekic if (flags & M_PKTHDR) { 475129906Sbmilekic m->m_pkthdr.rcvif = NULL; 476151976Sandre m->m_pkthdr.len = 0; 477151976Sandre m->m_pkthdr.header = NULL; 478129906Sbmilekic m->m_pkthdr.csum_flags = 0; 479151976Sandre m->m_pkthdr.csum_data = 0; 480129906Sbmilekic SLIST_INIT(&m->m_pkthdr.tags); 481129906Sbmilekic#ifdef MAC 482129906Sbmilekic /* If the label init fails, fail the alloc */ 483132987Sgreen error = mac_init_mbuf(m, how); 484132987Sgreen if (error) 485132987Sgreen return (error); 486129906Sbmilekic#endif 487129906Sbmilekic } 488151976Sandre /* m_ext is already initialized. */ 489151976Sandre 490132987Sgreen return (0); 491129906Sbmilekic} 492129906Sbmilekic 493129906Sbmilekic/* 494129906Sbmilekic * This is the protocol drain routine. 495129906Sbmilekic * 496129906Sbmilekic * No locks should be held when this is called. The drain routines have to 497129906Sbmilekic * presently acquire some locks which raises the possibility of lock order 498129906Sbmilekic * reversal. 499129906Sbmilekic */ 500129906Sbmilekicstatic void 501129906Sbmilekicmb_reclaim(void *junk) 502129906Sbmilekic{ 503129906Sbmilekic struct domain *dp; 504129906Sbmilekic struct protosw *pr; 505129906Sbmilekic 506129906Sbmilekic WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL, 507129906Sbmilekic "mb_reclaim()"); 508129906Sbmilekic 509129906Sbmilekic for (dp = domains; dp != NULL; dp = dp->dom_next) 510129906Sbmilekic for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) 511129906Sbmilekic if (pr->pr_drain != NULL) 512129906Sbmilekic (*pr->pr_drain)(); 513129906Sbmilekic} 514