kern_mbuf.c revision 162377
1129906Sbmilekic/*- 2141991Sbmilekic * Copyright (c) 2004, 2005, 3141991Sbmilekic * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved. 4129906Sbmilekic * 5129906Sbmilekic * Redistribution and use in source and binary forms, with or without 6129906Sbmilekic * modification, are permitted provided that the following conditions 7129906Sbmilekic * are met: 8129906Sbmilekic * 1. Redistributions of source code must retain the above copyright 9129906Sbmilekic * notice unmodified, this list of conditions and the following 10129906Sbmilekic * disclaimer. 11129906Sbmilekic * 2. Redistributions in binary form must reproduce the above copyright 12129906Sbmilekic * notice, this list of conditions and the following disclaimer in the 13129906Sbmilekic * documentation and/or other materials provided with the distribution. 14129906Sbmilekic * 15129906Sbmilekic * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16129906Sbmilekic * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17129906Sbmilekic * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18129906Sbmilekic * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19129906Sbmilekic * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20129906Sbmilekic * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21129906Sbmilekic * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22129906Sbmilekic * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23129906Sbmilekic * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24129906Sbmilekic * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25129906Sbmilekic * SUCH DAMAGE. 26129906Sbmilekic */ 27129906Sbmilekic 28129906Sbmilekic#include <sys/cdefs.h> 29129906Sbmilekic__FBSDID("$FreeBSD: head/sys/kern/kern_mbuf.c 162377 2006-09-17 13:44:32Z andre $"); 30129906Sbmilekic 31129906Sbmilekic#include "opt_mac.h" 32129906Sbmilekic#include "opt_param.h" 33129906Sbmilekic 34129906Sbmilekic#include <sys/param.h> 35129906Sbmilekic#include <sys/mac.h> 36129906Sbmilekic#include <sys/malloc.h> 37129906Sbmilekic#include <sys/systm.h> 38129906Sbmilekic#include <sys/mbuf.h> 39129906Sbmilekic#include <sys/domain.h> 40129906Sbmilekic#include <sys/eventhandler.h> 41129906Sbmilekic#include <sys/kernel.h> 42129906Sbmilekic#include <sys/protosw.h> 43129906Sbmilekic#include <sys/smp.h> 44129906Sbmilekic#include <sys/sysctl.h> 45129906Sbmilekic 46129906Sbmilekic#include <vm/vm.h> 47129906Sbmilekic#include <vm/vm_page.h> 48129906Sbmilekic#include <vm/uma.h> 49147537Ssilby#include <vm/uma_int.h> 50147537Ssilby#include <vm/uma_dbg.h> 51129906Sbmilekic 52129906Sbmilekic/* 53129906Sbmilekic * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA 54129906Sbmilekic * Zones. 55129906Sbmilekic * 56129906Sbmilekic * Mbuf Clusters (2K, contiguous) are allocated from the Cluster 57129906Sbmilekic * Zone. The Zone can be capped at kern.ipc.nmbclusters, if the 58129906Sbmilekic * administrator so desires. 59129906Sbmilekic * 60129906Sbmilekic * Mbufs are allocated from a UMA Master Zone called the Mbuf 61129906Sbmilekic * Zone. 62129906Sbmilekic * 63129906Sbmilekic * Additionally, FreeBSD provides a Packet Zone, which it 64129906Sbmilekic * configures as a Secondary Zone to the Mbuf Master Zone, 65129906Sbmilekic * thus sharing backend Slab kegs with the Mbuf Master Zone. 66129906Sbmilekic * 67129906Sbmilekic * Thus common-case allocations and locking are simplified: 68129906Sbmilekic * 69129906Sbmilekic * m_clget() m_getcl() 70129906Sbmilekic * | | 71129906Sbmilekic * | .------------>[(Packet Cache)] m_get(), m_gethdr() 72129906Sbmilekic * | | [ Packet ] | 73129906Sbmilekic * [(Cluster Cache)] [ Secondary ] [ (Mbuf Cache) ] 74129906Sbmilekic * [ Cluster Zone ] [ Zone ] [ Mbuf Master Zone ] 75129906Sbmilekic * | \________ | 76129906Sbmilekic * [ Cluster Keg ] \ / 77156023Sglebius * | [ Mbuf Keg ] 78129906Sbmilekic * [ Cluster Slabs ] | 79129906Sbmilekic * | [ Mbuf Slabs ] 80129906Sbmilekic * \____________(VM)_________________/ 81151976Sandre * 82151976Sandre * 83156023Sglebius * Whenever an object is allocated with uma_zalloc() out of 84151976Sandre * one of the Zones its _ctor_ function is executed. The same 85156023Sglebius * for any deallocation through uma_zfree() the _dtor_ function 86151976Sandre * is executed. 87156023Sglebius * 88151976Sandre * Caches are per-CPU and are filled from the Master Zone. 89151976Sandre * 90156023Sglebius * Whenever an object is allocated from the underlying global 91151976Sandre * memory pool it gets pre-initialized with the _zinit_ functions. 92151976Sandre * When the Keg's are overfull objects get decomissioned with 93151976Sandre * _zfini_ functions and free'd back to the global memory pool. 94151976Sandre * 95129906Sbmilekic */ 96129906Sbmilekic 97151976Sandreint nmbclusters; /* limits number of mbuf clusters */ 98155780Sandreint nmbjumbop; /* limits number of page size jumbo clusters */ 99151976Sandreint nmbjumbo9; /* limits number of 9k jumbo clusters */ 100151976Sandreint nmbjumbo16; /* limits number of 16k jumbo clusters */ 101129906Sbmilekicstruct mbstat mbstat; 102129906Sbmilekic 103129906Sbmilekicstatic void 104129906Sbmilekictunable_mbinit(void *dummy) 105129906Sbmilekic{ 106129906Sbmilekic 107129906Sbmilekic /* This has to be done before VM init. */ 108129906Sbmilekic nmbclusters = 1024 + maxusers * 64; 109129906Sbmilekic TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters); 110129906Sbmilekic} 111129906SbmilekicSYSINIT(tunable_mbinit, SI_SUB_TUNABLES, SI_ORDER_ANY, tunable_mbinit, NULL); 112129906Sbmilekic 113151976Sandre/* XXX: These should be tuneables. Can't change UMA limits on the fly. */ 114157927Spsstatic int 115157927Spssysctl_nmbclusters(SYSCTL_HANDLER_ARGS) 116157927Sps{ 117157927Sps int error, newnmbclusters; 118157927Sps 119157927Sps newnmbclusters = nmbclusters; 120157927Sps error = sysctl_handle_int(oidp, &newnmbclusters, sizeof(int), req); 121157927Sps if (error == 0 && req->newptr) { 122157927Sps if (newnmbclusters > nmbclusters) { 123157927Sps nmbclusters = newnmbclusters; 124157927Sps uma_zone_set_max(zone_clust, nmbclusters); 125157927Sps EVENTHANDLER_INVOKE(nmbclusters_change); 126157927Sps } else 127157927Sps error = EINVAL; 128157927Sps } 129157927Sps return (error); 130157927Sps} 131157927SpsSYSCTL_PROC(_kern_ipc, OID_AUTO, nmbclusters, CTLTYPE_INT|CTLFLAG_RW, 132157927Sps&nmbclusters, 0, sysctl_nmbclusters, "IU", 133129906Sbmilekic "Maximum number of mbuf clusters allowed"); 134155780SandreSYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbop, CTLFLAG_RW, &nmbjumbop, 0, 135155780Sandre "Maximum number of mbuf page size jumbo clusters allowed"); 136151976SandreSYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbo9, CTLFLAG_RW, &nmbjumbo9, 0, 137151976Sandre "Maximum number of mbuf 9k jumbo clusters allowed"); 138151976SandreSYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbo16, CTLFLAG_RW, &nmbjumbo16, 0, 139151976Sandre "Maximum number of mbuf 16k jumbo clusters allowed"); 140129906SbmilekicSYSCTL_STRUCT(_kern_ipc, OID_AUTO, mbstat, CTLFLAG_RD, &mbstat, mbstat, 141129906Sbmilekic "Mbuf general information and statistics"); 142129906Sbmilekic 143129906Sbmilekic/* 144129906Sbmilekic * Zones from which we allocate. 145129906Sbmilekic */ 146129906Sbmilekicuma_zone_t zone_mbuf; 147129906Sbmilekicuma_zone_t zone_clust; 148129906Sbmilekicuma_zone_t zone_pack; 149155780Sandreuma_zone_t zone_jumbop; 150151976Sandreuma_zone_t zone_jumbo9; 151151976Sandreuma_zone_t zone_jumbo16; 152151976Sandreuma_zone_t zone_ext_refcnt; 153129906Sbmilekic 154129906Sbmilekic/* 155129906Sbmilekic * Local prototypes. 156129906Sbmilekic */ 157132987Sgreenstatic int mb_ctor_mbuf(void *, int, void *, int); 158132987Sgreenstatic int mb_ctor_clust(void *, int, void *, int); 159132987Sgreenstatic int mb_ctor_pack(void *, int, void *, int); 160129906Sbmilekicstatic void mb_dtor_mbuf(void *, int, void *); 161151976Sandrestatic void mb_dtor_clust(void *, int, void *); 162151976Sandrestatic void mb_dtor_pack(void *, int, void *); 163151976Sandrestatic int mb_zinit_pack(void *, int, int); 164151976Sandrestatic void mb_zfini_pack(void *, int); 165129906Sbmilekic 166129906Sbmilekicstatic void mb_reclaim(void *); 167129906Sbmilekicstatic void mbuf_init(void *); 168129906Sbmilekic 169135510Sbrian/* Ensure that MSIZE doesn't break dtom() - it must be a power of 2 */ 170135510SbrianCTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE); 171135510Sbrian 172129906Sbmilekic/* 173129906Sbmilekic * Initialize FreeBSD Network buffer allocation. 174129906Sbmilekic */ 175129906SbmilekicSYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL) 176129906Sbmilekicstatic void 177129906Sbmilekicmbuf_init(void *dummy) 178129906Sbmilekic{ 179129906Sbmilekic 180129906Sbmilekic /* 181129906Sbmilekic * Configure UMA zones for Mbufs, Clusters, and Packets. 182129906Sbmilekic */ 183151976Sandre zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE, 184151976Sandre mb_ctor_mbuf, mb_dtor_mbuf, 185147537Ssilby#ifdef INVARIANTS 186151976Sandre trash_init, trash_fini, 187147537Ssilby#else 188151976Sandre NULL, NULL, 189147537Ssilby#endif 190151976Sandre MSIZE - 1, UMA_ZONE_MAXBUCKET); 191151976Sandre 192148095Srwatson zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES, 193151976Sandre mb_ctor_clust, mb_dtor_clust, 194147537Ssilby#ifdef INVARIANTS 195151976Sandre trash_init, trash_fini, 196147537Ssilby#else 197151976Sandre NULL, NULL, 198147537Ssilby#endif 199151976Sandre UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 200129906Sbmilekic if (nmbclusters > 0) 201129906Sbmilekic uma_zone_set_max(zone_clust, nmbclusters); 202151976Sandre 203148095Srwatson zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack, 204151976Sandre mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf); 205129906Sbmilekic 206156023Sglebius /* Make jumbo frame zone too. Page size, 9k and 16k. */ 207155780Sandre zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE, 208153232Sandre mb_ctor_clust, mb_dtor_clust, 209153232Sandre#ifdef INVARIANTS 210153232Sandre trash_init, trash_fini, 211153232Sandre#else 212153232Sandre NULL, NULL, 213153232Sandre#endif 214153232Sandre UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 215155780Sandre if (nmbjumbop > 0) 216155780Sandre uma_zone_set_max(zone_jumbop, nmbjumbop); 217153232Sandre 218151976Sandre zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES, 219151976Sandre mb_ctor_clust, mb_dtor_clust, 220151976Sandre#ifdef INVARIANTS 221151976Sandre trash_init, trash_fini, 222151976Sandre#else 223151976Sandre NULL, NULL, 224151976Sandre#endif 225151976Sandre UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 226151976Sandre if (nmbjumbo9 > 0) 227151976Sandre uma_zone_set_max(zone_jumbo9, nmbjumbo9); 228129906Sbmilekic 229151976Sandre zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES, 230151976Sandre mb_ctor_clust, mb_dtor_clust, 231151976Sandre#ifdef INVARIANTS 232151976Sandre trash_init, trash_fini, 233151976Sandre#else 234151976Sandre NULL, NULL, 235151976Sandre#endif 236151976Sandre UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 237151976Sandre if (nmbjumbo16 > 0) 238151976Sandre uma_zone_set_max(zone_jumbo16, nmbjumbo16); 239151976Sandre 240151976Sandre zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int), 241151976Sandre NULL, NULL, 242151976Sandre NULL, NULL, 243151976Sandre UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 244151976Sandre 245151976Sandre /* uma_prealloc() goes here... */ 246151976Sandre 247129906Sbmilekic /* 248129906Sbmilekic * Hook event handler for low-memory situation, used to 249129906Sbmilekic * drain protocols and push data back to the caches (UMA 250129906Sbmilekic * later pushes it back to VM). 251129906Sbmilekic */ 252129906Sbmilekic EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL, 253129906Sbmilekic EVENTHANDLER_PRI_FIRST); 254129906Sbmilekic 255129906Sbmilekic /* 256129906Sbmilekic * [Re]set counters and local statistics knobs. 257129906Sbmilekic * XXX Some of these should go and be replaced, but UMA stat 258129906Sbmilekic * gathering needs to be revised. 259129906Sbmilekic */ 260129906Sbmilekic mbstat.m_mbufs = 0; 261129906Sbmilekic mbstat.m_mclusts = 0; 262129906Sbmilekic mbstat.m_drain = 0; 263129906Sbmilekic mbstat.m_msize = MSIZE; 264129906Sbmilekic mbstat.m_mclbytes = MCLBYTES; 265129906Sbmilekic mbstat.m_minclsize = MINCLSIZE; 266129906Sbmilekic mbstat.m_mlen = MLEN; 267129906Sbmilekic mbstat.m_mhlen = MHLEN; 268129906Sbmilekic mbstat.m_numtypes = MT_NTYPES; 269129906Sbmilekic 270129906Sbmilekic mbstat.m_mcfail = mbstat.m_mpfail = 0; 271129906Sbmilekic mbstat.sf_iocnt = 0; 272129906Sbmilekic mbstat.sf_allocwait = mbstat.sf_allocfail = 0; 273129906Sbmilekic} 274129906Sbmilekic 275129906Sbmilekic/* 276129906Sbmilekic * Constructor for Mbuf master zone. 277129906Sbmilekic * 278129906Sbmilekic * The 'arg' pointer points to a mb_args structure which 279129906Sbmilekic * contains call-specific information required to support the 280151976Sandre * mbuf allocation API. See mbuf.h. 281129906Sbmilekic */ 282132987Sgreenstatic int 283132987Sgreenmb_ctor_mbuf(void *mem, int size, void *arg, int how) 284129906Sbmilekic{ 285129906Sbmilekic struct mbuf *m; 286129906Sbmilekic struct mb_args *args; 287132987Sgreen#ifdef MAC 288132987Sgreen int error; 289132987Sgreen#endif 290129906Sbmilekic int flags; 291129906Sbmilekic short type; 292129906Sbmilekic 293147537Ssilby#ifdef INVARIANTS 294147537Ssilby trash_ctor(mem, size, arg, how); 295147537Ssilby#endif 296129906Sbmilekic m = (struct mbuf *)mem; 297129906Sbmilekic args = (struct mb_args *)arg; 298129906Sbmilekic flags = args->flags; 299129906Sbmilekic type = args->type; 300129906Sbmilekic 301151976Sandre /* 302151976Sandre * The mbuf is initialized later. The caller has the 303156059Sglebius * responsibility to set up any MAC labels too. 304151976Sandre */ 305151976Sandre if (type == MT_NOINIT) 306151976Sandre return (0); 307151976Sandre 308129906Sbmilekic m->m_next = NULL; 309129906Sbmilekic m->m_nextpkt = NULL; 310151976Sandre m->m_len = 0; 311129947Sbmilekic m->m_flags = flags; 312151976Sandre m->m_type = type; 313129906Sbmilekic if (flags & M_PKTHDR) { 314129906Sbmilekic m->m_data = m->m_pktdat; 315129906Sbmilekic m->m_pkthdr.rcvif = NULL; 316151976Sandre m->m_pkthdr.len = 0; 317151976Sandre m->m_pkthdr.header = NULL; 318129906Sbmilekic m->m_pkthdr.csum_flags = 0; 319151976Sandre m->m_pkthdr.csum_data = 0; 320162377Sandre m->m_pkthdr.tso_segsz = 0; 321162377Sandre m->m_pkthdr.ether_vtag = 0; 322129906Sbmilekic SLIST_INIT(&m->m_pkthdr.tags); 323129906Sbmilekic#ifdef MAC 324129906Sbmilekic /* If the label init fails, fail the alloc */ 325132987Sgreen error = mac_init_mbuf(m, how); 326132987Sgreen if (error) 327132987Sgreen return (error); 328129906Sbmilekic#endif 329129947Sbmilekic } else 330129906Sbmilekic m->m_data = m->m_dat; 331132987Sgreen return (0); 332129906Sbmilekic} 333129906Sbmilekic 334129906Sbmilekic/* 335151976Sandre * The Mbuf master zone destructor. 336129906Sbmilekic */ 337129906Sbmilekicstatic void 338129906Sbmilekicmb_dtor_mbuf(void *mem, int size, void *arg) 339129906Sbmilekic{ 340129906Sbmilekic struct mbuf *m; 341129906Sbmilekic 342129906Sbmilekic m = (struct mbuf *)mem; 343129906Sbmilekic if ((m->m_flags & M_PKTHDR) != 0) 344129906Sbmilekic m_tag_delete_chain(m, NULL); 345151976Sandre KASSERT((m->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__)); 346147537Ssilby#ifdef INVARIANTS 347147537Ssilby trash_dtor(mem, size, arg); 348147537Ssilby#endif 349129906Sbmilekic} 350129906Sbmilekic 351151976Sandre/* 352151976Sandre * The Mbuf Packet zone destructor. 353151976Sandre */ 354129906Sbmilekicstatic void 355129906Sbmilekicmb_dtor_pack(void *mem, int size, void *arg) 356129906Sbmilekic{ 357129906Sbmilekic struct mbuf *m; 358129906Sbmilekic 359129906Sbmilekic m = (struct mbuf *)mem; 360129906Sbmilekic if ((m->m_flags & M_PKTHDR) != 0) 361129906Sbmilekic m_tag_delete_chain(m, NULL); 362151976Sandre 363151976Sandre /* Make sure we've got a clean cluster back. */ 364151976Sandre KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__)); 365151976Sandre KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__)); 366151976Sandre KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__)); 367151976Sandre KASSERT(m->m_ext.ext_args == NULL, ("%s: ext_args != NULL", __func__)); 368151976Sandre KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__)); 369152130Sglebius KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__)); 370151976Sandre KASSERT(*m->m_ext.ref_cnt == 1, ("%s: ref_cnt != 1", __func__)); 371147537Ssilby#ifdef INVARIANTS 372147537Ssilby trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg); 373147537Ssilby#endif 374129906Sbmilekic} 375129906Sbmilekic 376129906Sbmilekic/* 377155780Sandre * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor. 378129906Sbmilekic * 379129906Sbmilekic * Here the 'arg' pointer points to the Mbuf which we 380151976Sandre * are configuring cluster storage for. If 'arg' is 381151976Sandre * empty we allocate just the cluster without setting 382151976Sandre * the mbuf to it. See mbuf.h. 383129906Sbmilekic */ 384132987Sgreenstatic int 385132987Sgreenmb_ctor_clust(void *mem, int size, void *arg, int how) 386129906Sbmilekic{ 387129906Sbmilekic struct mbuf *m; 388151976Sandre u_int *refcnt; 389151976Sandre int type = 0; 390129906Sbmilekic 391147537Ssilby#ifdef INVARIANTS 392147537Ssilby trash_ctor(mem, size, arg, how); 393147537Ssilby#endif 394129906Sbmilekic m = (struct mbuf *)arg; 395151976Sandre if (m != NULL) { 396151976Sandre switch (size) { 397151976Sandre case MCLBYTES: 398151976Sandre type = EXT_CLUSTER; 399151976Sandre break; 400155780Sandre#if MJUMPAGESIZE != MCLBYTES 401155780Sandre case MJUMPAGESIZE: 402155780Sandre type = EXT_JUMBOP; 403153232Sandre break; 404153294Sandre#endif 405151976Sandre case MJUM9BYTES: 406151976Sandre type = EXT_JUMBO9; 407151976Sandre break; 408151976Sandre case MJUM16BYTES: 409151976Sandre type = EXT_JUMBO16; 410151976Sandre break; 411151976Sandre default: 412151976Sandre panic("unknown cluster size"); 413151976Sandre break; 414151976Sandre } 415151976Sandre m->m_ext.ext_buf = (caddr_t)mem; 416151976Sandre m->m_data = m->m_ext.ext_buf; 417151976Sandre m->m_flags |= M_EXT; 418151976Sandre m->m_ext.ext_free = NULL; 419151976Sandre m->m_ext.ext_args = NULL; 420151976Sandre m->m_ext.ext_size = size; 421151976Sandre m->m_ext.ext_type = type; 422151976Sandre m->m_ext.ref_cnt = uma_find_refcnt(zone_clust, mem); 423151976Sandre *m->m_ext.ref_cnt = 1; 424151976Sandre } else { 425151976Sandre refcnt = uma_find_refcnt(zone_clust, mem); 426151976Sandre *refcnt = 1; 427151976Sandre } 428132987Sgreen return (0); 429129906Sbmilekic} 430129906Sbmilekic 431151976Sandre/* 432151976Sandre * The Mbuf Cluster zone destructor. 433151976Sandre */ 434129906Sbmilekicstatic void 435129906Sbmilekicmb_dtor_clust(void *mem, int size, void *arg) 436129906Sbmilekic{ 437151976Sandre 438152035Sandre KASSERT(*(uma_find_refcnt(zone_clust, mem)) <= 1, 439152035Sandre ("%s: refcnt incorrect %u", __func__, 440152035Sandre *(uma_find_refcnt(zone_clust, mem))) ); 441147537Ssilby#ifdef INVARIANTS 442147537Ssilby trash_dtor(mem, size, arg); 443147537Ssilby#endif 444129906Sbmilekic} 445129906Sbmilekic 446129906Sbmilekic/* 447129906Sbmilekic * The Packet secondary zone's init routine, executed on the 448151976Sandre * object's transition from mbuf keg slab to zone cache. 449129906Sbmilekic */ 450132987Sgreenstatic int 451151976Sandremb_zinit_pack(void *mem, int size, int how) 452129906Sbmilekic{ 453129906Sbmilekic struct mbuf *m; 454129906Sbmilekic 455151976Sandre m = (struct mbuf *)mem; /* m is virgin. */ 456156428Sandre if (uma_zalloc_arg(zone_clust, m, how) == NULL || 457156428Sandre m->m_ext.ext_buf == NULL) 458132987Sgreen return (ENOMEM); 459152101Sandre m->m_ext.ext_type = EXT_PACKET; /* Override. */ 460147537Ssilby#ifdef INVARIANTS 461147537Ssilby trash_init(m->m_ext.ext_buf, MCLBYTES, how); 462147537Ssilby#endif 463132987Sgreen return (0); 464129906Sbmilekic} 465129906Sbmilekic 466129906Sbmilekic/* 467129906Sbmilekic * The Packet secondary zone's fini routine, executed on the 468129906Sbmilekic * object's transition from zone cache to keg slab. 469129906Sbmilekic */ 470129906Sbmilekicstatic void 471151976Sandremb_zfini_pack(void *mem, int size) 472129906Sbmilekic{ 473129906Sbmilekic struct mbuf *m; 474129906Sbmilekic 475129906Sbmilekic m = (struct mbuf *)mem; 476147537Ssilby#ifdef INVARIANTS 477147537Ssilby trash_fini(m->m_ext.ext_buf, MCLBYTES); 478147537Ssilby#endif 479129906Sbmilekic uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL); 480147652Ssilby#ifdef INVARIANTS 481147652Ssilby trash_dtor(mem, size, NULL); 482147652Ssilby#endif 483129906Sbmilekic} 484129906Sbmilekic 485129906Sbmilekic/* 486129906Sbmilekic * The "packet" keg constructor. 487129906Sbmilekic */ 488132987Sgreenstatic int 489132987Sgreenmb_ctor_pack(void *mem, int size, void *arg, int how) 490129906Sbmilekic{ 491129906Sbmilekic struct mbuf *m; 492129906Sbmilekic struct mb_args *args; 493132987Sgreen#ifdef MAC 494132987Sgreen int error; 495132987Sgreen#endif 496132987Sgreen int flags; 497129906Sbmilekic short type; 498129906Sbmilekic 499129906Sbmilekic m = (struct mbuf *)mem; 500129906Sbmilekic args = (struct mb_args *)arg; 501129906Sbmilekic flags = args->flags; 502129906Sbmilekic type = args->type; 503129906Sbmilekic 504147537Ssilby#ifdef INVARIANTS 505147537Ssilby trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how); 506147537Ssilby#endif 507129906Sbmilekic m->m_next = NULL; 508129947Sbmilekic m->m_nextpkt = NULL; 509129906Sbmilekic m->m_data = m->m_ext.ext_buf; 510151976Sandre m->m_len = 0; 511151976Sandre m->m_flags = (flags | M_EXT); 512151976Sandre m->m_type = type; 513129906Sbmilekic 514129906Sbmilekic if (flags & M_PKTHDR) { 515129906Sbmilekic m->m_pkthdr.rcvif = NULL; 516151976Sandre m->m_pkthdr.len = 0; 517151976Sandre m->m_pkthdr.header = NULL; 518129906Sbmilekic m->m_pkthdr.csum_flags = 0; 519151976Sandre m->m_pkthdr.csum_data = 0; 520162377Sandre m->m_pkthdr.tso_segsz = 0; 521162377Sandre m->m_pkthdr.ether_vtag = 0; 522129906Sbmilekic SLIST_INIT(&m->m_pkthdr.tags); 523129906Sbmilekic#ifdef MAC 524129906Sbmilekic /* If the label init fails, fail the alloc */ 525132987Sgreen error = mac_init_mbuf(m, how); 526132987Sgreen if (error) 527132987Sgreen return (error); 528129906Sbmilekic#endif 529129906Sbmilekic } 530151976Sandre /* m_ext is already initialized. */ 531151976Sandre 532132987Sgreen return (0); 533129906Sbmilekic} 534129906Sbmilekic 535129906Sbmilekic/* 536129906Sbmilekic * This is the protocol drain routine. 537129906Sbmilekic * 538129906Sbmilekic * No locks should be held when this is called. The drain routines have to 539129906Sbmilekic * presently acquire some locks which raises the possibility of lock order 540129906Sbmilekic * reversal. 541129906Sbmilekic */ 542129906Sbmilekicstatic void 543129906Sbmilekicmb_reclaim(void *junk) 544129906Sbmilekic{ 545129906Sbmilekic struct domain *dp; 546129906Sbmilekic struct protosw *pr; 547129906Sbmilekic 548129906Sbmilekic WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL, 549129906Sbmilekic "mb_reclaim()"); 550129906Sbmilekic 551129906Sbmilekic for (dp = domains; dp != NULL; dp = dp->dom_next) 552129906Sbmilekic for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) 553129906Sbmilekic if (pr->pr_drain != NULL) 554129906Sbmilekic (*pr->pr_drain)(); 555129906Sbmilekic} 556