bsd/kern/uipc_mbuf.c

144  * Allocation paths are different depending on the class of objects:
208  * paths are different depending on the class of objects:
361  * gets removed (SLF_DETACHED) from the class's slab list.  A chunk that is
363  * it also causes the slab to be reinserted back to class's slab list, if
368  * well as to convert or move a slab from one class to another; e.g. the
381 	u_int8_t	sl_class;	/* controlling mbuf class */
546 	mbuf_class_t	mtbl_class;	/* class type */
547 	mcache_t	*mtbl_cache;	/* mcache for this buffer class */
829  * The structure that holds all mbuf class statistics exportable via sysctl.
976 		/* Calculate total count specific to each class */
1475 	 * so that m_total(class) is at least m_minlimit(class).
1497 	/* Create the cache for each class */
1580  * Obtain a slab of object(s) from the class's freelist.
1583 slab_alloc(mbuf_class_t class, int wait)
1590 	VERIFY(class != MC_16KCL || njcl > 0);
1593 	VERIFY(m_cobjlist(class) == NULL);
1602 	if ((class == MC_MBUF || class == MC_CL) && (wait & MCR_COMP))
1603 		sp = (mcl_slab_t *)TAILQ_LAST(&m_slablist(class), mcl_slhead);
1605 		sp = (mcl_slab_t *)TAILQ_FIRST(&m_slablist(class));
1608 		VERIFY(m_infree(class) == 0 && m_slab_cnt(class) == 0);
1609 		/* The slab list for this class is empty */
1613 	VERIFY(m_infree(class) > 0);
1615 	VERIFY(sp->sl_class == class &&
1620 	if (class == MC_MBUF) {
1623 	} else if (class == MC_CL) {
1640 		mcache_audit_t *mca = mcl_audit_buf2mca(class, buf);
1643 		if (class == MC_MBUF)
1647 	if (class == MC_CL) {
1656 	} else if (class == MC_BIGCL) {
1663 		    sp->sl_len == m_maxsize(class) && sp->sl_head == NULL);
1664 	} else if (class == MC_16KCL) {
1670 		    sp->sl_len == m_maxsize(class) && sp->sl_head == NULL);
1689 		VERIFY(class == MC_MBUF);
1709 	/* If empty, remove this slab from the class's freelist */
1711 		VERIFY(class != MC_MBUF || sp->sl_refcnt == NMBPBG);
1712 		VERIFY(class != MC_CL || sp->sl_refcnt == NCLPBG);
1713 		slab_remove(sp, class);
1720  * Place a slab of object(s) back into a class's slab list.
1723 slab_free(mbuf_class_t class, mcache_obj_t *buf)
1729 	VERIFY(class != MC_16KCL || njcl > 0);
1732 	VERIFY(sp->sl_class == class && slab_inrange(sp, buf) &&
1738 	if (class == MC_CL) {
1749 	} else if (class == MC_BIGCL) {
1756 		    sp->sl_len == m_maxsize(class) && sp->sl_head == NULL);
1758 	} else if (class == MC_16KCL) {
1767 		    sp->sl_len == m_maxsize(class) && sp->sl_head == NULL);
1787 		VERIFY(class == MC_MBUF);
1801 		mcache_audit_t *mca = mcl_audit_buf2mca(class, buf);
1803 			mcache_audit_free_verify(mca, buf, 0, m_maxsize(class));
1808 	if (class == MC_CL) {
1811 	} else if (class == MC_BIGCL) {
1814 	} else if (class == MC_16KCL) {
1827 	if (class == MC_MBUF && sp->sl_refcnt == 0 &&
1828 	    m_total(class) > m_minlimit(class) &&
1850 		/* Remove the slab from the mbuf class's slab list */
1851 		slab_remove(sp, class);
1865 		/* And finally switch class */
1866 		class = MC_BIGCL;
1867 	} else if (class == MC_CL && sp->sl_refcnt == 0 &&
1868 	    m_total(class) > m_minlimit(class) &&
1888 		/* Remove the slab from the 2KB cluster class's slab list */
1889 		slab_remove(sp, class);
1903 		/* And finally switch class */
1904 		class = MC_BIGCL;
1907 	/* Reinsert the slab to the class's slab list */
1909 		slab_insert(sp, class);
1922 	mbuf_class_t class = (mbuf_class_t)arg;
1926 	ASSERT(MBUF_CLASS_VALID(class) && !MBUF_CLASS_COMPOSITE(class));
1932 		if ((*list = slab_alloc(class, wait)) != NULL) {
1943 				if (!mbuf_cached_above(class, wait) &&
1944 				    m_infree(class) < m_total(class) >> 5) {
1945 					(void) freelist_populate(class, 1,
1951 			VERIFY(m_infree(class) == 0 || class == MC_CL);
1953 			(void) freelist_populate(class, 1,
1956 			if (m_infree(class) > 0)
1960 			if (mbuf_cached_above(class, wait))
1969 					m_fail_cnt(class)++;
1983 			    mbuf_sleep(class, need, wait))
1990 	m_alloc_cnt(class) += num - need;
2004 	mbuf_class_t class = (mbuf_class_t)arg;
2009 	ASSERT(MBUF_CLASS_VALID(class) && !MBUF_CLASS_COMPOSITE(class));
2016 		slab_free(class, list);
2021 	m_free_cnt(class) += num;
2043 	mbuf_class_t class = (mbuf_class_t)arg;
2046 	ASSERT(MBUF_CLASS_VALID(class) && !MBUF_CLASS_COMPOSITE(class));
2050 		mca = mcl_audit_buf2mca(class, list);
2053 		if (class == MC_MBUF) {
2057 			mcl_audit_cluster(mca, list, m_maxsize(class),
2063 			mcache_buffer_log(mca, list, m_cache(class));
2086 	mbuf_class_t class = (mbuf_class_t)arg;
2089 	ASSERT(MBUF_CLASS_VALID(class));
2096 		m_notified(class)++;
2106  * Obtain object(s) from the composite class's freelist.
2109 cslab_alloc(mbuf_class_t class, mcache_obj_t ***plist, unsigned int num)
2118 	VERIFY(class != MC_MBUF_16KCL || njcl > 0);
2122 	while ((*list = m_cobjlist(class)) != NULL) {
2132 		if (class == MC_MBUF_CL) {
2139 		if (class == MC_MBUF_16KCL) {
2149 		if ((m_cobjlist(class) = (*list)->obj_next) != NULL &&
2150 		    !MBUF_IN_MAP(m_cobjlist(class))) {
2151 			slab_nextptr_panic(sp, m_cobjlist(class));
2160 	m_infree(class) -= (num - need);
2166  * Place object(s) back into a composite class's freelist.
2169 cslab_free(mbuf_class_t class, mcache_obj_t *list, int purged)
2180 	ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
2181 	VERIFY(class != MC_MBUF_16KCL || njcl > 0);
2184 	if (class == MC_MBUF_CL) {
2186 	} else if (class == MC_MBUF_BIGCL) {
2189 		VERIFY(class == MC_MBUF_16KCL);
2262 			VERIFY(m_total(class) > 0);
2263 			m_total(class)--;
2271 			if (class == MC_MBUF_CL)
2273 			else if (class == MC_MBUF_BIGCL)
2285 		tail->obj_next = m_cobjlist(class);
2286 		m_cobjlist(class) = list;
2287 		m_infree(class) += num;
2307 	mbuf_class_t class = (mbuf_class_t)arg;
2318 	ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
2321 	VERIFY(class != MC_MBUF_16KCL || njcl > 0);
2323 	/* There should not be any slab for this class */
2324 	VERIFY(m_slab_cnt(class) == 0 &&
2325 	    m_slablist(class).tqh_first == NULL &&
2326 	    m_slablist(class).tqh_last == NULL);
2331 	num = cslab_alloc(class, plist, needed);
2334 		m_alloc_cnt(class) += num;
2367 	if (class == MC_MBUF_CL) {
2369 	} else if (class == MC_MBUF_BIGCL) {
2372 		VERIFY(class == MC_MBUF_16KCL);
2441 				if (class == MC_MBUF_CL)
2443 				else if (class == MC_MBUF_BIGCL)
2454 		if (class == MC_MBUF_16KCL) {
2456 		} else if (class == MC_MBUF_BIGCL) {
2482 		m_total(class) += cnum;
2483 		VERIFY(m_total(class) <= m_maxlimit(class));
2484 		m_alloc_cnt(class) += num + cnum;
2487 		m_fail_cnt(class) += (want - (num + cnum));
2501 	mbuf_class_t class = (mbuf_class_t)arg;
2505 	ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
2509 	num = cslab_free(class, list, purged);
2510 	m_free_cnt(class) += num;
2532 	mbuf_class_t class = (mbuf_class_t)arg;
2539 	ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
2547 			mcache_buffer_log(mca, m, m_cache(class));
2567 		if (class == MC_MBUF_CL)
2573 		if (class == MC_MBUF_16KCL) {
2584 		if (class == MC_MBUF_CL)
2586 		else if (class == MC_MBUF_BIGCL)
2592 			mcache_buffer_log(mca, cl, m_cache(class));
2628 	 * the maximum allowed for each class.  To prevent it, this entire
2854  * Populate the global freelist of the corresponding buffer class.
2857 freelist_populate(mbuf_class_t class, unsigned int num, int wait)
2862 	VERIFY(class == MC_MBUF || class == MC_CL || class == MC_BIGCL ||
2863 	    class == MC_16KCL);
2867 	switch (class) {
2871 		numpages = (num * m_size(class) + NBPG - 1) / NBPG;
2877 			if (class != MC_BIGCL || (wait & MCR_COMP))
2880 		if (class == MC_BIGCL)
2885 		return (m_clalloc(num, wait, m_maxsize(class)) != 0);
2893 	VERIFY(class == MC_MBUF || class == MC_CL);
2896 	int numobj = (class == MC_MBUF ? NMBPBG : NCLPBG);
2902 		    m_total(class) >= m_maxlimit(class))
2927 		slab_init(sp, class, sp->sl_flags,
2937 		m_total(class) += numobj;
2938 		m_infree(class) += numobj;
2941 		VERIFY(m_total(class) <= m_maxlimit(class));
2944 		if (class == MC_MBUF) {
2980 		slab_insert(sp, class);
2991  * For each class, initialize the freelist to hold m_minlimit() objects.
2994 freelist_init(mbuf_class_t class)
2998 	VERIFY(class == MC_CL || class == MC_BIGCL);
2999 	VERIFY(m_total(class) == 0);
3000 	VERIFY(m_minlimit(class) > 0);
3002 	while (m_total(class) < m_minlimit(class))
3003 		(void) freelist_populate(class, m_minlimit(class), M_WAIT);
3005 	VERIFY(m_total(class) >= m_minlimit(class));
3014 mbuf_cached_above(mbuf_class_t class, int wait)
3016 	switch (class) {
3048 	return (!mcache_bkt_isempty(m_cache(class)));
3055 mbuf_steal(mbuf_class_t class, unsigned int num)
3063 	switch (class) {
3074 		if (m_infree(class) > m_minlimit(class)) {
3075 			tot = cslab_alloc(class, &list,
3076 			    MIN(num, m_infree(class)));
3081 			(void) cslab_free(class, top, 1);
3093 m_reclaim(mbuf_class_t class, unsigned int num, boolean_t comp)
3107 	switch (class) {
3142 	 * Run through each marked class and check if we really need to
3144 	 * layer used by the class.  If so, remember the classes since
3588 	m_service_class_init(from);		/* reset svc class from src */
5861 mbuf_waiter_inc(mbuf_class_t class, boolean_t comp)
5863 	mcache_waiter_inc(m_cache(class));
5865 		if (class == MC_CL) {
5867 		} else if (class == MC_BIGCL) {
5869 		} else if (class == MC_16KCL) {
5882 mbuf_waiter_dec(mbuf_class_t class, boolean_t comp)
5884 	mcache_waiter_dec(m_cache(class));
5886 		if (class == MC_CL) {
5888 		} else if (class == MC_BIGCL) {
5890 		} else if (class == MC_16KCL) {
5928 mbuf_sleep(mbuf_class_t class, unsigned int num, int wait)
5935 	if (mbuf_cached_above(class, wait)) {
5941 	m_reclaim(class, num, (wait & MCR_COMP));
5944 	if (m_infree(class) > 0) {
5947 	} else if (mbuf_cached_above(class, wait)) {
5960 	mbuf_waiter_inc(class, (wait & MCR_COMP));
5974 	(void) msleep(mb_waitchan, mbuf_mlock, (PZERO-1), m_cname(class), NULL);
5977 	mbuf_waiter_dec(class, (wait & MCR_COMP));
5980 	if (m_infree(class) > 0) {
5983 	} else if (mbuf_cached_above(class, wait)) {
6134 slab_init(mcl_slab_t *sp, mbuf_class_t class, u_int32_t flags,
6137 	sp->sl_class = class;
6148 slab_insert(mcl_slab_t *sp, mbuf_class_t class)
6151 	m_slab_cnt(class)++;
6152 	TAILQ_INSERT_TAIL(&m_slablist(class), sp, sl_link);
6154 	if (class == MC_16KCL) {
6167 slab_remove(mcl_slab_t *sp, mbuf_class_t class)
6170 	VERIFY(m_slab_cnt(class) > 0);
6171 	m_slab_cnt(class)--;
6172 	TAILQ_REMOVE(&m_slablist(class), sp, sl_link);
6174 	if (class == MC_16KCL) {
6295 mcl_audit_buf2mca(mbuf_class_t class, mcache_obj_t *o)
6301 	VERIFY(IS_P2ALIGNED(o, MIN(m_maxsize(class), NBPG)));
6303 	switch (class) {