uipc_mbuf.c revision 254842
1/*-
2 * Copyright (c) 1982, 1986, 1988, 1991, 1993
3 *	The Regents of the University of California.  All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 * 4. Neither the name of the University nor the names of its contributors
14 *    may be used to endorse or promote products derived from this software
15 *    without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 *	@(#)uipc_mbuf.c	8.2 (Berkeley) 1/4/94
30 */
31
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD: head/sys/kern/uipc_mbuf.c 254842 2013-08-25 10:57:09Z andre $");
34
35#include "opt_param.h"
36#include "opt_mbuf_stress_test.h"
37#include "opt_mbuf_profiling.h"
38
39#include <sys/param.h>
40#include <sys/systm.h>
41#include <sys/kernel.h>
42#include <sys/limits.h>
43#include <sys/lock.h>
44#include <sys/malloc.h>
45#include <sys/mbuf.h>
46#include <sys/sysctl.h>
47#include <sys/domain.h>
48#include <sys/protosw.h>
49#include <sys/uio.h>
50
51int	max_linkhdr;
52int	max_protohdr;
53int	max_hdr;
54int	max_datalen;
55#ifdef MBUF_STRESS_TEST
56int	m_defragpackets;
57int	m_defragbytes;
58int	m_defraguseless;
59int	m_defragfailure;
60int	m_defragrandomfailures;
61#endif
62
63/*
64 * sysctl(8) exported objects
65 */
66SYSCTL_INT(_kern_ipc, KIPC_MAX_LINKHDR, max_linkhdr, CTLFLAG_RD,
67	   &max_linkhdr, 0, "Size of largest link layer header");
68SYSCTL_INT(_kern_ipc, KIPC_MAX_PROTOHDR, max_protohdr, CTLFLAG_RD,
69	   &max_protohdr, 0, "Size of largest protocol layer header");
70SYSCTL_INT(_kern_ipc, KIPC_MAX_HDR, max_hdr, CTLFLAG_RD,
71	   &max_hdr, 0, "Size of largest link plus protocol header");
72SYSCTL_INT(_kern_ipc, KIPC_MAX_DATALEN, max_datalen, CTLFLAG_RD,
73	   &max_datalen, 0, "Minimum space left in mbuf after max_hdr");
74#ifdef MBUF_STRESS_TEST
75SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragpackets, CTLFLAG_RD,
76	   &m_defragpackets, 0, "");
77SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragbytes, CTLFLAG_RD,
78	   &m_defragbytes, 0, "");
79SYSCTL_INT(_kern_ipc, OID_AUTO, m_defraguseless, CTLFLAG_RD,
80	   &m_defraguseless, 0, "");
81SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragfailure, CTLFLAG_RD,
82	   &m_defragfailure, 0, "");
83SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW,
84	   &m_defragrandomfailures, 0, "");
85#endif
86
87/*
88 * m_get2() allocates minimum mbuf that would fit "size" argument.
89 */
90struct mbuf *
91m_get2(int size, int how, short type, int flags)
92{
93	struct mb_args args;
94	struct mbuf *m, *n;
95
96	args.flags = flags;
97	args.type = type;
98
99	if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0))
100		return (uma_zalloc_arg(zone_mbuf, &args, how));
101	if (size <= MCLBYTES)
102		return (uma_zalloc_arg(zone_pack, &args, how));
103
104	if (size > MJUMPAGESIZE)
105		return (NULL);
106
107	m = uma_zalloc_arg(zone_mbuf, &args, how);
108	if (m == NULL)
109		return (NULL);
110
111	n = uma_zalloc_arg(zone_jumbop, m, how);
112	if (n == NULL) {
113		uma_zfree(zone_mbuf, m);
114		return (NULL);
115	}
116
117	return (m);
118}
119
120/*
121 * m_getjcl() returns an mbuf with a cluster of the specified size attached.
122 * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
123 */
124struct mbuf *
125m_getjcl(int how, short type, int flags, int size)
126{
127	struct mb_args args;
128	struct mbuf *m, *n;
129	uma_zone_t zone;
130
131	if (size == MCLBYTES)
132		return m_getcl(how, type, flags);
133
134	args.flags = flags;
135	args.type = type;
136
137	m = uma_zalloc_arg(zone_mbuf, &args, how);
138	if (m == NULL)
139		return (NULL);
140
141	zone = m_getzone(size);
142	n = uma_zalloc_arg(zone, m, how);
143	if (n == NULL) {
144		uma_zfree(zone_mbuf, m);
145		return (NULL);
146	}
147	return (m);
148}
149
150/*
151 * Allocate a given length worth of mbufs and/or clusters (whatever fits
152 * best) and return a pointer to the top of the allocated chain.  If an
153 * existing mbuf chain is provided, then we will append the new chain
154 * to the existing one but still return the top of the newly allocated
155 * chain.
156 */
157struct mbuf *
158m_getm2(struct mbuf *m, int len, int how, short type, int flags)
159{
160	struct mbuf *mb, *nm = NULL, *mtail = NULL;
161
162	KASSERT(len >= 0, ("%s: len is < 0", __func__));
163
164	/* Validate flags. */
165	flags &= (M_PKTHDR | M_EOR);
166
167	/* Packet header mbuf must be first in chain. */
168	if ((flags & M_PKTHDR) && m != NULL)
169		flags &= ~M_PKTHDR;
170
171	/* Loop and append maximum sized mbufs to the chain tail. */
172	while (len > 0) {
173		if (len > MCLBYTES)
174			mb = m_getjcl(how, type, (flags & M_PKTHDR),
175			    MJUMPAGESIZE);
176		else if (len >= MINCLSIZE)
177			mb = m_getcl(how, type, (flags & M_PKTHDR));
178		else if (flags & M_PKTHDR)
179			mb = m_gethdr(how, type);
180		else
181			mb = m_get(how, type);
182
183		/* Fail the whole operation if one mbuf can't be allocated. */
184		if (mb == NULL) {
185			if (nm != NULL)
186				m_freem(nm);
187			return (NULL);
188		}
189
190		/* Book keeping. */
191		len -= (mb->m_flags & M_EXT) ? mb->m_ext.ext_size :
192			((mb->m_flags & M_PKTHDR) ? MHLEN : MLEN);
193		if (mtail != NULL)
194			mtail->m_next = mb;
195		else
196			nm = mb;
197		mtail = mb;
198		flags &= ~M_PKTHDR;	/* Only valid on the first mbuf. */
199	}
200	if (flags & M_EOR)
201		mtail->m_flags |= M_EOR;  /* Only valid on the last mbuf. */
202
203	/* If mbuf was supplied, append new chain to the end of it. */
204	if (m != NULL) {
205		for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next)
206			;
207		mtail->m_next = nm;
208		mtail->m_flags &= ~M_EOR;
209	} else
210		m = nm;
211
212	return (m);
213}
214
215/*
216 * Free an entire chain of mbufs and associated external buffers, if
217 * applicable.
218 */
219void
220m_freem(struct mbuf *mb)
221{
222
223	while (mb != NULL)
224		mb = m_free(mb);
225}
226
227/*-
228 * Configure a provided mbuf to refer to the provided external storage
229 * buffer and setup a reference count for said buffer.  If the setting
230 * up of the reference count fails, the M_EXT bit will not be set.  If
231 * successfull, the M_EXT bit is set in the mbuf's flags.
232 *
233 * Arguments:
234 *    mb     The existing mbuf to which to attach the provided buffer.
235 *    buf    The address of the provided external storage buffer.
236 *    size   The size of the provided buffer.
237 *    freef  A pointer to a routine that is responsible for freeing the
238 *           provided external storage buffer.
239 *    args   A pointer to an argument structure (of any type) to be passed
240 *           to the provided freef routine (may be NULL).
241 *    flags  Any other flags to be passed to the provided mbuf.
242 *    type   The type that the external storage buffer should be
243 *           labeled with.
244 *
245 * Returns:
246 *    Nothing.
247 */
248int
249m_extadd(struct mbuf *mb, caddr_t buf, u_int size,
250    int (*freef)(struct mbuf *, void *, void *), void *arg1, void *arg2,
251    int flags, int type, int wait)
252{
253	KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__));
254
255	if (type != EXT_EXTREF)
256		mb->m_ext.ref_cnt = uma_zalloc(zone_ext_refcnt, wait);
257
258	if (mb->m_ext.ref_cnt == NULL)
259		return (ENOMEM);
260
261	*(mb->m_ext.ref_cnt) = 1;
262	mb->m_flags |= (M_EXT | flags);
263	mb->m_ext.ext_buf = buf;
264	mb->m_data = mb->m_ext.ext_buf;
265	mb->m_ext.ext_size = size;
266	mb->m_ext.ext_free = freef;
267	mb->m_ext.ext_arg1 = arg1;
268	mb->m_ext.ext_arg2 = arg2;
269	mb->m_ext.ext_type = type;
270	mb->m_ext.ext_flags = 0;
271
272	return (0);
273}
274
275/*
276 * Non-directly-exported function to clean up after mbufs with M_EXT
277 * storage attached to them if the reference count hits 1.
278 */
279void
280mb_free_ext(struct mbuf *m)
281{
282	int skipmbuf;
283
284	KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
285	KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__));
286
287	/*
288	 * check if the header is embedded in the cluster
289	 */
290	skipmbuf = (m->m_flags & M_NOFREE);
291
292	/* Free attached storage if this mbuf is the only reference to it. */
293	if (*(m->m_ext.ref_cnt) == 1 ||
294	    atomic_fetchadd_int(m->m_ext.ref_cnt, -1) == 1) {
295		switch (m->m_ext.ext_type) {
296		case EXT_PACKET:	/* The packet zone is special. */
297			if (*(m->m_ext.ref_cnt) == 0)
298				*(m->m_ext.ref_cnt) = 1;
299			uma_zfree(zone_pack, m);
300			return;		/* Job done. */
301		case EXT_CLUSTER:
302			uma_zfree(zone_clust, m->m_ext.ext_buf);
303			break;
304		case EXT_JUMBOP:
305			uma_zfree(zone_jumbop, m->m_ext.ext_buf);
306			break;
307		case EXT_JUMBO9:
308			uma_zfree(zone_jumbo9, m->m_ext.ext_buf);
309			break;
310		case EXT_JUMBO16:
311			uma_zfree(zone_jumbo16, m->m_ext.ext_buf);
312			break;
313		case EXT_SFBUF:
314		case EXT_NET_DRV:
315		case EXT_MOD_TYPE:
316		case EXT_DISPOSABLE:
317			*(m->m_ext.ref_cnt) = 0;
318			uma_zfree(zone_ext_refcnt, __DEVOLATILE(u_int *,
319				m->m_ext.ref_cnt));
320			/* FALLTHROUGH */
321		case EXT_EXTREF:
322			KASSERT(m->m_ext.ext_free != NULL,
323				("%s: ext_free not set", __func__));
324			(void)(*(m->m_ext.ext_free))(m, m->m_ext.ext_arg1,
325			    m->m_ext.ext_arg2);
326			break;
327		default:
328			KASSERT(m->m_ext.ext_type == 0,
329				("%s: unknown ext_type", __func__));
330		}
331	}
332	if (skipmbuf)
333		return;
334
335	/*
336	 * Free this mbuf back to the mbuf zone with all m_ext
337	 * information purged.
338	 */
339	m->m_ext.ext_buf = NULL;
340	m->m_ext.ext_free = NULL;
341	m->m_ext.ext_arg1 = NULL;
342	m->m_ext.ext_arg2 = NULL;
343	m->m_ext.ref_cnt = NULL;
344	m->m_ext.ext_size = 0;
345	m->m_ext.ext_type = 0;
346	m->m_ext.ext_flags = 0;
347	m->m_flags &= ~M_EXT;
348	uma_zfree(zone_mbuf, m);
349}
350
351/*
352 * Attach the cluster from *m to *n, set up m_ext in *n
353 * and bump the refcount of the cluster.
354 */
355static void
356mb_dupcl(struct mbuf *n, struct mbuf *m)
357{
358	KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
359	KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__));
360	KASSERT((n->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__));
361
362	if (*(m->m_ext.ref_cnt) == 1)
363		*(m->m_ext.ref_cnt) += 1;
364	else
365		atomic_add_int(m->m_ext.ref_cnt, 1);
366	n->m_ext.ext_buf = m->m_ext.ext_buf;
367	n->m_ext.ext_free = m->m_ext.ext_free;
368	n->m_ext.ext_arg1 = m->m_ext.ext_arg1;
369	n->m_ext.ext_arg2 = m->m_ext.ext_arg2;
370	n->m_ext.ext_size = m->m_ext.ext_size;
371	n->m_ext.ref_cnt = m->m_ext.ref_cnt;
372	n->m_ext.ext_type = m->m_ext.ext_type;
373	n->m_ext.ext_flags = m->m_ext.ext_flags;
374	n->m_flags |= M_EXT;
375	n->m_flags |= m->m_flags & M_RDONLY;
376}
377
378/*
379 * Clean up mbuf (chain) from any tags and packet headers.
380 * If "all" is set then the first mbuf in the chain will be
381 * cleaned too.
382 */
383void
384m_demote(struct mbuf *m0, int all)
385{
386	struct mbuf *m;
387
388	for (m = all ? m0 : m0->m_next; m != NULL; m = m->m_next) {
389		if (m->m_flags & M_PKTHDR) {
390			m_tag_delete_chain(m, NULL);
391			m->m_flags &= ~M_PKTHDR;
392			bzero(&m->m_pkthdr, sizeof(struct pkthdr));
393		}
394		if (m != m0 && m->m_nextpkt != NULL) {
395			KASSERT(m->m_nextpkt == NULL,
396			    ("%s: m_nextpkt not NULL", __func__));
397			m_freem(m->m_nextpkt);
398			m->m_nextpkt = NULL;
399		}
400		m->m_flags = m->m_flags & (M_EXT|M_RDONLY|M_NOFREE);
401	}
402}
403
404/*
405 * Sanity checks on mbuf (chain) for use in KASSERT() and general
406 * debugging.
407 * Returns 0 or panics when bad and 1 on all tests passed.
408 * Sanitize, 0 to run M_SANITY_ACTION, 1 to garble things so they
409 * blow up later.
410 */
411int
412m_sanity(struct mbuf *m0, int sanitize)
413{
414	struct mbuf *m;
415	caddr_t a, b;
416	int pktlen = 0;
417
418#ifdef INVARIANTS
419#define	M_SANITY_ACTION(s)	panic("mbuf %p: " s, m)
420#else
421#define	M_SANITY_ACTION(s)	printf("mbuf %p: " s, m)
422#endif
423
424	for (m = m0; m != NULL; m = m->m_next) {
425		/*
426		 * Basic pointer checks.  If any of these fails then some
427		 * unrelated kernel memory before or after us is trashed.
428		 * No way to recover from that.
429		 */
430		a = ((m->m_flags & M_EXT) ? m->m_ext.ext_buf :
431			((m->m_flags & M_PKTHDR) ? (caddr_t)(&m->m_pktdat) :
432			 (caddr_t)(&m->m_dat)) );
433		b = (caddr_t)(a + (m->m_flags & M_EXT ? m->m_ext.ext_size :
434			((m->m_flags & M_PKTHDR) ? MHLEN : MLEN)));
435		if ((caddr_t)m->m_data < a)
436			M_SANITY_ACTION("m_data outside mbuf data range left");
437		if ((caddr_t)m->m_data > b)
438			M_SANITY_ACTION("m_data outside mbuf data range right");
439		if ((caddr_t)m->m_data + m->m_len > b)
440			M_SANITY_ACTION("m_data + m_len exeeds mbuf space");
441
442		/* m->m_nextpkt may only be set on first mbuf in chain. */
443		if (m != m0 && m->m_nextpkt != NULL) {
444			if (sanitize) {
445				m_freem(m->m_nextpkt);
446				m->m_nextpkt = (struct mbuf *)0xDEADC0DE;
447			} else
448				M_SANITY_ACTION("m->m_nextpkt on in-chain mbuf");
449		}
450
451		/* packet length (not mbuf length!) calculation */
452		if (m0->m_flags & M_PKTHDR)
453			pktlen += m->m_len;
454
455		/* m_tags may only be attached to first mbuf in chain. */
456		if (m != m0 && m->m_flags & M_PKTHDR &&
457		    !SLIST_EMPTY(&m->m_pkthdr.tags)) {
458			if (sanitize) {
459				m_tag_delete_chain(m, NULL);
460				/* put in 0xDEADC0DE perhaps? */
461			} else
462				M_SANITY_ACTION("m_tags on in-chain mbuf");
463		}
464
465		/* M_PKTHDR may only be set on first mbuf in chain */
466		if (m != m0 && m->m_flags & M_PKTHDR) {
467			if (sanitize) {
468				bzero(&m->m_pkthdr, sizeof(m->m_pkthdr));
469				m->m_flags &= ~M_PKTHDR;
470				/* put in 0xDEADCODE and leave hdr flag in */
471			} else
472				M_SANITY_ACTION("M_PKTHDR on in-chain mbuf");
473		}
474	}
475	m = m0;
476	if (pktlen && pktlen != m->m_pkthdr.len) {
477		if (sanitize)
478			m->m_pkthdr.len = 0;
479		else
480			M_SANITY_ACTION("m_pkthdr.len != mbuf chain length");
481	}
482	return 1;
483
484#undef	M_SANITY_ACTION
485}
486
487
488/*
489 * "Move" mbuf pkthdr from "from" to "to".
490 * "from" must have M_PKTHDR set, and "to" must be empty.
491 */
492void
493m_move_pkthdr(struct mbuf *to, struct mbuf *from)
494{
495
496#if 0
497	/* see below for why these are not enabled */
498	M_ASSERTPKTHDR(to);
499	/* Note: with MAC, this may not be a good assertion. */
500	KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags),
501	    ("m_move_pkthdr: to has tags"));
502#endif
503#ifdef MAC
504	/*
505	 * XXXMAC: It could be this should also occur for non-MAC?
506	 */
507	if (to->m_flags & M_PKTHDR)
508		m_tag_delete_chain(to, NULL);
509#endif
510	to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
511	if ((to->m_flags & M_EXT) == 0)
512		to->m_data = to->m_pktdat;
513	to->m_pkthdr = from->m_pkthdr;		/* especially tags */
514	SLIST_INIT(&from->m_pkthdr.tags);	/* purge tags from src */
515	from->m_flags &= ~M_PKTHDR;
516}
517
518/*
519 * Duplicate "from"'s mbuf pkthdr in "to".
520 * "from" must have M_PKTHDR set, and "to" must be empty.
521 * In particular, this does a deep copy of the packet tags.
522 */
523int
524m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
525{
526
527#if 0
528	/*
529	 * The mbuf allocator only initializes the pkthdr
530	 * when the mbuf is allocated with m_gethdr(). Many users
531	 * (e.g. m_copy*, m_prepend) use m_get() and then
532	 * smash the pkthdr as needed causing these
533	 * assertions to trip.  For now just disable them.
534	 */
535	M_ASSERTPKTHDR(to);
536	/* Note: with MAC, this may not be a good assertion. */
537	KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), ("m_dup_pkthdr: to has tags"));
538#endif
539	MBUF_CHECKSLEEP(how);
540#ifdef MAC
541	if (to->m_flags & M_PKTHDR)
542		m_tag_delete_chain(to, NULL);
543#endif
544	to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
545	if ((to->m_flags & M_EXT) == 0)
546		to->m_data = to->m_pktdat;
547	to->m_pkthdr = from->m_pkthdr;
548	SLIST_INIT(&to->m_pkthdr.tags);
549	return (m_tag_copy_chain(to, from, MBTOM(how)));
550}
551
552/*
553 * Lesser-used path for M_PREPEND:
554 * allocate new mbuf to prepend to chain,
555 * copy junk along.
556 */
557struct mbuf *
558m_prepend(struct mbuf *m, int len, int how)
559{
560	struct mbuf *mn;
561
562	if (m->m_flags & M_PKTHDR)
563		mn = m_gethdr(how, m->m_type);
564	else
565		mn = m_get(how, m->m_type);
566	if (mn == NULL) {
567		m_freem(m);
568		return (NULL);
569	}
570	if (m->m_flags & M_PKTHDR)
571		m_move_pkthdr(mn, m);
572	mn->m_next = m;
573	m = mn;
574	if(m->m_flags & M_PKTHDR) {
575		if (len < MHLEN)
576			MH_ALIGN(m, len);
577	} else {
578		if (len < MLEN)
579			M_ALIGN(m, len);
580	}
581	m->m_len = len;
582	return (m);
583}
584
585/*
586 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
587 * continuing for "len" bytes.  If len is M_COPYALL, copy to end of mbuf.
588 * The wait parameter is a choice of M_WAITOK/M_NOWAIT from caller.
589 * Note that the copy is read-only, because clusters are not copied,
590 * only their reference counts are incremented.
591 */
592struct mbuf *
593m_copym(struct mbuf *m, int off0, int len, int wait)
594{
595	struct mbuf *n, **np;
596	int off = off0;
597	struct mbuf *top;
598	int copyhdr = 0;
599
600	KASSERT(off >= 0, ("m_copym, negative off %d", off));
601	KASSERT(len >= 0, ("m_copym, negative len %d", len));
602	MBUF_CHECKSLEEP(wait);
603	if (off == 0 && m->m_flags & M_PKTHDR)
604		copyhdr = 1;
605	while (off > 0) {
606		KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain"));
607		if (off < m->m_len)
608			break;
609		off -= m->m_len;
610		m = m->m_next;
611	}
612	np = &top;
613	top = 0;
614	while (len > 0) {
615		if (m == NULL) {
616			KASSERT(len == M_COPYALL,
617			    ("m_copym, length > size of mbuf chain"));
618			break;
619		}
620		if (copyhdr)
621			n = m_gethdr(wait, m->m_type);
622		else
623			n = m_get(wait, m->m_type);
624		*np = n;
625		if (n == NULL)
626			goto nospace;
627		if (copyhdr) {
628			if (!m_dup_pkthdr(n, m, wait))
629				goto nospace;
630			if (len == M_COPYALL)
631				n->m_pkthdr.len -= off0;
632			else
633				n->m_pkthdr.len = len;
634			copyhdr = 0;
635		}
636		n->m_len = min(len, m->m_len - off);
637		if (m->m_flags & M_EXT) {
638			n->m_data = m->m_data + off;
639			mb_dupcl(n, m);
640		} else
641			bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
642			    (u_int)n->m_len);
643		if (len != M_COPYALL)
644			len -= n->m_len;
645		off = 0;
646		m = m->m_next;
647		np = &n->m_next;
648	}
649
650	return (top);
651nospace:
652	m_freem(top);
653	return (NULL);
654}
655
656/*
657 * Returns mbuf chain with new head for the prepending case.
658 * Copies from mbuf (chain) n from off for len to mbuf (chain) m
659 * either prepending or appending the data.
660 * The resulting mbuf (chain) m is fully writeable.
661 * m is destination (is made writeable)
662 * n is source, off is offset in source, len is len from offset
663 * dir, 0 append, 1 prepend
664 * how, wait or nowait
665 */
666
667static int
668m_bcopyxxx(void *s, void *t, u_int len)
669{
670	bcopy(s, t, (size_t)len);
671	return 0;
672}
673
674struct mbuf *
675m_copymdata(struct mbuf *m, struct mbuf *n, int off, int len,
676    int prep, int how)
677{
678	struct mbuf *mm, *x, *z, *prev = NULL;
679	caddr_t p;
680	int i, nlen = 0;
681	caddr_t buf[MLEN];
682
683	KASSERT(m != NULL && n != NULL, ("m_copymdata, no target or source"));
684	KASSERT(off >= 0, ("m_copymdata, negative off %d", off));
685	KASSERT(len >= 0, ("m_copymdata, negative len %d", len));
686	KASSERT(prep == 0 || prep == 1, ("m_copymdata, unknown direction %d", prep));
687
688	mm = m;
689	if (!prep) {
690		while(mm->m_next) {
691			prev = mm;
692			mm = mm->m_next;
693		}
694	}
695	for (z = n; z != NULL; z = z->m_next)
696		nlen += z->m_len;
697	if (len == M_COPYALL)
698		len = nlen - off;
699	if (off + len > nlen || len < 1)
700		return NULL;
701
702	if (!M_WRITABLE(mm)) {
703		/* XXX: Use proper m_xxx function instead. */
704		x = m_getcl(how, MT_DATA, mm->m_flags);
705		if (x == NULL)
706			return NULL;
707		bcopy(mm->m_ext.ext_buf, x->m_ext.ext_buf, x->m_ext.ext_size);
708		p = x->m_ext.ext_buf + (mm->m_data - mm->m_ext.ext_buf);
709		x->m_data = p;
710		mm->m_next = NULL;
711		if (mm != m)
712			prev->m_next = x;
713		m_free(mm);
714		mm = x;
715	}
716
717	/*
718	 * Append/prepend the data.  Allocating mbufs as necessary.
719	 */
720	/* Shortcut if enough free space in first/last mbuf. */
721	if (!prep && M_TRAILINGSPACE(mm) >= len) {
722		m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t) +
723			 mm->m_len);
724		mm->m_len += len;
725		mm->m_pkthdr.len += len;
726		return m;
727	}
728	if (prep && M_LEADINGSPACE(mm) >= len) {
729		mm->m_data = mtod(mm, caddr_t) - len;
730		m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t));
731		mm->m_len += len;
732		mm->m_pkthdr.len += len;
733		return mm;
734	}
735
736	/* Expand first/last mbuf to cluster if possible. */
737	if (!prep && !(mm->m_flags & M_EXT) && len > M_TRAILINGSPACE(mm)) {
738		bcopy(mm->m_data, &buf, mm->m_len);
739		m_clget(mm, how);
740		if (!(mm->m_flags & M_EXT))
741			return NULL;
742		bcopy(&buf, mm->m_ext.ext_buf, mm->m_len);
743		mm->m_data = mm->m_ext.ext_buf;
744	}
745	if (prep && !(mm->m_flags & M_EXT) && len > M_LEADINGSPACE(mm)) {
746		bcopy(mm->m_data, &buf, mm->m_len);
747		m_clget(mm, how);
748		if (!(mm->m_flags & M_EXT))
749			return NULL;
750		bcopy(&buf, (caddr_t *)mm->m_ext.ext_buf +
751		       mm->m_ext.ext_size - mm->m_len, mm->m_len);
752		mm->m_data = (caddr_t)mm->m_ext.ext_buf +
753			      mm->m_ext.ext_size - mm->m_len;
754	}
755
756	/* Append/prepend as many mbuf (clusters) as necessary to fit len. */
757	if (!prep && len > M_TRAILINGSPACE(mm)) {
758		if (!m_getm(mm, len - M_TRAILINGSPACE(mm), how, MT_DATA))
759			return NULL;
760	}
761	if (prep && len > M_LEADINGSPACE(mm)) {
762		if (!(z = m_getm(NULL, len - M_LEADINGSPACE(mm), how, MT_DATA)))
763			return NULL;
764		i = 0;
765		for (x = z; x != NULL; x = x->m_next) {
766			i += x->m_flags & M_EXT ? x->m_ext.ext_size :
767			      (x->m_flags & M_PKTHDR ? MHLEN : MLEN);
768			if (!x->m_next)
769				break;
770		}
771		z->m_data += i - len;
772		m_move_pkthdr(mm, z);
773		x->m_next = mm;
774		mm = z;
775	}
776
777	/* Seek to start position in source mbuf. Optimization for long chains. */
778	while (off > 0) {
779		if (off < n->m_len)
780			break;
781		off -= n->m_len;
782		n = n->m_next;
783	}
784
785	/* Copy data into target mbuf. */
786	z = mm;
787	while (len > 0) {
788		KASSERT(z != NULL, ("m_copymdata, falling off target edge"));
789		i = M_TRAILINGSPACE(z);
790		m_apply(n, off, i, m_bcopyxxx, mtod(z, caddr_t) + z->m_len);
791		z->m_len += i;
792		/* fixup pkthdr.len if necessary */
793		if ((prep ? mm : m)->m_flags & M_PKTHDR)
794			(prep ? mm : m)->m_pkthdr.len += i;
795		off += i;
796		len -= i;
797		z = z->m_next;
798	}
799	return (prep ? mm : m);
800}
801
802/*
803 * Copy an entire packet, including header (which must be present).
804 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'.
805 * Note that the copy is read-only, because clusters are not copied,
806 * only their reference counts are incremented.
807 * Preserve alignment of the first mbuf so if the creator has left
808 * some room at the beginning (e.g. for inserting protocol headers)
809 * the copies still have the room available.
810 */
811struct mbuf *
812m_copypacket(struct mbuf *m, int how)
813{
814	struct mbuf *top, *n, *o;
815
816	MBUF_CHECKSLEEP(how);
817	n = m_get(how, m->m_type);
818	top = n;
819	if (n == NULL)
820		goto nospace;
821
822	if (!m_dup_pkthdr(n, m, how))
823		goto nospace;
824	n->m_len = m->m_len;
825	if (m->m_flags & M_EXT) {
826		n->m_data = m->m_data;
827		mb_dupcl(n, m);
828	} else {
829		n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat );
830		bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
831	}
832
833	m = m->m_next;
834	while (m) {
835		o = m_get(how, m->m_type);
836		if (o == NULL)
837			goto nospace;
838
839		n->m_next = o;
840		n = n->m_next;
841
842		n->m_len = m->m_len;
843		if (m->m_flags & M_EXT) {
844			n->m_data = m->m_data;
845			mb_dupcl(n, m);
846		} else {
847			bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
848		}
849
850		m = m->m_next;
851	}
852	return top;
853nospace:
854	m_freem(top);
855	return (NULL);
856}
857
858/*
859 * Copy data from an mbuf chain starting "off" bytes from the beginning,
860 * continuing for "len" bytes, into the indicated buffer.
861 */
862void
863m_copydata(const struct mbuf *m, int off, int len, caddr_t cp)
864{
865	u_int count;
866
867	KASSERT(off >= 0, ("m_copydata, negative off %d", off));
868	KASSERT(len >= 0, ("m_copydata, negative len %d", len));
869	while (off > 0) {
870		KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain"));
871		if (off < m->m_len)
872			break;
873		off -= m->m_len;
874		m = m->m_next;
875	}
876	while (len > 0) {
877		KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain"));
878		count = min(m->m_len - off, len);
879		bcopy(mtod(m, caddr_t) + off, cp, count);
880		len -= count;
881		cp += count;
882		off = 0;
883		m = m->m_next;
884	}
885}
886
887/*
888 * Copy a packet header mbuf chain into a completely new chain, including
889 * copying any mbuf clusters.  Use this instead of m_copypacket() when
890 * you need a writable copy of an mbuf chain.
891 */
892struct mbuf *
893m_dup(struct mbuf *m, int how)
894{
895	struct mbuf **p, *top = NULL;
896	int remain, moff, nsize;
897
898	MBUF_CHECKSLEEP(how);
899	/* Sanity check */
900	if (m == NULL)
901		return (NULL);
902	M_ASSERTPKTHDR(m);
903
904	/* While there's more data, get a new mbuf, tack it on, and fill it */
905	remain = m->m_pkthdr.len;
906	moff = 0;
907	p = &top;
908	while (remain > 0 || top == NULL) {	/* allow m->m_pkthdr.len == 0 */
909		struct mbuf *n;
910
911		/* Get the next new mbuf */
912		if (remain >= MINCLSIZE) {
913			n = m_getcl(how, m->m_type, 0);
914			nsize = MCLBYTES;
915		} else {
916			n = m_get(how, m->m_type);
917			nsize = MLEN;
918		}
919		if (n == NULL)
920			goto nospace;
921
922		if (top == NULL) {		/* First one, must be PKTHDR */
923			if (!m_dup_pkthdr(n, m, how)) {
924				m_free(n);
925				goto nospace;
926			}
927			if ((n->m_flags & M_EXT) == 0)
928				nsize = MHLEN;
929		}
930		n->m_len = 0;
931
932		/* Link it into the new chain */
933		*p = n;
934		p = &n->m_next;
935
936		/* Copy data from original mbuf(s) into new mbuf */
937		while (n->m_len < nsize && m != NULL) {
938			int chunk = min(nsize - n->m_len, m->m_len - moff);
939
940			bcopy(m->m_data + moff, n->m_data + n->m_len, chunk);
941			moff += chunk;
942			n->m_len += chunk;
943			remain -= chunk;
944			if (moff == m->m_len) {
945				m = m->m_next;
946				moff = 0;
947			}
948		}
949
950		/* Check correct total mbuf length */
951		KASSERT((remain > 0 && m != NULL) || (remain == 0 && m == NULL),
952		    	("%s: bogus m_pkthdr.len", __func__));
953	}
954	return (top);
955
956nospace:
957	m_freem(top);
958	return (NULL);
959}
960
961/*
962 * Concatenate mbuf chain n to m.
963 * Both chains must be of the same type (e.g. MT_DATA).
964 * Any m_pkthdr is not updated.
965 */
966void
967m_cat(struct mbuf *m, struct mbuf *n)
968{
969	while (m->m_next)
970		m = m->m_next;
971	while (n) {
972		if (!M_WRITABLE(m) ||
973		    M_TRAILINGSPACE(m) < n->m_len) {
974			/* just join the two chains */
975			m->m_next = n;
976			return;
977		}
978		/* splat the data from one into the other */
979		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
980		    (u_int)n->m_len);
981		m->m_len += n->m_len;
982		n = m_free(n);
983	}
984}
985
986void
987m_adj(struct mbuf *mp, int req_len)
988{
989	int len = req_len;
990	struct mbuf *m;
991	int count;
992
993	if ((m = mp) == NULL)
994		return;
995	if (len >= 0) {
996		/*
997		 * Trim from head.
998		 */
999		while (m != NULL && len > 0) {
1000			if (m->m_len <= len) {
1001				len -= m->m_len;
1002				m->m_len = 0;
1003				m = m->m_next;
1004			} else {
1005				m->m_len -= len;
1006				m->m_data += len;
1007				len = 0;
1008			}
1009		}
1010		if (mp->m_flags & M_PKTHDR)
1011			mp->m_pkthdr.len -= (req_len - len);
1012	} else {
1013		/*
1014		 * Trim from tail.  Scan the mbuf chain,
1015		 * calculating its length and finding the last mbuf.
1016		 * If the adjustment only affects this mbuf, then just
1017		 * adjust and return.  Otherwise, rescan and truncate
1018		 * after the remaining size.
1019		 */
1020		len = -len;
1021		count = 0;
1022		for (;;) {
1023			count += m->m_len;
1024			if (m->m_next == (struct mbuf *)0)
1025				break;
1026			m = m->m_next;
1027		}
1028		if (m->m_len >= len) {
1029			m->m_len -= len;
1030			if (mp->m_flags & M_PKTHDR)
1031				mp->m_pkthdr.len -= len;
1032			return;
1033		}
1034		count -= len;
1035		if (count < 0)
1036			count = 0;
1037		/*
1038		 * Correct length for chain is "count".
1039		 * Find the mbuf with last data, adjust its length,
1040		 * and toss data from remaining mbufs on chain.
1041		 */
1042		m = mp;
1043		if (m->m_flags & M_PKTHDR)
1044			m->m_pkthdr.len = count;
1045		for (; m; m = m->m_next) {
1046			if (m->m_len >= count) {
1047				m->m_len = count;
1048				if (m->m_next != NULL) {
1049					m_freem(m->m_next);
1050					m->m_next = NULL;
1051				}
1052				break;
1053			}
1054			count -= m->m_len;
1055		}
1056	}
1057}
1058
1059/*
1060 * Rearange an mbuf chain so that len bytes are contiguous
1061 * and in the data area of an mbuf (so that mtod will work
1062 * for a structure of size len).  Returns the resulting
1063 * mbuf chain on success, frees it and returns null on failure.
1064 * If there is room, it will add up to max_protohdr-len extra bytes to the
1065 * contiguous region in an attempt to avoid being called next time.
1066 */
1067struct mbuf *
1068m_pullup(struct mbuf *n, int len)
1069{
1070	struct mbuf *m;
1071	int count;
1072	int space;
1073
1074	/*
1075	 * If first mbuf has no cluster, and has room for len bytes
1076	 * without shifting current data, pullup into it,
1077	 * otherwise allocate a new mbuf to prepend to the chain.
1078	 */
1079	if ((n->m_flags & M_EXT) == 0 &&
1080	    n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
1081		if (n->m_len >= len)
1082			return (n);
1083		m = n;
1084		n = n->m_next;
1085		len -= m->m_len;
1086	} else {
1087		if (len > MHLEN)
1088			goto bad;
1089		m = m_get(M_NOWAIT, n->m_type);
1090		if (m == NULL)
1091			goto bad;
1092		if (n->m_flags & M_PKTHDR)
1093			m_move_pkthdr(m, n);
1094	}
1095	space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
1096	do {
1097		count = min(min(max(len, max_protohdr), space), n->m_len);
1098		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1099		  (u_int)count);
1100		len -= count;
1101		m->m_len += count;
1102		n->m_len -= count;
1103		space -= count;
1104		if (n->m_len)
1105			n->m_data += count;
1106		else
1107			n = m_free(n);
1108	} while (len > 0 && n);
1109	if (len > 0) {
1110		(void) m_free(m);
1111		goto bad;
1112	}
1113	m->m_next = n;
1114	return (m);
1115bad:
1116	m_freem(n);
1117	return (NULL);
1118}
1119
1120/*
1121 * Like m_pullup(), except a new mbuf is always allocated, and we allow
1122 * the amount of empty space before the data in the new mbuf to be specified
1123 * (in the event that the caller expects to prepend later).
1124 */
1125int MSFail;
1126
1127struct mbuf *
1128m_copyup(struct mbuf *n, int len, int dstoff)
1129{
1130	struct mbuf *m;
1131	int count, space;
1132
1133	if (len > (MHLEN - dstoff))
1134		goto bad;
1135	m = m_get(M_NOWAIT, n->m_type);
1136	if (m == NULL)
1137		goto bad;
1138	if (n->m_flags & M_PKTHDR)
1139		m_move_pkthdr(m, n);
1140	m->m_data += dstoff;
1141	space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
1142	do {
1143		count = min(min(max(len, max_protohdr), space), n->m_len);
1144		memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t),
1145		    (unsigned)count);
1146		len -= count;
1147		m->m_len += count;
1148		n->m_len -= count;
1149		space -= count;
1150		if (n->m_len)
1151			n->m_data += count;
1152		else
1153			n = m_free(n);
1154	} while (len > 0 && n);
1155	if (len > 0) {
1156		(void) m_free(m);
1157		goto bad;
1158	}
1159	m->m_next = n;
1160	return (m);
1161 bad:
1162	m_freem(n);
1163	MSFail++;
1164	return (NULL);
1165}
1166
1167/*
1168 * Partition an mbuf chain in two pieces, returning the tail --
1169 * all but the first len0 bytes.  In case of failure, it returns NULL and
1170 * attempts to restore the chain to its original state.
1171 *
1172 * Note that the resulting mbufs might be read-only, because the new
1173 * mbuf can end up sharing an mbuf cluster with the original mbuf if
1174 * the "breaking point" happens to lie within a cluster mbuf. Use the
1175 * M_WRITABLE() macro to check for this case.
1176 */
1177struct mbuf *
1178m_split(struct mbuf *m0, int len0, int wait)
1179{
1180	struct mbuf *m, *n;
1181	u_int len = len0, remain;
1182
1183	MBUF_CHECKSLEEP(wait);
1184	for (m = m0; m && len > m->m_len; m = m->m_next)
1185		len -= m->m_len;
1186	if (m == NULL)
1187		return (NULL);
1188	remain = m->m_len - len;
1189	if (m0->m_flags & M_PKTHDR && remain == 0) {
1190		n = m_gethdr(wait, m0->m_type);
1191			return (NULL);
1192		n->m_next = m->m_next;
1193		m->m_next = NULL;
1194		n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1195		n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1196		m0->m_pkthdr.len = len0;
1197		return (n);
1198	} else if (m0->m_flags & M_PKTHDR) {
1199		n = m_gethdr(wait, m0->m_type);
1200		if (n == NULL)
1201			return (NULL);
1202		n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1203		n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1204		m0->m_pkthdr.len = len0;
1205		if (m->m_flags & M_EXT)
1206			goto extpacket;
1207		if (remain > MHLEN) {
1208			/* m can't be the lead packet */
1209			MH_ALIGN(n, 0);
1210			n->m_next = m_split(m, len, wait);
1211			if (n->m_next == NULL) {
1212				(void) m_free(n);
1213				return (NULL);
1214			} else {
1215				n->m_len = 0;
1216				return (n);
1217			}
1218		} else
1219			MH_ALIGN(n, remain);
1220	} else if (remain == 0) {
1221		n = m->m_next;
1222		m->m_next = NULL;
1223		return (n);
1224	} else {
1225		n = m_get(wait, m->m_type);
1226		if (n == NULL)
1227			return (NULL);
1228		M_ALIGN(n, remain);
1229	}
1230extpacket:
1231	if (m->m_flags & M_EXT) {
1232		n->m_data = m->m_data + len;
1233		mb_dupcl(n, m);
1234	} else {
1235		bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
1236	}
1237	n->m_len = remain;
1238	m->m_len = len;
1239	n->m_next = m->m_next;
1240	m->m_next = NULL;
1241	return (n);
1242}
1243/*
1244 * Routine to copy from device local memory into mbufs.
1245 * Note that `off' argument is offset into first mbuf of target chain from
1246 * which to begin copying the data to.
1247 */
1248struct mbuf *
1249m_devget(char *buf, int totlen, int off, struct ifnet *ifp,
1250    void (*copy)(char *from, caddr_t to, u_int len))
1251{
1252	struct mbuf *m;
1253	struct mbuf *top = NULL, **mp = &top;
1254	int len;
1255
1256	if (off < 0 || off > MHLEN)
1257		return (NULL);
1258
1259	while (totlen > 0) {
1260		if (top == NULL) {	/* First one, must be PKTHDR */
1261			if (totlen + off >= MINCLSIZE) {
1262				m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
1263				len = MCLBYTES;
1264			} else {
1265				m = m_gethdr(M_NOWAIT, MT_DATA);
1266				len = MHLEN;
1267
1268				/* Place initial small packet/header at end of mbuf */
1269				if (m && totlen + off + max_linkhdr <= MLEN) {
1270					m->m_data += max_linkhdr;
1271					len -= max_linkhdr;
1272				}
1273			}
1274			if (m == NULL)
1275				return NULL;
1276			m->m_pkthdr.rcvif = ifp;
1277			m->m_pkthdr.len = totlen;
1278		} else {
1279			if (totlen + off >= MINCLSIZE) {
1280				m = m_getcl(M_NOWAIT, MT_DATA, 0);
1281				len = MCLBYTES;
1282			} else {
1283				m = m_get(M_NOWAIT, MT_DATA);
1284				len = MLEN;
1285			}
1286			if (m == NULL) {
1287				m_freem(top);
1288				return NULL;
1289			}
1290		}
1291		if (off) {
1292			m->m_data += off;
1293			len -= off;
1294			off = 0;
1295		}
1296		m->m_len = len = min(totlen, len);
1297		if (copy)
1298			copy(buf, mtod(m, caddr_t), (u_int)len);
1299		else
1300			bcopy(buf, mtod(m, caddr_t), (u_int)len);
1301		buf += len;
1302		*mp = m;
1303		mp = &m->m_next;
1304		totlen -= len;
1305	}
1306	return (top);
1307}
1308
1309/*
1310 * Copy data from a buffer back into the indicated mbuf chain,
1311 * starting "off" bytes from the beginning, extending the mbuf
1312 * chain if necessary.
1313 */
1314void
1315m_copyback(struct mbuf *m0, int off, int len, c_caddr_t cp)
1316{
1317	int mlen;
1318	struct mbuf *m = m0, *n;
1319	int totlen = 0;
1320
1321	if (m0 == NULL)
1322		return;
1323	while (off > (mlen = m->m_len)) {
1324		off -= mlen;
1325		totlen += mlen;
1326		if (m->m_next == NULL) {
1327			n = m_get(M_NOWAIT, m->m_type);
1328			if (n == NULL)
1329				goto out;
1330			bzero(mtod(n, caddr_t), MLEN);
1331			n->m_len = min(MLEN, len + off);
1332			m->m_next = n;
1333		}
1334		m = m->m_next;
1335	}
1336	while (len > 0) {
1337		if (m->m_next == NULL && (len > m->m_len - off)) {
1338			m->m_len += min(len - (m->m_len - off),
1339			    M_TRAILINGSPACE(m));
1340		}
1341		mlen = min (m->m_len - off, len);
1342		bcopy(cp, off + mtod(m, caddr_t), (u_int)mlen);
1343		cp += mlen;
1344		len -= mlen;
1345		mlen += off;
1346		off = 0;
1347		totlen += mlen;
1348		if (len == 0)
1349			break;
1350		if (m->m_next == NULL) {
1351			n = m_get(M_NOWAIT, m->m_type);
1352			if (n == NULL)
1353				break;
1354			n->m_len = min(MLEN, len);
1355			m->m_next = n;
1356		}
1357		m = m->m_next;
1358	}
1359out:	if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
1360		m->m_pkthdr.len = totlen;
1361}
1362
1363/*
1364 * Append the specified data to the indicated mbuf chain,
1365 * Extend the mbuf chain if the new data does not fit in
1366 * existing space.
1367 *
1368 * Return 1 if able to complete the job; otherwise 0.
1369 */
1370int
1371m_append(struct mbuf *m0, int len, c_caddr_t cp)
1372{
1373	struct mbuf *m, *n;
1374	int remainder, space;
1375
1376	for (m = m0; m->m_next != NULL; m = m->m_next)
1377		;
1378	remainder = len;
1379	space = M_TRAILINGSPACE(m);
1380	if (space > 0) {
1381		/*
1382		 * Copy into available space.
1383		 */
1384		if (space > remainder)
1385			space = remainder;
1386		bcopy(cp, mtod(m, caddr_t) + m->m_len, space);
1387		m->m_len += space;
1388		cp += space, remainder -= space;
1389	}
1390	while (remainder > 0) {
1391		/*
1392		 * Allocate a new mbuf; could check space
1393		 * and allocate a cluster instead.
1394		 */
1395		n = m_get(M_NOWAIT, m->m_type);
1396		if (n == NULL)
1397			break;
1398		n->m_len = min(MLEN, remainder);
1399		bcopy(cp, mtod(n, caddr_t), n->m_len);
1400		cp += n->m_len, remainder -= n->m_len;
1401		m->m_next = n;
1402		m = n;
1403	}
1404	if (m0->m_flags & M_PKTHDR)
1405		m0->m_pkthdr.len += len - remainder;
1406	return (remainder == 0);
1407}
1408
1409/*
1410 * Apply function f to the data in an mbuf chain starting "off" bytes from
1411 * the beginning, continuing for "len" bytes.
1412 */
1413int
1414m_apply(struct mbuf *m, int off, int len,
1415    int (*f)(void *, void *, u_int), void *arg)
1416{
1417	u_int count;
1418	int rval;
1419
1420	KASSERT(off >= 0, ("m_apply, negative off %d", off));
1421	KASSERT(len >= 0, ("m_apply, negative len %d", len));
1422	while (off > 0) {
1423		KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain"));
1424		if (off < m->m_len)
1425			break;
1426		off -= m->m_len;
1427		m = m->m_next;
1428	}
1429	while (len > 0) {
1430		KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain"));
1431		count = min(m->m_len - off, len);
1432		rval = (*f)(arg, mtod(m, caddr_t) + off, count);
1433		if (rval)
1434			return (rval);
1435		len -= count;
1436		off = 0;
1437		m = m->m_next;
1438	}
1439	return (0);
1440}
1441
1442/*
1443 * Return a pointer to mbuf/offset of location in mbuf chain.
1444 */
1445struct mbuf *
1446m_getptr(struct mbuf *m, int loc, int *off)
1447{
1448
1449	while (loc >= 0) {
1450		/* Normal end of search. */
1451		if (m->m_len > loc) {
1452			*off = loc;
1453			return (m);
1454		} else {
1455			loc -= m->m_len;
1456			if (m->m_next == NULL) {
1457				if (loc == 0) {
1458					/* Point at the end of valid data. */
1459					*off = m->m_len;
1460					return (m);
1461				}
1462				return (NULL);
1463			}
1464			m = m->m_next;
1465		}
1466	}
1467	return (NULL);
1468}
1469
1470void
1471m_print(const struct mbuf *m, int maxlen)
1472{
1473	int len;
1474	int pdata;
1475	const struct mbuf *m2;
1476
1477	if (m == NULL) {
1478		printf("mbuf: %p\n", m);
1479		return;
1480	}
1481
1482	if (m->m_flags & M_PKTHDR)
1483		len = m->m_pkthdr.len;
1484	else
1485		len = -1;
1486	m2 = m;
1487	while (m2 != NULL && (len == -1 || len)) {
1488		pdata = m2->m_len;
1489		if (maxlen != -1 && pdata > maxlen)
1490			pdata = maxlen;
1491		printf("mbuf: %p len: %d, next: %p, %b%s", m2, m2->m_len,
1492		    m2->m_next, m2->m_flags, "\20\20freelist\17skipfw"
1493		    "\11proto5\10proto4\7proto3\6proto2\5proto1\4rdonly"
1494		    "\3eor\2pkthdr\1ext", pdata ? "" : "\n");
1495		if (pdata)
1496			printf(", %*D\n", pdata, (u_char *)m2->m_data, "-");
1497		if (len != -1)
1498			len -= m2->m_len;
1499		m2 = m2->m_next;
1500	}
1501	if (len > 0)
1502		printf("%d bytes unaccounted for.\n", len);
1503	return;
1504}
1505
1506u_int
1507m_fixhdr(struct mbuf *m0)
1508{
1509	u_int len;
1510
1511	len = m_length(m0, NULL);
1512	m0->m_pkthdr.len = len;
1513	return (len);
1514}
1515
1516u_int
1517m_length(struct mbuf *m0, struct mbuf **last)
1518{
1519	struct mbuf *m;
1520	u_int len;
1521
1522	len = 0;
1523	for (m = m0; m != NULL; m = m->m_next) {
1524		len += m->m_len;
1525		if (m->m_next == NULL)
1526			break;
1527	}
1528	if (last != NULL)
1529		*last = m;
1530	return (len);
1531}
1532
1533/*
1534 * Defragment a mbuf chain, returning the shortest possible
1535 * chain of mbufs and clusters.  If allocation fails and
1536 * this cannot be completed, NULL will be returned, but
1537 * the passed in chain will be unchanged.  Upon success,
1538 * the original chain will be freed, and the new chain
1539 * will be returned.
1540 *
1541 * If a non-packet header is passed in, the original
1542 * mbuf (chain?) will be returned unharmed.
1543 */
1544struct mbuf *
1545m_defrag(struct mbuf *m0, int how)
1546{
1547	struct mbuf *m_new = NULL, *m_final = NULL;
1548	int progress = 0, length;
1549
1550	MBUF_CHECKSLEEP(how);
1551	if (!(m0->m_flags & M_PKTHDR))
1552		return (m0);
1553
1554	m_fixhdr(m0); /* Needed sanity check */
1555
1556#ifdef MBUF_STRESS_TEST
1557	if (m_defragrandomfailures) {
1558		int temp = arc4random() & 0xff;
1559		if (temp == 0xba)
1560			goto nospace;
1561	}
1562#endif
1563
1564	if (m0->m_pkthdr.len > MHLEN)
1565		m_final = m_getcl(how, MT_DATA, M_PKTHDR);
1566	else
1567		m_final = m_gethdr(how, MT_DATA);
1568
1569	if (m_final == NULL)
1570		goto nospace;
1571
1572	if (m_dup_pkthdr(m_final, m0, how) == 0)
1573		goto nospace;
1574
1575	m_new = m_final;
1576
1577	while (progress < m0->m_pkthdr.len) {
1578		length = m0->m_pkthdr.len - progress;
1579		if (length > MCLBYTES)
1580			length = MCLBYTES;
1581
1582		if (m_new == NULL) {
1583			if (length > MLEN)
1584				m_new = m_getcl(how, MT_DATA, 0);
1585			else
1586				m_new = m_get(how, MT_DATA);
1587			if (m_new == NULL)
1588				goto nospace;
1589		}
1590
1591		m_copydata(m0, progress, length, mtod(m_new, caddr_t));
1592		progress += length;
1593		m_new->m_len = length;
1594		if (m_new != m_final)
1595			m_cat(m_final, m_new);
1596		m_new = NULL;
1597	}
1598#ifdef MBUF_STRESS_TEST
1599	if (m0->m_next == NULL)
1600		m_defraguseless++;
1601#endif
1602	m_freem(m0);
1603	m0 = m_final;
1604#ifdef MBUF_STRESS_TEST
1605	m_defragpackets++;
1606	m_defragbytes += m0->m_pkthdr.len;
1607#endif
1608	return (m0);
1609nospace:
1610#ifdef MBUF_STRESS_TEST
1611	m_defragfailure++;
1612#endif
1613	if (m_final)
1614		m_freem(m_final);
1615	return (NULL);
1616}
1617
1618/*
1619 * Defragment an mbuf chain, returning at most maxfrags separate
1620 * mbufs+clusters.  If this is not possible NULL is returned and
1621 * the original mbuf chain is left in it's present (potentially
1622 * modified) state.  We use two techniques: collapsing consecutive
1623 * mbufs and replacing consecutive mbufs by a cluster.
1624 *
1625 * NB: this should really be named m_defrag but that name is taken
1626 */
1627struct mbuf *
1628m_collapse(struct mbuf *m0, int how, int maxfrags)
1629{
1630	struct mbuf *m, *n, *n2, **prev;
1631	u_int curfrags;
1632
1633	/*
1634	 * Calculate the current number of frags.
1635	 */
1636	curfrags = 0;
1637	for (m = m0; m != NULL; m = m->m_next)
1638		curfrags++;
1639	/*
1640	 * First, try to collapse mbufs.  Note that we always collapse
1641	 * towards the front so we don't need to deal with moving the
1642	 * pkthdr.  This may be suboptimal if the first mbuf has much
1643	 * less data than the following.
1644	 */
1645	m = m0;
1646again:
1647	for (;;) {
1648		n = m->m_next;
1649		if (n == NULL)
1650			break;
1651		if (M_WRITABLE(m) &&
1652		    n->m_len < M_TRAILINGSPACE(m)) {
1653			bcopy(mtod(n, void *), mtod(m, char *) + m->m_len,
1654				n->m_len);
1655			m->m_len += n->m_len;
1656			m->m_next = n->m_next;
1657			m_free(n);
1658			if (--curfrags <= maxfrags)
1659				return m0;
1660		} else
1661			m = n;
1662	}
1663	KASSERT(maxfrags > 1,
1664		("maxfrags %u, but normal collapse failed", maxfrags));
1665	/*
1666	 * Collapse consecutive mbufs to a cluster.
1667	 */
1668	prev = &m0->m_next;		/* NB: not the first mbuf */
1669	while ((n = *prev) != NULL) {
1670		if ((n2 = n->m_next) != NULL &&
1671		    n->m_len + n2->m_len < MCLBYTES) {
1672			m = m_getcl(how, MT_DATA, 0);
1673			if (m == NULL)
1674				goto bad;
1675			bcopy(mtod(n, void *), mtod(m, void *), n->m_len);
1676			bcopy(mtod(n2, void *), mtod(m, char *) + n->m_len,
1677				n2->m_len);
1678			m->m_len = n->m_len + n2->m_len;
1679			m->m_next = n2->m_next;
1680			*prev = m;
1681			m_free(n);
1682			m_free(n2);
1683			if (--curfrags <= maxfrags)	/* +1 cl -2 mbufs */
1684				return m0;
1685			/*
1686			 * Still not there, try the normal collapse
1687			 * again before we allocate another cluster.
1688			 */
1689			goto again;
1690		}
1691		prev = &n->m_next;
1692	}
1693	/*
1694	 * No place where we can collapse to a cluster; punt.
1695	 * This can occur if, for example, you request 2 frags
1696	 * but the packet requires that both be clusters (we
1697	 * never reallocate the first mbuf to avoid moving the
1698	 * packet header).
1699	 */
1700bad:
1701	return NULL;
1702}
1703
1704#ifdef MBUF_STRESS_TEST
1705
1706/*
1707 * Fragment an mbuf chain.  There's no reason you'd ever want to do
1708 * this in normal usage, but it's great for stress testing various
1709 * mbuf consumers.
1710 *
1711 * If fragmentation is not possible, the original chain will be
1712 * returned.
1713 *
1714 * Possible length values:
1715 * 0	 no fragmentation will occur
1716 * > 0	each fragment will be of the specified length
1717 * -1	each fragment will be the same random value in length
1718 * -2	each fragment's length will be entirely random
1719 * (Random values range from 1 to 256)
1720 */
1721struct mbuf *
1722m_fragment(struct mbuf *m0, int how, int length)
1723{
1724	struct mbuf *m_new = NULL, *m_final = NULL;
1725	int progress = 0;
1726
1727	if (!(m0->m_flags & M_PKTHDR))
1728		return (m0);
1729
1730	if ((length == 0) || (length < -2))
1731		return (m0);
1732
1733	m_fixhdr(m0); /* Needed sanity check */
1734
1735	m_final = m_getcl(how, MT_DATA, M_PKTHDR);
1736
1737	if (m_final == NULL)
1738		goto nospace;
1739
1740	if (m_dup_pkthdr(m_final, m0, how) == 0)
1741		goto nospace;
1742
1743	m_new = m_final;
1744
1745	if (length == -1)
1746		length = 1 + (arc4random() & 255);
1747
1748	while (progress < m0->m_pkthdr.len) {
1749		int fraglen;
1750
1751		if (length > 0)
1752			fraglen = length;
1753		else
1754			fraglen = 1 + (arc4random() & 255);
1755		if (fraglen > m0->m_pkthdr.len - progress)
1756			fraglen = m0->m_pkthdr.len - progress;
1757
1758		if (fraglen > MCLBYTES)
1759			fraglen = MCLBYTES;
1760
1761		if (m_new == NULL) {
1762			m_new = m_getcl(how, MT_DATA, 0);
1763			if (m_new == NULL)
1764				goto nospace;
1765		}
1766
1767		m_copydata(m0, progress, fraglen, mtod(m_new, caddr_t));
1768		progress += fraglen;
1769		m_new->m_len = fraglen;
1770		if (m_new != m_final)
1771			m_cat(m_final, m_new);
1772		m_new = NULL;
1773	}
1774	m_freem(m0);
1775	m0 = m_final;
1776	return (m0);
1777nospace:
1778	if (m_final)
1779		m_freem(m_final);
1780	/* Return the original chain on failure */
1781	return (m0);
1782}
1783
1784#endif
1785
1786/*
1787 * Copy the contents of uio into a properly sized mbuf chain.
1788 */
1789struct mbuf *
1790m_uiotombuf(struct uio *uio, int how, int len, int align, int flags)
1791{
1792	struct mbuf *m, *mb;
1793	int error, length;
1794	ssize_t total;
1795	int progress = 0;
1796
1797	/*
1798	 * len can be zero or an arbitrary large value bound by
1799	 * the total data supplied by the uio.
1800	 */
1801	if (len > 0)
1802		total = min(uio->uio_resid, len);
1803	else
1804		total = uio->uio_resid;
1805
1806	/*
1807	 * The smallest unit returned by m_getm2() is a single mbuf
1808	 * with pkthdr.  We can't align past it.
1809	 */
1810	if (align >= MHLEN)
1811		return (NULL);
1812
1813	/*
1814	 * Give us the full allocation or nothing.
1815	 * If len is zero return the smallest empty mbuf.
1816	 */
1817	m = m_getm2(NULL, max(total + align, 1), how, MT_DATA, flags);
1818	if (m == NULL)
1819		return (NULL);
1820	m->m_data += align;
1821
1822	/* Fill all mbufs with uio data and update header information. */
1823	for (mb = m; mb != NULL; mb = mb->m_next) {
1824		length = min(M_TRAILINGSPACE(mb), total - progress);
1825
1826		error = uiomove(mtod(mb, void *), length, uio);
1827		if (error) {
1828			m_freem(m);
1829			return (NULL);
1830		}
1831
1832		mb->m_len = length;
1833		progress += length;
1834		if (flags & M_PKTHDR)
1835			m->m_pkthdr.len += length;
1836	}
1837	KASSERT(progress == total, ("%s: progress != total", __func__));
1838
1839	return (m);
1840}
1841
1842/*
1843 * Copy an mbuf chain into a uio limited by len if set.
1844 */
1845int
1846m_mbuftouio(struct uio *uio, struct mbuf *m, int len)
1847{
1848	int error, length, total;
1849	int progress = 0;
1850
1851	if (len > 0)
1852		total = min(uio->uio_resid, len);
1853	else
1854		total = uio->uio_resid;
1855
1856	/* Fill the uio with data from the mbufs. */
1857	for (; m != NULL; m = m->m_next) {
1858		length = min(m->m_len, total - progress);
1859
1860		error = uiomove(mtod(m, void *), length, uio);
1861		if (error)
1862			return (error);
1863
1864		progress += length;
1865	}
1866
1867	return (0);
1868}
1869
1870/*
1871 * Set the m_data pointer of a newly-allocated mbuf
1872 * to place an object of the specified size at the
1873 * end of the mbuf, longword aligned.
1874 */
1875void
1876m_align(struct mbuf *m, int len)
1877{
1878#ifdef INVARIANTS
1879	const char *msg = "%s: not a virgin mbuf";
1880#endif
1881	int adjust;
1882
1883	if (m->m_flags & M_EXT) {
1884		KASSERT(m->m_data == m->m_ext.ext_buf, (msg, __func__));
1885		adjust = m->m_ext.ext_size - len;
1886	} else if (m->m_flags & M_PKTHDR) {
1887		KASSERT(m->m_data == m->m_pktdat, (msg, __func__));
1888		adjust = MHLEN - len;
1889	} else {
1890		KASSERT(m->m_data == m->m_dat, (msg, __func__));
1891		adjust = MLEN - len;
1892	}
1893
1894	m->m_data += adjust &~ (sizeof(long)-1);
1895}
1896
1897/*
1898 * Create a writable copy of the mbuf chain.  While doing this
1899 * we compact the chain with a goal of producing a chain with
1900 * at most two mbufs.  The second mbuf in this chain is likely
1901 * to be a cluster.  The primary purpose of this work is to create
1902 * a writable packet for encryption, compression, etc.  The
1903 * secondary goal is to linearize the data so the data can be
1904 * passed to crypto hardware in the most efficient manner possible.
1905 */
1906struct mbuf *
1907m_unshare(struct mbuf *m0, int how)
1908{
1909	struct mbuf *m, *mprev;
1910	struct mbuf *n, *mfirst, *mlast;
1911	int len, off;
1912
1913	mprev = NULL;
1914	for (m = m0; m != NULL; m = mprev->m_next) {
1915		/*
1916		 * Regular mbufs are ignored unless there's a cluster
1917		 * in front of it that we can use to coalesce.  We do
1918		 * the latter mainly so later clusters can be coalesced
1919		 * also w/o having to handle them specially (i.e. convert
1920		 * mbuf+cluster -> cluster).  This optimization is heavily
1921		 * influenced by the assumption that we're running over
1922		 * Ethernet where MCLBYTES is large enough that the max
1923		 * packet size will permit lots of coalescing into a
1924		 * single cluster.  This in turn permits efficient
1925		 * crypto operations, especially when using hardware.
1926		 */
1927		if ((m->m_flags & M_EXT) == 0) {
1928			if (mprev && (mprev->m_flags & M_EXT) &&
1929			    m->m_len <= M_TRAILINGSPACE(mprev)) {
1930				/* XXX: this ignores mbuf types */
1931				memcpy(mtod(mprev, caddr_t) + mprev->m_len,
1932				       mtod(m, caddr_t), m->m_len);
1933				mprev->m_len += m->m_len;
1934				mprev->m_next = m->m_next;	/* unlink from chain */
1935				m_free(m);			/* reclaim mbuf */
1936#if 0
1937				newipsecstat.ips_mbcoalesced++;
1938#endif
1939			} else {
1940				mprev = m;
1941			}
1942			continue;
1943		}
1944		/*
1945		 * Writable mbufs are left alone (for now).
1946		 */
1947		if (M_WRITABLE(m)) {
1948			mprev = m;
1949			continue;
1950		}
1951
1952		/*
1953		 * Not writable, replace with a copy or coalesce with
1954		 * the previous mbuf if possible (since we have to copy
1955		 * it anyway, we try to reduce the number of mbufs and
1956		 * clusters so that future work is easier).
1957		 */
1958		KASSERT(m->m_flags & M_EXT, ("m_flags 0x%x", m->m_flags));
1959		/* NB: we only coalesce into a cluster or larger */
1960		if (mprev != NULL && (mprev->m_flags & M_EXT) &&
1961		    m->m_len <= M_TRAILINGSPACE(mprev)) {
1962			/* XXX: this ignores mbuf types */
1963			memcpy(mtod(mprev, caddr_t) + mprev->m_len,
1964			       mtod(m, caddr_t), m->m_len);
1965			mprev->m_len += m->m_len;
1966			mprev->m_next = m->m_next;	/* unlink from chain */
1967			m_free(m);			/* reclaim mbuf */
1968#if 0
1969			newipsecstat.ips_clcoalesced++;
1970#endif
1971			continue;
1972		}
1973
1974		/*
1975		 * Allocate new space to hold the copy and copy the data.
1976		 * We deal with jumbo mbufs (i.e. m_len > MCLBYTES) by
1977		 * splitting them into clusters.  We could just malloc a
1978		 * buffer and make it external but too many device drivers
1979		 * don't know how to break up the non-contiguous memory when
1980		 * doing DMA.
1981		 */
1982		n = m_getcl(how, m->m_type, m->m_flags);
1983		if (n == NULL) {
1984			m_freem(m0);
1985			return (NULL);
1986		}
1987		len = m->m_len;
1988		off = 0;
1989		mfirst = n;
1990		mlast = NULL;
1991		for (;;) {
1992			int cc = min(len, MCLBYTES);
1993			memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, cc);
1994			n->m_len = cc;
1995			if (mlast != NULL)
1996				mlast->m_next = n;
1997			mlast = n;
1998#if 0
1999			newipsecstat.ips_clcopied++;
2000#endif
2001
2002			len -= cc;
2003			if (len <= 0)
2004				break;
2005			off += cc;
2006
2007			n = m_getcl(how, m->m_type, m->m_flags);
2008			if (n == NULL) {
2009				m_freem(mfirst);
2010				m_freem(m0);
2011				return (NULL);
2012			}
2013		}
2014		n->m_next = m->m_next;
2015		if (mprev == NULL)
2016			m0 = mfirst;		/* new head of chain */
2017		else
2018			mprev->m_next = mfirst;	/* replace old mbuf */
2019		m_free(m);			/* release old mbuf */
2020		mprev = mfirst;
2021	}
2022	return (m0);
2023}
2024
2025#ifdef MBUF_PROFILING
2026
2027#define MP_BUCKETS 32 /* don't just change this as things may overflow.*/
2028struct mbufprofile {
2029	uintmax_t wasted[MP_BUCKETS];
2030	uintmax_t used[MP_BUCKETS];
2031	uintmax_t segments[MP_BUCKETS];
2032} mbprof;
2033
2034#define MP_MAXDIGITS 21	/* strlen("16,000,000,000,000,000,000") == 21 */
2035#define MP_NUMLINES 6
2036#define MP_NUMSPERLINE 16
2037#define MP_EXTRABYTES 64	/* > strlen("used:\nwasted:\nsegments:\n") */
2038/* work out max space needed and add a bit of spare space too */
2039#define MP_MAXLINE ((MP_MAXDIGITS+1) * MP_NUMSPERLINE)
2040#define MP_BUFSIZE ((MP_MAXLINE * MP_NUMLINES) + 1 + MP_EXTRABYTES)
2041
2042char mbprofbuf[MP_BUFSIZE];
2043
2044void
2045m_profile(struct mbuf *m)
2046{
2047	int segments = 0;
2048	int used = 0;
2049	int wasted = 0;
2050
2051	while (m) {
2052		segments++;
2053		used += m->m_len;
2054		if (m->m_flags & M_EXT) {
2055			wasted += MHLEN - sizeof(m->m_ext) +
2056			    m->m_ext.ext_size - m->m_len;
2057		} else {
2058			if (m->m_flags & M_PKTHDR)
2059				wasted += MHLEN - m->m_len;
2060			else
2061				wasted += MLEN - m->m_len;
2062		}
2063		m = m->m_next;
2064	}
2065	/* be paranoid.. it helps */
2066	if (segments > MP_BUCKETS - 1)
2067		segments = MP_BUCKETS - 1;
2068	if (used > 100000)
2069		used = 100000;
2070	if (wasted > 100000)
2071		wasted = 100000;
2072	/* store in the appropriate bucket */
2073	/* don't bother locking. if it's slightly off, so what? */
2074	mbprof.segments[segments]++;
2075	mbprof.used[fls(used)]++;
2076	mbprof.wasted[fls(wasted)]++;
2077}
2078
2079static void
2080mbprof_textify(void)
2081{
2082	int offset;
2083	char *c;
2084	uint64_t *p;
2085
2086
2087	p = &mbprof.wasted[0];
2088	c = mbprofbuf;
2089	offset = snprintf(c, MP_MAXLINE + 10,
2090	    "wasted:\n"
2091	    "%ju %ju %ju %ju %ju %ju %ju %ju "
2092	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2093	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2094	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2095#ifdef BIG_ARRAY
2096	p = &mbprof.wasted[16];
2097	c += offset;
2098	offset = snprintf(c, MP_MAXLINE,
2099	    "%ju %ju %ju %ju %ju %ju %ju %ju "
2100	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2101	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2102	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2103#endif
2104	p = &mbprof.used[0];
2105	c += offset;
2106	offset = snprintf(c, MP_MAXLINE + 10,
2107	    "used:\n"
2108	    "%ju %ju %ju %ju %ju %ju %ju %ju "
2109	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2110	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2111	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2112#ifdef BIG_ARRAY
2113	p = &mbprof.used[16];
2114	c += offset;
2115	offset = snprintf(c, MP_MAXLINE,
2116	    "%ju %ju %ju %ju %ju %ju %ju %ju "
2117	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2118	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2119	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2120#endif
2121	p = &mbprof.segments[0];
2122	c += offset;
2123	offset = snprintf(c, MP_MAXLINE + 10,
2124	    "segments:\n"
2125	    "%ju %ju %ju %ju %ju %ju %ju %ju "
2126	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2127	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2128	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2129#ifdef BIG_ARRAY
2130	p = &mbprof.segments[16];
2131	c += offset;
2132	offset = snprintf(c, MP_MAXLINE,
2133	    "%ju %ju %ju %ju %ju %ju %ju %ju "
2134	    "%ju %ju %ju %ju %ju %ju %ju %jju",
2135	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2136	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2137#endif
2138}
2139
2140static int
2141mbprof_handler(SYSCTL_HANDLER_ARGS)
2142{
2143	int error;
2144
2145	mbprof_textify();
2146	error = SYSCTL_OUT(req, mbprofbuf, strlen(mbprofbuf) + 1);
2147	return (error);
2148}
2149
2150static int
2151mbprof_clr_handler(SYSCTL_HANDLER_ARGS)
2152{
2153	int clear, error;
2154
2155	clear = 0;
2156	error = sysctl_handle_int(oidp, &clear, 0, req);
2157	if (error || !req->newptr)
2158		return (error);
2159
2160	if (clear) {
2161		bzero(&mbprof, sizeof(mbprof));
2162	}
2163
2164	return (error);
2165}
2166
2167
2168SYSCTL_PROC(_kern_ipc, OID_AUTO, mbufprofile, CTLTYPE_STRING|CTLFLAG_RD,
2169	    NULL, 0, mbprof_handler, "A", "mbuf profiling statistics");
2170
2171SYSCTL_PROC(_kern_ipc, OID_AUTO, mbufprofileclr, CTLTYPE_INT|CTLFLAG_RW,
2172	    NULL, 0, mbprof_clr_handler, "I", "clear mbuf profiling statistics");
2173#endif
2174
2175