uipc_mbuf.c revision 130357
178344Sobrien/*
278344Sobrien * Copyright (c) 1982, 1986, 1988, 1991, 1993
398184Sgordon *	The Regents of the University of California.  All rights reserved.
478344Sobrien *
578344Sobrien * Redistribution and use in source and binary forms, with or without
678344Sobrien * modification, are permitted provided that the following conditions
7197144Shrs * are met:
8136224Smtm * 1. Redistributions of source code must retain the above copyright
978344Sobrien *    notice, this list of conditions and the following disclaimer.
1078344Sobrien * 2. Redistributions in binary form must reproduce the above copyright
1178344Sobrien *    notice, this list of conditions and the following disclaimer in the
1278344Sobrien *    documentation and/or other materials provided with the distribution.
13298514Slme * 4. Neither the name of the University nor the names of its contributors
14230099Sdougb *    may be used to endorse or promote products derived from this software
1578344Sobrien *    without specific prior written permission.
16197144Shrs *
17197144Shrs * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
18197144Shrs * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19197144Shrs * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
2078344Sobrien * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
2178344Sobrien * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 *	@(#)uipc_mbuf.c	8.2 (Berkeley) 1/4/94
30 */
31
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD: head/sys/kern/uipc_mbuf.c 130357 2004-06-11 18:17:25Z bmilekic $");
34
35#include "opt_mac.h"
36#include "opt_param.h"
37#include "opt_mbuf_stress_test.h"
38
39#include <sys/param.h>
40#include <sys/systm.h>
41#include <sys/kernel.h>
42#include <sys/limits.h>
43#include <sys/lock.h>
44#include <sys/mac.h>
45#include <sys/malloc.h>
46#include <sys/mbuf.h>
47#include <sys/sysctl.h>
48#include <sys/domain.h>
49#include <sys/protosw.h>
50#include <sys/uio.h>
51
52int	max_linkhdr;
53int	max_protohdr;
54int	max_hdr;
55int	max_datalen;
56#ifdef MBUF_STRESS_TEST
57int	m_defragpackets;
58int	m_defragbytes;
59int	m_defraguseless;
60int	m_defragfailure;
61int	m_defragrandomfailures;
62#endif
63
64/*
65 * sysctl(8) exported objects
66 */
67SYSCTL_DECL(_kern_ipc);
68SYSCTL_INT(_kern_ipc, KIPC_MAX_LINKHDR, max_linkhdr, CTLFLAG_RW,
69	   &max_linkhdr, 0, "");
70SYSCTL_INT(_kern_ipc, KIPC_MAX_PROTOHDR, max_protohdr, CTLFLAG_RW,
71	   &max_protohdr, 0, "");
72SYSCTL_INT(_kern_ipc, KIPC_MAX_HDR, max_hdr, CTLFLAG_RW, &max_hdr, 0, "");
73SYSCTL_INT(_kern_ipc, KIPC_MAX_DATALEN, max_datalen, CTLFLAG_RW,
74	   &max_datalen, 0, "");
75#ifdef MBUF_STRESS_TEST
76SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragpackets, CTLFLAG_RD,
77	   &m_defragpackets, 0, "");
78SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragbytes, CTLFLAG_RD,
79	   &m_defragbytes, 0, "");
80SYSCTL_INT(_kern_ipc, OID_AUTO, m_defraguseless, CTLFLAG_RD,
81	   &m_defraguseless, 0, "");
82SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragfailure, CTLFLAG_RD,
83	   &m_defragfailure, 0, "");
84SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW,
85	   &m_defragrandomfailures, 0, "");
86#endif
87
88/*
89 * Malloc-type for external ext_buf ref counts.
90 */
91MALLOC_DEFINE(M_MBUF, "mbextcnt", "mbuf external ref counts");
92
93/*
94 * Allocate a given length worth of mbufs and/or clusters (whatever fits
95 * best) and return a pointer to the top of the allocated chain.  If an
96 * existing mbuf chain is provided, then we will append the new chain
97 * to the existing one but still return the top of the newly allocated
98 * chain.
99 */
100struct mbuf *
101m_getm(struct mbuf *m, int len, int how, short type)
102{
103	struct mbuf *mb, *top, *cur, *mtail;
104	int num, rem;
105	int i;
106
107	KASSERT(len >= 0, ("m_getm(): len is < 0"));
108
109	/* If m != NULL, we will append to the end of that chain. */
110	if (m != NULL)
111		for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next);
112	else
113		mtail = NULL;
114
115	/*
116	 * Calculate how many mbufs+clusters ("packets") we need and how much
117	 * leftover there is after that and allocate the first mbuf+cluster
118	 * if required.
119	 */
120	num = len / MCLBYTES;
121	rem = len % MCLBYTES;
122	top = cur = NULL;
123	if (num > 0) {
124		if ((top = cur = m_getcl(how, type, 0)) == NULL)
125			goto failed;
126		top->m_len = 0;
127	}
128	num--;
129
130	for (i = 0; i < num; i++) {
131		mb = m_getcl(how, type, 0);
132		if (mb == NULL)
133			goto failed;
134		mb->m_len = 0;
135		cur = (cur->m_next = mb);
136	}
137	if (rem > 0) {
138		mb = (rem > MINCLSIZE) ?
139		    m_getcl(how, type, 0) : m_get(how, type);
140		if (mb == NULL)
141			goto failed;
142		mb->m_len = 0;
143		if (cur == NULL)
144			top = mb;
145		else
146			cur->m_next = mb;
147	}
148
149	if (mtail != NULL)
150		mtail->m_next = top;
151	return top;
152failed:
153	if (top != NULL)
154		m_freem(top);
155	return NULL;
156}
157
158/*
159 * Free an entire chain of mbufs and associated external buffers, if
160 * applicable.
161 */
162void
163m_freem(struct mbuf *mb)
164{
165
166	while (mb != NULL)
167		mb = m_free(mb);
168}
169
170/*-
171 * Configure a provided mbuf to refer to the provided external storage
172 * buffer and setup a reference count for said buffer.  If the setting
173 * up of the reference count fails, the M_EXT bit will not be set.  If
174 * successfull, the M_EXT bit is set in the mbuf's flags.
175 *
176 * Arguments:
177 *    mb     The existing mbuf to which to attach the provided buffer.
178 *    buf    The address of the provided external storage buffer.
179 *    size   The size of the provided buffer.
180 *    freef  A pointer to a routine that is responsible for freeing the
181 *           provided external storage buffer.
182 *    args   A pointer to an argument structure (of any type) to be passed
183 *           to the provided freef routine (may be NULL).
184 *    flags  Any other flags to be passed to the provided mbuf.
185 *    type   The type that the external storage buffer should be
186 *           labeled with.
187 *
188 * Returns:
189 *    Nothing.
190 */
191void
192m_extadd(struct mbuf *mb, caddr_t buf, u_int size,
193    void (*freef)(void *, void *), void *args, int flags, int type)
194{
195	u_int *ref_cnt = NULL;
196
197	/* XXX Shouldn't be adding EXT_CLUSTER with this API */
198	if (type == EXT_CLUSTER)
199		ref_cnt = (u_int *)uma_find_refcnt(zone_clust,
200		    mb->m_ext.ext_buf);
201	else if (type == EXT_EXTREF)
202		ref_cnt = mb->m_ext.ref_cnt;
203	mb->m_ext.ref_cnt = (ref_cnt == NULL) ?
204	    malloc(sizeof(u_int), M_MBUF, M_NOWAIT) : (u_int *)ref_cnt;
205	if (mb->m_ext.ref_cnt != NULL) {
206		*(mb->m_ext.ref_cnt) = 1;
207		mb->m_flags |= (M_EXT | flags);
208		mb->m_ext.ext_buf = buf;
209		mb->m_data = mb->m_ext.ext_buf;
210		mb->m_ext.ext_size = size;
211		mb->m_ext.ext_free = freef;
212		mb->m_ext.ext_args = args;
213		mb->m_ext.ext_type = type;
214        }
215}
216
217/*
218 * Non-directly-exported function to clean up after mbufs with M_EXT
219 * storage attached to them if the reference count hits 0.
220 */
221void
222mb_free_ext(struct mbuf *m)
223{
224	u_int cnt;
225
226	/*
227	 * This is tricky.  We need to make sure to decrement the
228	 * refcount in a safe way but to also clean up if we're the
229	 * last reference.  This method seems to do it without race.
230	 */
231	do {
232		cnt = *(m->m_ext.ref_cnt);
233		if (atomic_cmpset_int(m->m_ext.ref_cnt, cnt, cnt - 1)) {
234			if (cnt == 1) {
235				/*
236				 * Do the free, should be safe.
237				 */
238				if (m->m_ext.ext_type == EXT_PACKET) {
239					uma_zfree(zone_pack, m);
240					return;
241				} else if (m->m_ext.ext_type == EXT_CLUSTER) {
242					uma_zfree(zone_clust, m->m_ext.ext_buf);
243					m->m_ext.ext_buf = NULL;
244				} else {
245					(*(m->m_ext.ext_free))(m->m_ext.ext_buf,
246					    m->m_ext.ext_args);
247					if (m->m_ext.ext_type != EXT_EXTREF)
248						free(m->m_ext.ref_cnt, M_MBUF);
249					m->m_ext.ext_buf = NULL;
250				}
251			}
252			/* Decrement (and potentially free) done, safely. */
253			break;
254		}
255	} while (1);
256	uma_zfree(zone_mbuf, m);
257}
258
259/*
260 * "Move" mbuf pkthdr from "from" to "to".
261 * "from" must have M_PKTHDR set, and "to" must be empty.
262 */
263void
264m_move_pkthdr(struct mbuf *to, struct mbuf *from)
265{
266
267#if 0
268	/* see below for why these are not enabled */
269	M_ASSERTPKTHDR(to);
270	/* Note: with MAC, this may not be a good assertion. */
271	KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags),
272	    ("m_move_pkthdr: to has tags"));
273#endif
274	KASSERT((to->m_flags & M_EXT) == 0, ("m_move_pkthdr: to has cluster"));
275#ifdef MAC
276	/*
277	 * XXXMAC: It could be this should also occur for non-MAC?
278	 */
279	if (to->m_flags & M_PKTHDR)
280		m_tag_delete_chain(to, NULL);
281#endif
282	to->m_flags = from->m_flags & M_COPYFLAGS;
283	to->m_data = to->m_pktdat;
284	to->m_pkthdr = from->m_pkthdr;		/* especially tags */
285	SLIST_INIT(&from->m_pkthdr.tags);	/* purge tags from src */
286	from->m_flags &= ~M_PKTHDR;
287}
288
289/*
290 * Duplicate "from"'s mbuf pkthdr in "to".
291 * "from" must have M_PKTHDR set, and "to" must be empty.
292 * In particular, this does a deep copy of the packet tags.
293 */
294int
295m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
296{
297
298#if 0
299	/*
300	 * The mbuf allocator only initializes the pkthdr
301	 * when the mbuf is allocated with MGETHDR. Many users
302	 * (e.g. m_copy*, m_prepend) use MGET and then
303	 * smash the pkthdr as needed causing these
304	 * assertions to trip.  For now just disable them.
305	 */
306	M_ASSERTPKTHDR(to);
307	/* Note: with MAC, this may not be a good assertion. */
308	KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), ("m_dup_pkthdr: to has tags"));
309#endif
310#ifdef MAC
311	if (to->m_flags & M_PKTHDR)
312		m_tag_delete_chain(to, NULL);
313#endif
314	to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
315	if ((to->m_flags & M_EXT) == 0)
316		to->m_data = to->m_pktdat;
317	to->m_pkthdr = from->m_pkthdr;
318	SLIST_INIT(&to->m_pkthdr.tags);
319	return (m_tag_copy_chain(to, from, MBTOM(how)));
320}
321
322/*
323 * Lesser-used path for M_PREPEND:
324 * allocate new mbuf to prepend to chain,
325 * copy junk along.
326 */
327struct mbuf *
328m_prepend(struct mbuf *m, int len, int how)
329{
330	struct mbuf *mn;
331
332	if (m->m_flags & M_PKTHDR)
333		MGETHDR(mn, how, m->m_type);
334	else
335		MGET(mn, how, m->m_type);
336	if (mn == NULL) {
337		m_freem(m);
338		return (NULL);
339	}
340	if (m->m_flags & M_PKTHDR)
341		M_MOVE_PKTHDR(mn, m);
342	mn->m_next = m;
343	m = mn;
344	if (len < MHLEN)
345		MH_ALIGN(m, len);
346	m->m_len = len;
347	return (m);
348}
349
350/*
351 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
352 * continuing for "len" bytes.  If len is M_COPYALL, copy to end of mbuf.
353 * The wait parameter is a choice of M_TRYWAIT/M_DONTWAIT from caller.
354 * Note that the copy is read-only, because clusters are not copied,
355 * only their reference counts are incremented.
356 */
357struct mbuf *
358m_copym(struct mbuf *m, int off0, int len, int wait)
359{
360	struct mbuf *n, **np;
361	int off = off0;
362	struct mbuf *top;
363	int copyhdr = 0;
364
365	KASSERT(off >= 0, ("m_copym, negative off %d", off));
366	KASSERT(len >= 0, ("m_copym, negative len %d", len));
367	if (off == 0 && m->m_flags & M_PKTHDR)
368		copyhdr = 1;
369	while (off > 0) {
370		KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain"));
371		if (off < m->m_len)
372			break;
373		off -= m->m_len;
374		m = m->m_next;
375	}
376	np = &top;
377	top = 0;
378	while (len > 0) {
379		if (m == NULL) {
380			KASSERT(len == M_COPYALL,
381			    ("m_copym, length > size of mbuf chain"));
382			break;
383		}
384		if (copyhdr)
385			MGETHDR(n, wait, m->m_type);
386		else
387			MGET(n, wait, m->m_type);
388		*np = n;
389		if (n == NULL)
390			goto nospace;
391		if (copyhdr) {
392			if (!m_dup_pkthdr(n, m, wait))
393				goto nospace;
394			if (len == M_COPYALL)
395				n->m_pkthdr.len -= off0;
396			else
397				n->m_pkthdr.len = len;
398			copyhdr = 0;
399		}
400		n->m_len = min(len, m->m_len - off);
401		if (m->m_flags & M_EXT) {
402			n->m_data = m->m_data + off;
403			n->m_ext = m->m_ext;
404			n->m_flags |= M_EXT;
405			MEXT_ADD_REF(m);
406		} else
407			bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
408			    (u_int)n->m_len);
409		if (len != M_COPYALL)
410			len -= n->m_len;
411		off = 0;
412		m = m->m_next;
413		np = &n->m_next;
414	}
415	if (top == NULL)
416		mbstat.m_mcfail++;	/* XXX: No consistency. */
417
418	return (top);
419nospace:
420	m_freem(top);
421	mbstat.m_mcfail++;	/* XXX: No consistency. */
422	return (NULL);
423}
424
425/*
426 * Copy an entire packet, including header (which must be present).
427 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'.
428 * Note that the copy is read-only, because clusters are not copied,
429 * only their reference counts are incremented.
430 * Preserve alignment of the first mbuf so if the creator has left
431 * some room at the beginning (e.g. for inserting protocol headers)
432 * the copies still have the room available.
433 */
434struct mbuf *
435m_copypacket(struct mbuf *m, int how)
436{
437	struct mbuf *top, *n, *o;
438
439	MGET(n, how, m->m_type);
440	top = n;
441	if (n == NULL)
442		goto nospace;
443
444	if (!m_dup_pkthdr(n, m, how))
445		goto nospace;
446	n->m_len = m->m_len;
447	if (m->m_flags & M_EXT) {
448		n->m_data = m->m_data;
449		n->m_ext = m->m_ext;
450		n->m_flags |= M_EXT;
451		MEXT_ADD_REF(m);
452	} else {
453		n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat );
454		bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
455	}
456
457	m = m->m_next;
458	while (m) {
459		MGET(o, how, m->m_type);
460		if (o == NULL)
461			goto nospace;
462
463		n->m_next = o;
464		n = n->m_next;
465
466		n->m_len = m->m_len;
467		if (m->m_flags & M_EXT) {
468			n->m_data = m->m_data;
469			n->m_ext = m->m_ext;
470			n->m_flags |= M_EXT;
471			MEXT_ADD_REF(m);
472		} else {
473			bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
474		}
475
476		m = m->m_next;
477	}
478	return top;
479nospace:
480	m_freem(top);
481	mbstat.m_mcfail++;	/* XXX: No consistency. */
482	return (NULL);
483}
484
485/*
486 * Copy data from an mbuf chain starting "off" bytes from the beginning,
487 * continuing for "len" bytes, into the indicated buffer.
488 */
489void
490m_copydata(const struct mbuf *m, int off, int len, caddr_t cp)
491{
492	u_int count;
493
494	KASSERT(off >= 0, ("m_copydata, negative off %d", off));
495	KASSERT(len >= 0, ("m_copydata, negative len %d", len));
496	while (off > 0) {
497		KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain"));
498		if (off < m->m_len)
499			break;
500		off -= m->m_len;
501		m = m->m_next;
502	}
503	while (len > 0) {
504		KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain"));
505		count = min(m->m_len - off, len);
506		bcopy(mtod(m, caddr_t) + off, cp, count);
507		len -= count;
508		cp += count;
509		off = 0;
510		m = m->m_next;
511	}
512}
513
514/*
515 * Copy a packet header mbuf chain into a completely new chain, including
516 * copying any mbuf clusters.  Use this instead of m_copypacket() when
517 * you need a writable copy of an mbuf chain.
518 */
519struct mbuf *
520m_dup(struct mbuf *m, int how)
521{
522	struct mbuf **p, *top = NULL;
523	int remain, moff, nsize;
524
525	/* Sanity check */
526	if (m == NULL)
527		return (NULL);
528	M_ASSERTPKTHDR(m);
529
530	/* While there's more data, get a new mbuf, tack it on, and fill it */
531	remain = m->m_pkthdr.len;
532	moff = 0;
533	p = &top;
534	while (remain > 0 || top == NULL) {	/* allow m->m_pkthdr.len == 0 */
535		struct mbuf *n;
536
537		/* Get the next new mbuf */
538		if (remain >= MINCLSIZE) {
539			n = m_getcl(how, m->m_type, 0);
540			nsize = MCLBYTES;
541		} else {
542			n = m_get(how, m->m_type);
543			nsize = MLEN;
544		}
545		if (n == NULL)
546			goto nospace;
547
548		if (top == NULL) {		/* First one, must be PKTHDR */
549			if (!m_dup_pkthdr(n, m, how)) {
550				m_free(n);
551				goto nospace;
552			}
553			nsize = MHLEN;
554		}
555		n->m_len = 0;
556
557		/* Link it into the new chain */
558		*p = n;
559		p = &n->m_next;
560
561		/* Copy data from original mbuf(s) into new mbuf */
562		while (n->m_len < nsize && m != NULL) {
563			int chunk = min(nsize - n->m_len, m->m_len - moff);
564
565			bcopy(m->m_data + moff, n->m_data + n->m_len, chunk);
566			moff += chunk;
567			n->m_len += chunk;
568			remain -= chunk;
569			if (moff == m->m_len) {
570				m = m->m_next;
571				moff = 0;
572			}
573		}
574
575		/* Check correct total mbuf length */
576		KASSERT((remain > 0 && m != NULL) || (remain == 0 && m == NULL),
577		    	("%s: bogus m_pkthdr.len", __func__));
578	}
579	return (top);
580
581nospace:
582	m_freem(top);
583	mbstat.m_mcfail++;	/* XXX: No consistency. */
584	return (NULL);
585}
586
587/*
588 * Concatenate mbuf chain n to m.
589 * Both chains must be of the same type (e.g. MT_DATA).
590 * Any m_pkthdr is not updated.
591 */
592void
593m_cat(struct mbuf *m, struct mbuf *n)
594{
595	while (m->m_next)
596		m = m->m_next;
597	while (n) {
598		if (m->m_flags & M_EXT ||
599		    m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
600			/* just join the two chains */
601			m->m_next = n;
602			return;
603		}
604		/* splat the data from one into the other */
605		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
606		    (u_int)n->m_len);
607		m->m_len += n->m_len;
608		n = m_free(n);
609	}
610}
611
612void
613m_adj(struct mbuf *mp, int req_len)
614{
615	int len = req_len;
616	struct mbuf *m;
617	int count;
618
619	if ((m = mp) == NULL)
620		return;
621	if (len >= 0) {
622		/*
623		 * Trim from head.
624		 */
625		while (m != NULL && len > 0) {
626			if (m->m_len <= len) {
627				len -= m->m_len;
628				m->m_len = 0;
629				m = m->m_next;
630			} else {
631				m->m_len -= len;
632				m->m_data += len;
633				len = 0;
634			}
635		}
636		m = mp;
637		if (mp->m_flags & M_PKTHDR)
638			m->m_pkthdr.len -= (req_len - len);
639	} else {
640		/*
641		 * Trim from tail.  Scan the mbuf chain,
642		 * calculating its length and finding the last mbuf.
643		 * If the adjustment only affects this mbuf, then just
644		 * adjust and return.  Otherwise, rescan and truncate
645		 * after the remaining size.
646		 */
647		len = -len;
648		count = 0;
649		for (;;) {
650			count += m->m_len;
651			if (m->m_next == (struct mbuf *)0)
652				break;
653			m = m->m_next;
654		}
655		if (m->m_len >= len) {
656			m->m_len -= len;
657			if (mp->m_flags & M_PKTHDR)
658				mp->m_pkthdr.len -= len;
659			return;
660		}
661		count -= len;
662		if (count < 0)
663			count = 0;
664		/*
665		 * Correct length for chain is "count".
666		 * Find the mbuf with last data, adjust its length,
667		 * and toss data from remaining mbufs on chain.
668		 */
669		m = mp;
670		if (m->m_flags & M_PKTHDR)
671			m->m_pkthdr.len = count;
672		for (; m; m = m->m_next) {
673			if (m->m_len >= count) {
674				m->m_len = count;
675				break;
676			}
677			count -= m->m_len;
678		}
679		while (m->m_next)
680			(m = m->m_next) ->m_len = 0;
681	}
682}
683
684/*
685 * Rearange an mbuf chain so that len bytes are contiguous
686 * and in the data area of an mbuf (so that mtod and dtom
687 * will work for a structure of size len).  Returns the resulting
688 * mbuf chain on success, frees it and returns null on failure.
689 * If there is room, it will add up to max_protohdr-len extra bytes to the
690 * contiguous region in an attempt to avoid being called next time.
691 */
692struct mbuf *
693m_pullup(struct mbuf *n, int len)
694{
695	struct mbuf *m;
696	int count;
697	int space;
698
699	/*
700	 * If first mbuf has no cluster, and has room for len bytes
701	 * without shifting current data, pullup into it,
702	 * otherwise allocate a new mbuf to prepend to the chain.
703	 */
704	if ((n->m_flags & M_EXT) == 0 &&
705	    n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
706		if (n->m_len >= len)
707			return (n);
708		m = n;
709		n = n->m_next;
710		len -= m->m_len;
711	} else {
712		if (len > MHLEN)
713			goto bad;
714		MGET(m, M_DONTWAIT, n->m_type);
715		if (m == NULL)
716			goto bad;
717		m->m_len = 0;
718		if (n->m_flags & M_PKTHDR)
719			M_MOVE_PKTHDR(m, n);
720	}
721	space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
722	do {
723		count = min(min(max(len, max_protohdr), space), n->m_len);
724		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
725		  (u_int)count);
726		len -= count;
727		m->m_len += count;
728		n->m_len -= count;
729		space -= count;
730		if (n->m_len)
731			n->m_data += count;
732		else
733			n = m_free(n);
734	} while (len > 0 && n);
735	if (len > 0) {
736		(void) m_free(m);
737		goto bad;
738	}
739	m->m_next = n;
740	return (m);
741bad:
742	m_freem(n);
743	mbstat.m_mpfail++;	/* XXX: No consistency. */
744	return (NULL);
745}
746
747/*
748 * Partition an mbuf chain in two pieces, returning the tail --
749 * all but the first len0 bytes.  In case of failure, it returns NULL and
750 * attempts to restore the chain to its original state.
751 *
752 * Note that the resulting mbufs might be read-only, because the new
753 * mbuf can end up sharing an mbuf cluster with the original mbuf if
754 * the "breaking point" happens to lie within a cluster mbuf. Use the
755 * M_WRITABLE() macro to check for this case.
756 */
757struct mbuf *
758m_split(struct mbuf *m0, int len0, int wait)
759{
760	struct mbuf *m, *n;
761	u_int len = len0, remain;
762
763	for (m = m0; m && len > m->m_len; m = m->m_next)
764		len -= m->m_len;
765	if (m == NULL)
766		return (NULL);
767	remain = m->m_len - len;
768	if (m0->m_flags & M_PKTHDR) {
769		MGETHDR(n, wait, m0->m_type);
770		if (n == NULL)
771			return (NULL);
772		n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
773		n->m_pkthdr.len = m0->m_pkthdr.len - len0;
774		m0->m_pkthdr.len = len0;
775		if (m->m_flags & M_EXT)
776			goto extpacket;
777		if (remain > MHLEN) {
778			/* m can't be the lead packet */
779			MH_ALIGN(n, 0);
780			n->m_next = m_split(m, len, wait);
781			if (n->m_next == NULL) {
782				(void) m_free(n);
783				return (NULL);
784			} else {
785				n->m_len = 0;
786				return (n);
787			}
788		} else
789			MH_ALIGN(n, remain);
790	} else if (remain == 0) {
791		n = m->m_next;
792		m->m_next = NULL;
793		return (n);
794	} else {
795		MGET(n, wait, m->m_type);
796		if (n == NULL)
797			return (NULL);
798		M_ALIGN(n, remain);
799	}
800extpacket:
801	if (m->m_flags & M_EXT) {
802		n->m_flags |= M_EXT;
803		n->m_ext = m->m_ext;
804		MEXT_ADD_REF(m);
805		n->m_data = m->m_data + len;
806	} else {
807		bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
808	}
809	n->m_len = remain;
810	m->m_len = len;
811	n->m_next = m->m_next;
812	m->m_next = NULL;
813	return (n);
814}
815/*
816 * Routine to copy from device local memory into mbufs.
817 * Note that `off' argument is offset into first mbuf of target chain from
818 * which to begin copying the data to.
819 */
820struct mbuf *
821m_devget(char *buf, int totlen, int off, struct ifnet *ifp,
822	 void (*copy)(char *from, caddr_t to, u_int len))
823{
824	struct mbuf *m;
825	struct mbuf *top = NULL, **mp = &top;
826	int len;
827
828	if (off < 0 || off > MHLEN)
829		return (NULL);
830
831	while (totlen > 0) {
832		if (top == NULL) {	/* First one, must be PKTHDR */
833			if (totlen + off >= MINCLSIZE) {
834				m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
835				len = MCLBYTES;
836			} else {
837				m = m_gethdr(M_DONTWAIT, MT_DATA);
838				len = MHLEN;
839
840				/* Place initial small packet/header at end of mbuf */
841				if (m && totlen + off + max_linkhdr <= MLEN) {
842					m->m_data += max_linkhdr;
843					len -= max_linkhdr;
844				}
845			}
846			if (m == NULL)
847				return NULL;
848			m->m_pkthdr.rcvif = ifp;
849			m->m_pkthdr.len = totlen;
850		} else {
851			if (totlen + off >= MINCLSIZE) {
852				m = m_getcl(M_DONTWAIT, MT_DATA, 0);
853				len = MCLBYTES;
854			} else {
855				m = m_get(M_DONTWAIT, MT_DATA);
856				len = MLEN;
857			}
858			if (m == NULL) {
859				m_freem(top);
860				return NULL;
861			}
862		}
863		if (off) {
864			m->m_data += off;
865			len -= off;
866			off = 0;
867		}
868		m->m_len = len = min(totlen, len);
869		if (copy)
870			copy(buf, mtod(m, caddr_t), (u_int)len);
871		else
872			bcopy(buf, mtod(m, caddr_t), (u_int)len);
873		buf += len;
874		*mp = m;
875		mp = &m->m_next;
876		totlen -= len;
877	}
878	return (top);
879}
880
881/*
882 * Copy data from a buffer back into the indicated mbuf chain,
883 * starting "off" bytes from the beginning, extending the mbuf
884 * chain if necessary.
885 */
886void
887m_copyback(struct mbuf *m0, int off, int len, c_caddr_t cp)
888{
889	int mlen;
890	struct mbuf *m = m0, *n;
891	int totlen = 0;
892
893	if (m0 == NULL)
894		return;
895	while (off > (mlen = m->m_len)) {
896		off -= mlen;
897		totlen += mlen;
898		if (m->m_next == NULL) {
899			n = m_get(M_DONTWAIT, m->m_type);
900			if (n == NULL)
901				goto out;
902			bzero(mtod(n, caddr_t), MLEN);
903			n->m_len = min(MLEN, len + off);
904			m->m_next = n;
905		}
906		m = m->m_next;
907	}
908	while (len > 0) {
909		mlen = min (m->m_len - off, len);
910		bcopy(cp, off + mtod(m, caddr_t), (u_int)mlen);
911		cp += mlen;
912		len -= mlen;
913		mlen += off;
914		off = 0;
915		totlen += mlen;
916		if (len == 0)
917			break;
918		if (m->m_next == NULL) {
919			n = m_get(M_DONTWAIT, m->m_type);
920			if (n == NULL)
921				break;
922			n->m_len = min(MLEN, len);
923			m->m_next = n;
924		}
925		m = m->m_next;
926	}
927out:	if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
928		m->m_pkthdr.len = totlen;
929}
930
931/*
932 * Apply function f to the data in an mbuf chain starting "off" bytes from
933 * the beginning, continuing for "len" bytes.
934 */
935int
936m_apply(struct mbuf *m, int off, int len,
937    int (*f)(void *, void *, u_int), void *arg)
938{
939	u_int count;
940	int rval;
941
942	KASSERT(off >= 0, ("m_apply, negative off %d", off));
943	KASSERT(len >= 0, ("m_apply, negative len %d", len));
944	while (off > 0) {
945		KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain"));
946		if (off < m->m_len)
947			break;
948		off -= m->m_len;
949		m = m->m_next;
950	}
951	while (len > 0) {
952		KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain"));
953		count = min(m->m_len - off, len);
954		rval = (*f)(arg, mtod(m, caddr_t) + off, count);
955		if (rval)
956			return (rval);
957		len -= count;
958		off = 0;
959		m = m->m_next;
960	}
961	return (0);
962}
963
964/*
965 * Return a pointer to mbuf/offset of location in mbuf chain.
966 */
967struct mbuf *
968m_getptr(struct mbuf *m, int loc, int *off)
969{
970
971	while (loc >= 0) {
972		/* Normal end of search. */
973		if (m->m_len > loc) {
974			*off = loc;
975			return (m);
976		} else {
977			loc -= m->m_len;
978			if (m->m_next == NULL) {
979				if (loc == 0) {
980					/* Point at the end of valid data. */
981					*off = m->m_len;
982					return (m);
983				}
984				return (NULL);
985			}
986			m = m->m_next;
987		}
988	}
989	return (NULL);
990}
991
992void
993m_print(const struct mbuf *m)
994{
995	int len;
996	const struct mbuf *m2;
997
998	len = m->m_pkthdr.len;
999	m2 = m;
1000	while (len) {
1001		printf("%p %*D\n", m2, m2->m_len, (u_char *)m2->m_data, "-");
1002		len -= m2->m_len;
1003		m2 = m2->m_next;
1004	}
1005	return;
1006}
1007
1008u_int
1009m_fixhdr(struct mbuf *m0)
1010{
1011	u_int len;
1012
1013	len = m_length(m0, NULL);
1014	m0->m_pkthdr.len = len;
1015	return (len);
1016}
1017
1018u_int
1019m_length(struct mbuf *m0, struct mbuf **last)
1020{
1021	struct mbuf *m;
1022	u_int len;
1023
1024	len = 0;
1025	for (m = m0; m != NULL; m = m->m_next) {
1026		len += m->m_len;
1027		if (m->m_next == NULL)
1028			break;
1029	}
1030	if (last != NULL)
1031		*last = m;
1032	return (len);
1033}
1034
1035/*
1036 * Defragment a mbuf chain, returning the shortest possible
1037 * chain of mbufs and clusters.  If allocation fails and
1038 * this cannot be completed, NULL will be returned, but
1039 * the passed in chain will be unchanged.  Upon success,
1040 * the original chain will be freed, and the new chain
1041 * will be returned.
1042 *
1043 * If a non-packet header is passed in, the original
1044 * mbuf (chain?) will be returned unharmed.
1045 */
1046struct mbuf *
1047m_defrag(struct mbuf *m0, int how)
1048{
1049	struct mbuf *m_new = NULL, *m_final = NULL;
1050	int progress = 0, length;
1051
1052	if (!(m0->m_flags & M_PKTHDR))
1053		return (m0);
1054
1055	m_fixhdr(m0); /* Needed sanity check */
1056
1057#ifdef MBUF_STRESS_TEST
1058	if (m_defragrandomfailures) {
1059		int temp = arc4random() & 0xff;
1060		if (temp == 0xba)
1061			goto nospace;
1062	}
1063#endif
1064
1065	if (m0->m_pkthdr.len > MHLEN)
1066		m_final = m_getcl(how, MT_DATA, M_PKTHDR);
1067	else
1068		m_final = m_gethdr(how, MT_DATA);
1069
1070	if (m_final == NULL)
1071		goto nospace;
1072
1073	if (m_dup_pkthdr(m_final, m0, how) == 0)
1074		goto nospace;
1075
1076	m_new = m_final;
1077
1078	while (progress < m0->m_pkthdr.len) {
1079		length = m0->m_pkthdr.len - progress;
1080		if (length > MCLBYTES)
1081			length = MCLBYTES;
1082
1083		if (m_new == NULL) {
1084			if (length > MLEN)
1085				m_new = m_getcl(how, MT_DATA, 0);
1086			else
1087				m_new = m_get(how, MT_DATA);
1088			if (m_new == NULL)
1089				goto nospace;
1090		}
1091
1092		m_copydata(m0, progress, length, mtod(m_new, caddr_t));
1093		progress += length;
1094		m_new->m_len = length;
1095		if (m_new != m_final)
1096			m_cat(m_final, m_new);
1097		m_new = NULL;
1098	}
1099#ifdef MBUF_STRESS_TEST
1100	if (m0->m_next == NULL)
1101		m_defraguseless++;
1102#endif
1103	m_freem(m0);
1104	m0 = m_final;
1105#ifdef MBUF_STRESS_TEST
1106	m_defragpackets++;
1107	m_defragbytes += m0->m_pkthdr.len;
1108#endif
1109	return (m0);
1110nospace:
1111#ifdef MBUF_STRESS_TEST
1112	m_defragfailure++;
1113#endif
1114	if (m_new)
1115		m_free(m_new);
1116	if (m_final)
1117		m_freem(m_final);
1118	return (NULL);
1119}
1120
1121#ifdef MBUF_STRESS_TEST
1122
1123/*
1124 * Fragment an mbuf chain.  There's no reason you'd ever want to do
1125 * this in normal usage, but it's great for stress testing various
1126 * mbuf consumers.
1127 *
1128 * If fragmentation is not possible, the original chain will be
1129 * returned.
1130 *
1131 * Possible length values:
1132 * 0	 no fragmentation will occur
1133 * > 0	each fragment will be of the specified length
1134 * -1	each fragment will be the same random value in length
1135 * -2	each fragment's length will be entirely random
1136 * (Random values range from 1 to 256)
1137 */
1138struct mbuf *
1139m_fragment(struct mbuf *m0, int how, int length)
1140{
1141	struct mbuf *m_new = NULL, *m_final = NULL;
1142	int progress = 0;
1143
1144	if (!(m0->m_flags & M_PKTHDR))
1145		return (m0);
1146
1147	if ((length == 0) || (length < -2))
1148		return (m0);
1149
1150	m_fixhdr(m0); /* Needed sanity check */
1151
1152	m_final = m_getcl(how, MT_DATA, M_PKTHDR);
1153
1154	if (m_final == NULL)
1155		goto nospace;
1156
1157	if (m_dup_pkthdr(m_final, m0, how) == 0)
1158		goto nospace;
1159
1160	m_new = m_final;
1161
1162	if (length == -1)
1163		length = 1 + (arc4random() & 255);
1164
1165	while (progress < m0->m_pkthdr.len) {
1166		int fraglen;
1167
1168		if (length > 0)
1169			fraglen = length;
1170		else
1171			fraglen = 1 + (arc4random() & 255);
1172		if (fraglen > m0->m_pkthdr.len - progress)
1173			fraglen = m0->m_pkthdr.len - progress;
1174
1175		if (fraglen > MCLBYTES)
1176			fraglen = MCLBYTES;
1177
1178		if (m_new == NULL) {
1179			m_new = m_getcl(how, MT_DATA, 0);
1180			if (m_new == NULL)
1181				goto nospace;
1182		}
1183
1184		m_copydata(m0, progress, fraglen, mtod(m_new, caddr_t));
1185		progress += fraglen;
1186		m_new->m_len = fraglen;
1187		if (m_new != m_final)
1188			m_cat(m_final, m_new);
1189		m_new = NULL;
1190	}
1191	m_freem(m0);
1192	m0 = m_final;
1193	return (m0);
1194nospace:
1195	if (m_new)
1196		m_free(m_new);
1197	if (m_final)
1198		m_freem(m_final);
1199	/* Return the original chain on failure */
1200	return (m0);
1201}
1202
1203#endif
1204
1205struct mbuf *
1206m_uiotombuf(struct uio *uio, int how, int len)
1207{
1208	struct mbuf *m_new = NULL, *m_final = NULL;
1209	int progress = 0, error = 0, length, total;
1210
1211	if (len > 0)
1212		total = min(uio->uio_resid, len);
1213	else
1214		total = uio->uio_resid;
1215	if (total > MHLEN)
1216		m_final = m_getcl(how, MT_DATA, M_PKTHDR);
1217	else
1218		m_final = m_gethdr(how, MT_DATA);
1219	if (m_final == NULL)
1220		goto nospace;
1221	m_new = m_final;
1222	while (progress < total) {
1223		length = total - progress;
1224		if (length > MCLBYTES)
1225			length = MCLBYTES;
1226		if (m_new == NULL) {
1227			if (length > MLEN)
1228				m_new = m_getcl(how, MT_DATA, 0);
1229			else
1230				m_new = m_get(how, MT_DATA);
1231			if (m_new == NULL)
1232				goto nospace;
1233		}
1234		error = uiomove(mtod(m_new, void *), length, uio);
1235		if (error)
1236			goto nospace;
1237		progress += length;
1238		m_new->m_len = length;
1239		if (m_new != m_final)
1240			m_cat(m_final, m_new);
1241		m_new = NULL;
1242	}
1243	m_fixhdr(m_final);
1244	return (m_final);
1245nospace:
1246	if (m_new)
1247		m_free(m_new);
1248	if (m_final)
1249		m_freem(m_final);
1250	return (NULL);
1251}
1252