uipc_mbuf.c revision 7066 
1/*
2 * Copyright (c) 1982, 1986, 1988, 1991, 1993
3 *	The Regents of the University of California.  All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 * 3. All advertising materials mentioning features or use of this software
14 *    must display the following acknowledgement:
15 *	This product includes software developed by the University of
16 *	California, Berkeley and its contributors.
17 * 4. Neither the name of the University nor the names of its contributors
18 *    may be used to endorse or promote products derived from this software
19 *    without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
32 *
33 *	@(#)uipc_mbuf.c	8.2 (Berkeley) 1/4/94
34 * $Id: uipc_mbuf.c,v 1.8 1995/02/23 19:10:21 davidg Exp $
35 */
36
37#include <sys/param.h>
38#include <sys/systm.h>
39#include <sys/proc.h>
40#include <sys/malloc.h>
41#define MBTYPES
42#include <sys/mbuf.h>
43#include <sys/kernel.h>
44#include <sys/syslog.h>
45#include <sys/domain.h>
46#include <sys/protosw.h>
47
48#include <vm/vm.h>
49
50extern	vm_map_t mb_map;
51struct	mbuf *mbutl;
52char	*mclrefcnt;
53int	mb_map_full;
54
55void
56mbinit()
57{
58	int s;
59
60#if CLBYTES < 4096
61#define NCL_INIT	(4096/CLBYTES)
62#else
63#define NCL_INIT	1
64#endif
65	s = splimp();
66	if (m_clalloc(NCL_INIT, M_DONTWAIT) == 0)
67		goto bad;
68	splx(s);
69	return;
70bad:
71	panic("mbinit");
72}
73
74/*
75 * Allocate some number of mbuf clusters
76 * and place on cluster free list.
77 * Must be called at splimp.
78 */
79/* ARGSUSED */
80int
81m_clalloc(ncl, nowait)
82	register int ncl;
83	int nowait;
84{
85	static int logged;
86	register caddr_t p;
87	register int i;
88	int npg;
89
90	/*
91	 * Once we run out of map space, it will be impossible
92	 * to get any more (nothing is ever freed back to the
93	 * map).
94	 */
95	if (mb_map_full)
96		return (0);
97
98	npg = ncl * CLSIZE;
99	p = (caddr_t)kmem_malloc(mb_map, ctob(npg),
100				 nowait ? M_NOWAIT : M_WAITOK);
101	/*
102	 * Either the map is now full, or this is nowait and there
103	 * are no pages left.
104	 */
105	if (p == NULL)
106		return (0);
107
108	ncl = ncl * CLBYTES / MCLBYTES;
109	for (i = 0; i < ncl; i++) {
110		((union mcluster *)p)->mcl_next = mclfree;
111		mclfree = (union mcluster *)p;
112		p += MCLBYTES;
113		mbstat.m_clfree++;
114	}
115	mbstat.m_clusters += ncl;
116	return (1);
117}
118
119/*
120 * When MGET failes, ask protocols to free space when short of memory,
121 * then re-attempt to allocate an mbuf.
122 */
123struct mbuf *
124m_retry(i, t)
125	int i, t;
126{
127	register struct mbuf *m;
128
129	m_reclaim();
130#define m_retry(i, t)	(struct mbuf *)0
131	MGET(m, i, t);
132#undef m_retry
133	if (m != NULL)
134		mbstat.m_wait++;
135	else
136		mbstat.m_drops++;
137	return (m);
138}
139
140/*
141 * As above; retry an MGETHDR.
142 */
143struct mbuf *
144m_retryhdr(i, t)
145	int i, t;
146{
147	register struct mbuf *m;
148
149	m_reclaim();
150#define m_retryhdr(i, t) (struct mbuf *)0
151	MGETHDR(m, i, t);
152#undef m_retryhdr
153	if (m != NULL)
154		mbstat.m_wait++;
155	else
156		mbstat.m_drops++;
157	return (m);
158}
159
160void
161m_reclaim()
162{
163	register struct domain *dp;
164	register struct protosw *pr;
165	int s = splimp();
166
167	for (dp = domains; dp; dp = dp->dom_next)
168		for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
169			if (pr->pr_drain)
170				(*pr->pr_drain)();
171	splx(s);
172	mbstat.m_drain++;
173}
174
175/*
176 * Space allocation routines.
177 * These are also available as macros
178 * for critical paths.
179 */
180struct mbuf *
181m_get(nowait, type)
182	int nowait, type;
183{
184	register struct mbuf *m;
185
186	MGET(m, nowait, type);
187	return (m);
188}
189
190struct mbuf *
191m_gethdr(nowait, type)
192	int nowait, type;
193{
194	register struct mbuf *m;
195
196	MGETHDR(m, nowait, type);
197	return (m);
198}
199
200struct mbuf *
201m_getclr(nowait, type)
202	int nowait, type;
203{
204	register struct mbuf *m;
205
206	MGET(m, nowait, type);
207	if (m == 0)
208		return (0);
209	bzero(mtod(m, caddr_t), MLEN);
210	return (m);
211}
212
213struct mbuf *
214m_free(m)
215	struct mbuf *m;
216{
217	register struct mbuf *n;
218
219	MFREE(m, n);
220	return (n);
221}
222
223void
224m_freem(m)
225	register struct mbuf *m;
226{
227	register struct mbuf *n;
228
229	if (m == NULL)
230		return;
231	do {
232		MFREE(m, n);
233		m = n;
234	} while (m);
235}
236
237/*
238 * Mbuffer utility routines.
239 */
240
241/*
242 * Lesser-used path for M_PREPEND:
243 * allocate new mbuf to prepend to chain,
244 * copy junk along.
245 */
246struct mbuf *
247m_prepend(m, len, how)
248	register struct mbuf *m;
249	int len, how;
250{
251	struct mbuf *mn;
252
253	MGET(mn, how, m->m_type);
254	if (mn == (struct mbuf *)NULL) {
255		m_freem(m);
256		return ((struct mbuf *)NULL);
257	}
258	if (m->m_flags & M_PKTHDR) {
259		M_COPY_PKTHDR(mn, m);
260		m->m_flags &= ~M_PKTHDR;
261	}
262	mn->m_next = m;
263	m = mn;
264	if (len < MHLEN)
265		MH_ALIGN(m, len);
266	m->m_len = len;
267	return (m);
268}
269
270/*
271 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
272 * continuing for "len" bytes.  If len is M_COPYALL, copy to end of mbuf.
273 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
274 */
275int MCFail;
276
277struct mbuf *
278m_copym(m, off0, len, wait)
279	register struct mbuf *m;
280	int off0, wait;
281	register int len;
282{
283	register struct mbuf *n, **np;
284	register int off = off0;
285	struct mbuf *top;
286	int copyhdr = 0;
287
288	if (off < 0 || len < 0)
289		panic("m_copym");
290	if (off == 0 && m->m_flags & M_PKTHDR)
291		copyhdr = 1;
292	while (off > 0) {
293		if (m == 0)
294			panic("m_copym");
295		if (off < m->m_len)
296			break;
297		off -= m->m_len;
298		m = m->m_next;
299	}
300	np = &top;
301	top = 0;
302	while (len > 0) {
303		if (m == 0) {
304			if (len != M_COPYALL)
305				panic("m_copym");
306			break;
307		}
308		MGET(n, wait, m->m_type);
309		*np = n;
310		if (n == 0)
311			goto nospace;
312		if (copyhdr) {
313			M_COPY_PKTHDR(n, m);
314			if (len == M_COPYALL)
315				n->m_pkthdr.len -= off0;
316			else
317				n->m_pkthdr.len = len;
318			copyhdr = 0;
319		}
320		n->m_len = min(len, m->m_len - off);
321		if (m->m_flags & M_EXT) {
322			n->m_data = m->m_data + off;
323			mclrefcnt[mtocl(m->m_ext.ext_buf)]++;
324			n->m_ext = m->m_ext;
325			n->m_flags |= M_EXT;
326		} else
327			bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
328			    (unsigned)n->m_len);
329		if (len != M_COPYALL)
330			len -= n->m_len;
331		off = 0;
332		m = m->m_next;
333		np = &n->m_next;
334	}
335	if (top == 0)
336		MCFail++;
337	return (top);
338nospace:
339	m_freem(top);
340	MCFail++;
341	return (0);
342}
343
344/*
345 * Copy data from an mbuf chain starting "off" bytes from the beginning,
346 * continuing for "len" bytes, into the indicated buffer.
347 */
348void
349m_copydata(m, off, len, cp)
350	register struct mbuf *m;
351	register int off;
352	register int len;
353	caddr_t cp;
354{
355	register unsigned count;
356
357	if (off < 0 || len < 0)
358		panic("m_copydata");
359	while (off > 0) {
360		if (m == 0)
361			panic("m_copydata");
362		if (off < m->m_len)
363			break;
364		off -= m->m_len;
365		m = m->m_next;
366	}
367	while (len > 0) {
368		if (m == 0)
369			panic("m_copydata");
370		count = min(m->m_len - off, len);
371		bcopy(mtod(m, caddr_t) + off, cp, count);
372		len -= count;
373		cp += count;
374		off = 0;
375		m = m->m_next;
376	}
377}
378
379/*
380 * Concatenate mbuf chain n to m.
381 * Both chains must be of the same type (e.g. MT_DATA).
382 * Any m_pkthdr is not updated.
383 */
384void
385m_cat(m, n)
386	register struct mbuf *m, *n;
387{
388	while (m->m_next)
389		m = m->m_next;
390	while (n) {
391		if (m->m_flags & M_EXT ||
392		    m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
393			/* just join the two chains */
394			m->m_next = n;
395			return;
396		}
397		/* splat the data from one into the other */
398		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
399		    (u_int)n->m_len);
400		m->m_len += n->m_len;
401		n = m_free(n);
402	}
403}
404
405void
406m_adj(mp, req_len)
407	struct mbuf *mp;
408	int req_len;
409{
410	register int len = req_len;
411	register struct mbuf *m;
412	register count;
413
414	if ((m = mp) == NULL)
415		return;
416	if (len >= 0) {
417		/*
418		 * Trim from head.
419		 */
420		while (m != NULL && len > 0) {
421			if (m->m_len <= len) {
422				len -= m->m_len;
423				m->m_len = 0;
424				m = m->m_next;
425			} else {
426				m->m_len -= len;
427				m->m_data += len;
428				len = 0;
429			}
430		}
431		m = mp;
432		if (mp->m_flags & M_PKTHDR)
433			m->m_pkthdr.len -= (req_len - len);
434	} else {
435		/*
436		 * Trim from tail.  Scan the mbuf chain,
437		 * calculating its length and finding the last mbuf.
438		 * If the adjustment only affects this mbuf, then just
439		 * adjust and return.  Otherwise, rescan and truncate
440		 * after the remaining size.
441		 */
442		len = -len;
443		count = 0;
444		for (;;) {
445			count += m->m_len;
446			if (m->m_next == (struct mbuf *)0)
447				break;
448			m = m->m_next;
449		}
450		if (m->m_len >= len) {
451			m->m_len -= len;
452			if (mp->m_flags & M_PKTHDR)
453				mp->m_pkthdr.len -= len;
454			return;
455		}
456		count -= len;
457		if (count < 0)
458			count = 0;
459		/*
460		 * Correct length for chain is "count".
461		 * Find the mbuf with last data, adjust its length,
462		 * and toss data from remaining mbufs on chain.
463		 */
464		m = mp;
465		if (m->m_flags & M_PKTHDR)
466			m->m_pkthdr.len = count;
467		for (; m; m = m->m_next) {
468			if (m->m_len >= count) {
469				m->m_len = count;
470				break;
471			}
472			count -= m->m_len;
473		}
474		while (m->m_next)
475			(m = m->m_next) ->m_len = 0;
476	}
477}
478
479/*
480 * Rearange an mbuf chain so that len bytes are contiguous
481 * and in the data area of an mbuf (so that mtod and dtom
482 * will work for a structure of size len).  Returns the resulting
483 * mbuf chain on success, frees it and returns null on failure.
484 * If there is room, it will add up to max_protohdr-len extra bytes to the
485 * contiguous region in an attempt to avoid being called next time.
486 */
487int MPFail;
488
489struct mbuf *
490m_pullup(n, len)
491	register struct mbuf *n;
492	int len;
493{
494	register struct mbuf *m;
495	register int count;
496	int space;
497
498	/*
499	 * If first mbuf has no cluster, and has room for len bytes
500	 * without shifting current data, pullup into it,
501	 * otherwise allocate a new mbuf to prepend to the chain.
502	 */
503	if ((n->m_flags & M_EXT) == 0 &&
504	    n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
505		if (n->m_len >= len)
506			return (n);
507		m = n;
508		n = n->m_next;
509		len -= m->m_len;
510	} else {
511		if (len > MHLEN)
512			goto bad;
513		MGET(m, M_DONTWAIT, n->m_type);
514		if (m == 0)
515			goto bad;
516		m->m_len = 0;
517		if (n->m_flags & M_PKTHDR) {
518			M_COPY_PKTHDR(m, n);
519			n->m_flags &= ~M_PKTHDR;
520		}
521	}
522	space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
523	do {
524		count = min(min(max(len, max_protohdr), space), n->m_len);
525		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
526		  (unsigned)count);
527		len -= count;
528		m->m_len += count;
529		n->m_len -= count;
530		space -= count;
531		if (n->m_len)
532			n->m_data += count;
533		else
534			n = m_free(n);
535	} while (len > 0 && n);
536	if (len > 0) {
537		(void) m_free(m);
538		goto bad;
539	}
540	m->m_next = n;
541	return (m);
542bad:
543	m_freem(n);
544	MPFail++;
545	return (0);
546}
547
548/*
549 * Partition an mbuf chain in two pieces, returning the tail --
550 * all but the first len0 bytes.  In case of failure, it returns NULL and
551 * attempts to restore the chain to its original state.
552 */
553struct mbuf *
554m_split(m0, len0, wait)
555	register struct mbuf *m0;
556	int len0, wait;
557{
558	register struct mbuf *m, *n;
559	unsigned len = len0, remain;
560
561	for (m = m0; m && len > m->m_len; m = m->m_next)
562		len -= m->m_len;
563	if (m == 0)
564		return (0);
565	remain = m->m_len - len;
566	if (m0->m_flags & M_PKTHDR) {
567		MGETHDR(n, wait, m0->m_type);
568		if (n == 0)
569			return (0);
570		n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
571		n->m_pkthdr.len = m0->m_pkthdr.len - len0;
572		m0->m_pkthdr.len = len0;
573		if (m->m_flags & M_EXT)
574			goto extpacket;
575		if (remain > MHLEN) {
576			/* m can't be the lead packet */
577			MH_ALIGN(n, 0);
578			n->m_next = m_split(m, len, wait);
579			if (n->m_next == 0) {
580				(void) m_free(n);
581				return (0);
582			} else
583				return (n);
584		} else
585			MH_ALIGN(n, remain);
586	} else if (remain == 0) {
587		n = m->m_next;
588		m->m_next = 0;
589		return (n);
590	} else {
591		MGET(n, wait, m->m_type);
592		if (n == 0)
593			return (0);
594		M_ALIGN(n, remain);
595	}
596extpacket:
597	if (m->m_flags & M_EXT) {
598		n->m_flags |= M_EXT;
599		n->m_ext = m->m_ext;
600		mclrefcnt[mtocl(m->m_ext.ext_buf)]++;
601		m->m_ext.ext_size = 0; /* For Accounting XXXXXX danger */
602		n->m_data = m->m_data + len;
603	} else {
604		bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
605	}
606	n->m_len = remain;
607	m->m_len = len;
608	n->m_next = m->m_next;
609	m->m_next = 0;
610	return (n);
611}
612/*
613 * Routine to copy from device local memory into mbufs.
614 */
615struct mbuf *
616m_devget(buf, totlen, off0, ifp, copy)
617	char *buf;
618	int totlen, off0;
619	struct ifnet *ifp;
620	void (*copy)();
621{
622	register struct mbuf *m;
623	struct mbuf *top = 0, **mp = &top;
624	register int off = off0, len;
625	register char *cp;
626	char *epkt;
627
628	cp = buf;
629	epkt = cp + totlen;
630	if (off) {
631		cp += off + 2 * sizeof(u_short);
632		totlen -= 2 * sizeof(u_short);
633	}
634	MGETHDR(m, M_DONTWAIT, MT_DATA);
635	if (m == 0)
636		return (0);
637	m->m_pkthdr.rcvif = ifp;
638	m->m_pkthdr.len = totlen;
639	m->m_len = MHLEN;
640
641	while (totlen > 0) {
642		if (top) {
643			MGET(m, M_DONTWAIT, MT_DATA);
644			if (m == 0) {
645				m_freem(top);
646				return (0);
647			}
648			m->m_len = MLEN;
649		}
650		len = min(totlen, epkt - cp);
651		if (len >= MINCLSIZE) {
652			MCLGET(m, M_DONTWAIT);
653			if (m->m_flags & M_EXT)
654				m->m_len = len = min(len, MCLBYTES);
655			else
656				len = m->m_len;
657		} else {
658			/*
659			 * Place initial small packet/header at end of mbuf.
660			 */
661			if (len < m->m_len) {
662				if (top == 0 && len + max_linkhdr <= m->m_len)
663					m->m_data += max_linkhdr;
664				m->m_len = len;
665			} else
666				len = m->m_len;
667		}
668		if (copy)
669			copy(cp, mtod(m, caddr_t), (unsigned)len);
670		else
671			bcopy(cp, mtod(m, caddr_t), (unsigned)len);
672		cp += len;
673		*mp = m;
674		mp = &m->m_next;
675		totlen -= len;
676		if (cp == epkt)
677			cp = buf;
678	}
679	return (top);
680}
681
682/*
683 * Copy data from a buffer back into the indicated mbuf chain,
684 * starting "off" bytes from the beginning, extending the mbuf
685 * chain if necessary.
686 */
687void
688m_copyback(m0, off, len, cp)
689	struct	mbuf *m0;
690	register int off;
691	register int len;
692	caddr_t cp;
693{
694	register int mlen;
695	register struct mbuf *m = m0, *n;
696	int totlen = 0;
697
698	if (m0 == 0)
699		return;
700	while (off > (mlen = m->m_len)) {
701		off -= mlen;
702		totlen += mlen;
703		if (m->m_next == 0) {
704			n = m_getclr(M_DONTWAIT, m->m_type);
705			if (n == 0)
706				goto out;
707			n->m_len = min(MLEN, len + off);
708			m->m_next = n;
709		}
710		m = m->m_next;
711	}
712	while (len > 0) {
713		mlen = min (m->m_len - off, len);
714		bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen);
715		cp += mlen;
716		len -= mlen;
717		mlen += off;
718		off = 0;
719		totlen += mlen;
720		if (len == 0)
721			break;
722		if (m->m_next == 0) {
723			n = m_get(M_DONTWAIT, m->m_type);
724			if (n == 0)
725				break;
726			n->m_len = min(MLEN, len);
727			m->m_next = n;
728		}
729		m = m->m_next;
730	}
731out:	if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
732		m->m_pkthdr.len = totlen;
733}
734