uipc_mbuf.c revision 1817 
1/*
2 * Copyright (c) 1982, 1986, 1988, 1991, 1993
3 *	The Regents of the University of California.  All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 * 3. All advertising materials mentioning features or use of this software
14 *    must display the following acknowledgement:
15 *	This product includes software developed by the University of
16 *	California, Berkeley and its contributors.
17 * 4. Neither the name of the University nor the names of its contributors
18 *    may be used to endorse or promote products derived from this software
19 *    without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
32 *
33 *	@(#)uipc_mbuf.c	8.2 (Berkeley) 1/4/94
34 * $Id$
35 */
36
37#include <sys/param.h>
38#include <sys/systm.h>
39#include <sys/proc.h>
40#include <sys/malloc.h>
41#include <sys/map.h>
42#define MBTYPES
43#include <sys/mbuf.h>
44#include <sys/kernel.h>
45#include <sys/syslog.h>
46#include <sys/domain.h>
47#include <sys/protosw.h>
48
49#include <vm/vm.h>
50
51void	m_reclaim	__P(());
52
53extern	vm_map_t mb_map;
54struct	mbuf *mbutl;
55char	*mclrefcnt;
56
57void
58mbinit()
59{
60	int s;
61
62#if CLBYTES < 4096
63#define NCL_INIT	(4096/CLBYTES)
64#else
65#define NCL_INIT	1
66#endif
67	s = splimp();
68	if (m_clalloc(NCL_INIT, M_DONTWAIT) == 0)
69		goto bad;
70	splx(s);
71	return;
72bad:
73	panic("mbinit");
74}
75
76/*
77 * Allocate some number of mbuf clusters
78 * and place on cluster free list.
79 * Must be called at splimp.
80 */
81/* ARGSUSED */
82int
83m_clalloc(ncl, nowait)
84	register int ncl;
85	int nowait;
86{
87	static int logged;
88	register caddr_t p;
89	register int i;
90	int npg;
91
92	npg = ncl * CLSIZE;
93	p = (caddr_t)kmem_malloc(mb_map, ctob(npg), !nowait);
94	if (p == NULL) {
95		if (logged == 0) {
96			logged++;
97			log(LOG_ERR, "mb_map full\n");
98		}
99		return (0);
100	}
101	ncl = ncl * CLBYTES / MCLBYTES;
102	for (i = 0; i < ncl; i++) {
103		((union mcluster *)p)->mcl_next = mclfree;
104		mclfree = (union mcluster *)p;
105		p += MCLBYTES;
106		mbstat.m_clfree++;
107	}
108	mbstat.m_clusters += ncl;
109	return (1);
110}
111
112/*
113 * When MGET failes, ask protocols to free space when short of memory,
114 * then re-attempt to allocate an mbuf.
115 */
116struct mbuf *
117m_retry(i, t)
118	int i, t;
119{
120	register struct mbuf *m;
121
122	m_reclaim();
123#define m_retry(i, t)	(struct mbuf *)0
124	MGET(m, i, t);
125#undef m_retry
126	return (m);
127}
128
129/*
130 * As above; retry an MGETHDR.
131 */
132struct mbuf *
133m_retryhdr(i, t)
134	int i, t;
135{
136	register struct mbuf *m;
137
138	m_reclaim();
139#define m_retryhdr(i, t) (struct mbuf *)0
140	MGETHDR(m, i, t);
141#undef m_retryhdr
142	return (m);
143}
144
145void
146m_reclaim()
147{
148	register struct domain *dp;
149	register struct protosw *pr;
150	int s = splimp();
151
152	for (dp = domains; dp; dp = dp->dom_next)
153		for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
154			if (pr->pr_drain)
155				(*pr->pr_drain)();
156	splx(s);
157	mbstat.m_drain++;
158}
159
160/*
161 * Space allocation routines.
162 * These are also available as macros
163 * for critical paths.
164 */
165struct mbuf *
166m_get(nowait, type)
167	int nowait, type;
168{
169	register struct mbuf *m;
170
171	MGET(m, nowait, type);
172	return (m);
173}
174
175struct mbuf *
176m_gethdr(nowait, type)
177	int nowait, type;
178{
179	register struct mbuf *m;
180
181	MGETHDR(m, nowait, type);
182	return (m);
183}
184
185struct mbuf *
186m_getclr(nowait, type)
187	int nowait, type;
188{
189	register struct mbuf *m;
190
191	MGET(m, nowait, type);
192	if (m == 0)
193		return (0);
194	bzero(mtod(m, caddr_t), MLEN);
195	return (m);
196}
197
198struct mbuf *
199m_free(m)
200	struct mbuf *m;
201{
202	register struct mbuf *n;
203
204	MFREE(m, n);
205	return (n);
206}
207
208void
209m_freem(m)
210	register struct mbuf *m;
211{
212	register struct mbuf *n;
213
214	if (m == NULL)
215		return;
216	do {
217		MFREE(m, n);
218	} while (m = n);
219}
220
221/*
222 * Mbuffer utility routines.
223 */
224
225/*
226 * Lesser-used path for M_PREPEND:
227 * allocate new mbuf to prepend to chain,
228 * copy junk along.
229 */
230struct mbuf *
231m_prepend(m, len, how)
232	register struct mbuf *m;
233	int len, how;
234{
235	struct mbuf *mn;
236
237	MGET(mn, how, m->m_type);
238	if (mn == (struct mbuf *)NULL) {
239		m_freem(m);
240		return ((struct mbuf *)NULL);
241	}
242	if (m->m_flags & M_PKTHDR) {
243		M_COPY_PKTHDR(mn, m);
244		m->m_flags &= ~M_PKTHDR;
245	}
246	mn->m_next = m;
247	m = mn;
248	if (len < MHLEN)
249		MH_ALIGN(m, len);
250	m->m_len = len;
251	return (m);
252}
253
254/*
255 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
256 * continuing for "len" bytes.  If len is M_COPYALL, copy to end of mbuf.
257 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
258 */
259int MCFail;
260
261struct mbuf *
262m_copym(m, off0, len, wait)
263	register struct mbuf *m;
264	int off0, wait;
265	register int len;
266{
267	register struct mbuf *n, **np;
268	register int off = off0;
269	struct mbuf *top;
270	int copyhdr = 0;
271
272	if (off < 0 || len < 0)
273		panic("m_copym");
274	if (off == 0 && m->m_flags & M_PKTHDR)
275		copyhdr = 1;
276	while (off > 0) {
277		if (m == 0)
278			panic("m_copym");
279		if (off < m->m_len)
280			break;
281		off -= m->m_len;
282		m = m->m_next;
283	}
284	np = &top;
285	top = 0;
286	while (len > 0) {
287		if (m == 0) {
288			if (len != M_COPYALL)
289				panic("m_copym");
290			break;
291		}
292		MGET(n, wait, m->m_type);
293		*np = n;
294		if (n == 0)
295			goto nospace;
296		if (copyhdr) {
297			M_COPY_PKTHDR(n, m);
298			if (len == M_COPYALL)
299				n->m_pkthdr.len -= off0;
300			else
301				n->m_pkthdr.len = len;
302			copyhdr = 0;
303		}
304		n->m_len = min(len, m->m_len - off);
305		if (m->m_flags & M_EXT) {
306			n->m_data = m->m_data + off;
307			mclrefcnt[mtocl(m->m_ext.ext_buf)]++;
308			n->m_ext = m->m_ext;
309			n->m_flags |= M_EXT;
310		} else
311			bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
312			    (unsigned)n->m_len);
313		if (len != M_COPYALL)
314			len -= n->m_len;
315		off = 0;
316		m = m->m_next;
317		np = &n->m_next;
318	}
319	if (top == 0)
320		MCFail++;
321	return (top);
322nospace:
323	m_freem(top);
324	MCFail++;
325	return (0);
326}
327
328/*
329 * Copy data from an mbuf chain starting "off" bytes from the beginning,
330 * continuing for "len" bytes, into the indicated buffer.
331 */
332void
333m_copydata(m, off, len, cp)
334	register struct mbuf *m;
335	register int off;
336	register int len;
337	caddr_t cp;
338{
339	register unsigned count;
340
341	if (off < 0 || len < 0)
342		panic("m_copydata");
343	while (off > 0) {
344		if (m == 0)
345			panic("m_copydata");
346		if (off < m->m_len)
347			break;
348		off -= m->m_len;
349		m = m->m_next;
350	}
351	while (len > 0) {
352		if (m == 0)
353			panic("m_copydata");
354		count = min(m->m_len - off, len);
355		bcopy(mtod(m, caddr_t) + off, cp, count);
356		len -= count;
357		cp += count;
358		off = 0;
359		m = m->m_next;
360	}
361}
362
363/*
364 * Concatenate mbuf chain n to m.
365 * Both chains must be of the same type (e.g. MT_DATA).
366 * Any m_pkthdr is not updated.
367 */
368void
369m_cat(m, n)
370	register struct mbuf *m, *n;
371{
372	while (m->m_next)
373		m = m->m_next;
374	while (n) {
375		if (m->m_flags & M_EXT ||
376		    m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
377			/* just join the two chains */
378			m->m_next = n;
379			return;
380		}
381		/* splat the data from one into the other */
382		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
383		    (u_int)n->m_len);
384		m->m_len += n->m_len;
385		n = m_free(n);
386	}
387}
388
389void
390m_adj(mp, req_len)
391	struct mbuf *mp;
392	int req_len;
393{
394	register int len = req_len;
395	register struct mbuf *m;
396	register count;
397
398	if ((m = mp) == NULL)
399		return;
400	if (len >= 0) {
401		/*
402		 * Trim from head.
403		 */
404		while (m != NULL && len > 0) {
405			if (m->m_len <= len) {
406				len -= m->m_len;
407				m->m_len = 0;
408				m = m->m_next;
409			} else {
410				m->m_len -= len;
411				m->m_data += len;
412				len = 0;
413			}
414		}
415		m = mp;
416		if (mp->m_flags & M_PKTHDR)
417			m->m_pkthdr.len -= (req_len - len);
418	} else {
419		/*
420		 * Trim from tail.  Scan the mbuf chain,
421		 * calculating its length and finding the last mbuf.
422		 * If the adjustment only affects this mbuf, then just
423		 * adjust and return.  Otherwise, rescan and truncate
424		 * after the remaining size.
425		 */
426		len = -len;
427		count = 0;
428		for (;;) {
429			count += m->m_len;
430			if (m->m_next == (struct mbuf *)0)
431				break;
432			m = m->m_next;
433		}
434		if (m->m_len >= len) {
435			m->m_len -= len;
436			if (mp->m_flags & M_PKTHDR)
437				mp->m_pkthdr.len -= len;
438			return;
439		}
440		count -= len;
441		if (count < 0)
442			count = 0;
443		/*
444		 * Correct length for chain is "count".
445		 * Find the mbuf with last data, adjust its length,
446		 * and toss data from remaining mbufs on chain.
447		 */
448		m = mp;
449		if (m->m_flags & M_PKTHDR)
450			m->m_pkthdr.len = count;
451		for (; m; m = m->m_next) {
452			if (m->m_len >= count) {
453				m->m_len = count;
454				break;
455			}
456			count -= m->m_len;
457		}
458		while (m = m->m_next)
459			m->m_len = 0;
460	}
461}
462
463/*
464 * Rearange an mbuf chain so that len bytes are contiguous
465 * and in the data area of an mbuf (so that mtod and dtom
466 * will work for a structure of size len).  Returns the resulting
467 * mbuf chain on success, frees it and returns null on failure.
468 * If there is room, it will add up to max_protohdr-len extra bytes to the
469 * contiguous region in an attempt to avoid being called next time.
470 */
471int MPFail;
472
473struct mbuf *
474m_pullup(n, len)
475	register struct mbuf *n;
476	int len;
477{
478	register struct mbuf *m;
479	register int count;
480	int space;
481
482	/*
483	 * If first mbuf has no cluster, and has room for len bytes
484	 * without shifting current data, pullup into it,
485	 * otherwise allocate a new mbuf to prepend to the chain.
486	 */
487	if ((n->m_flags & M_EXT) == 0 &&
488	    n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
489		if (n->m_len >= len)
490			return (n);
491		m = n;
492		n = n->m_next;
493		len -= m->m_len;
494	} else {
495		if (len > MHLEN)
496			goto bad;
497		MGET(m, M_DONTWAIT, n->m_type);
498		if (m == 0)
499			goto bad;
500		m->m_len = 0;
501		if (n->m_flags & M_PKTHDR) {
502			M_COPY_PKTHDR(m, n);
503			n->m_flags &= ~M_PKTHDR;
504		}
505	}
506	space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
507	do {
508		count = min(min(max(len, max_protohdr), space), n->m_len);
509		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
510		  (unsigned)count);
511		len -= count;
512		m->m_len += count;
513		n->m_len -= count;
514		space -= count;
515		if (n->m_len)
516			n->m_data += count;
517		else
518			n = m_free(n);
519	} while (len > 0 && n);
520	if (len > 0) {
521		(void) m_free(m);
522		goto bad;
523	}
524	m->m_next = n;
525	return (m);
526bad:
527	m_freem(n);
528	MPFail++;
529	return (0);
530}
531
532/*
533 * Partition an mbuf chain in two pieces, returning the tail --
534 * all but the first len0 bytes.  In case of failure, it returns NULL and
535 * attempts to restore the chain to its original state.
536 */
537struct mbuf *
538m_split(m0, len0, wait)
539	register struct mbuf *m0;
540	int len0, wait;
541{
542	register struct mbuf *m, *n;
543	unsigned len = len0, remain;
544
545	for (m = m0; m && len > m->m_len; m = m->m_next)
546		len -= m->m_len;
547	if (m == 0)
548		return (0);
549	remain = m->m_len - len;
550	if (m0->m_flags & M_PKTHDR) {
551		MGETHDR(n, wait, m0->m_type);
552		if (n == 0)
553			return (0);
554		n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
555		n->m_pkthdr.len = m0->m_pkthdr.len - len0;
556		m0->m_pkthdr.len = len0;
557		if (m->m_flags & M_EXT)
558			goto extpacket;
559		if (remain > MHLEN) {
560			/* m can't be the lead packet */
561			MH_ALIGN(n, 0);
562			n->m_next = m_split(m, len, wait);
563			if (n->m_next == 0) {
564				(void) m_free(n);
565				return (0);
566			} else
567				return (n);
568		} else
569			MH_ALIGN(n, remain);
570	} else if (remain == 0) {
571		n = m->m_next;
572		m->m_next = 0;
573		return (n);
574	} else {
575		MGET(n, wait, m->m_type);
576		if (n == 0)
577			return (0);
578		M_ALIGN(n, remain);
579	}
580extpacket:
581	if (m->m_flags & M_EXT) {
582		n->m_flags |= M_EXT;
583		n->m_ext = m->m_ext;
584		mclrefcnt[mtocl(m->m_ext.ext_buf)]++;
585		m->m_ext.ext_size = 0; /* For Accounting XXXXXX danger */
586		n->m_data = m->m_data + len;
587	} else {
588		bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
589	}
590	n->m_len = remain;
591	m->m_len = len;
592	n->m_next = m->m_next;
593	m->m_next = 0;
594	return (n);
595}
596/*
597 * Routine to copy from device local memory into mbufs.
598 */
599struct mbuf *
600m_devget(buf, totlen, off0, ifp, copy)
601	char *buf;
602	int totlen, off0;
603	struct ifnet *ifp;
604	void (*copy)();
605{
606	register struct mbuf *m;
607	struct mbuf *top = 0, **mp = &top;
608	register int off = off0, len;
609	register char *cp;
610	char *epkt;
611
612	cp = buf;
613	epkt = cp + totlen;
614	if (off) {
615		cp += off + 2 * sizeof(u_short);
616		totlen -= 2 * sizeof(u_short);
617	}
618	MGETHDR(m, M_DONTWAIT, MT_DATA);
619	if (m == 0)
620		return (0);
621	m->m_pkthdr.rcvif = ifp;
622	m->m_pkthdr.len = totlen;
623	m->m_len = MHLEN;
624
625	while (totlen > 0) {
626		if (top) {
627			MGET(m, M_DONTWAIT, MT_DATA);
628			if (m == 0) {
629				m_freem(top);
630				return (0);
631			}
632			m->m_len = MLEN;
633		}
634		len = min(totlen, epkt - cp);
635		if (len >= MINCLSIZE) {
636			MCLGET(m, M_DONTWAIT);
637			if (m->m_flags & M_EXT)
638				m->m_len = len = min(len, MCLBYTES);
639			else
640				len = m->m_len;
641		} else {
642			/*
643			 * Place initial small packet/header at end of mbuf.
644			 */
645			if (len < m->m_len) {
646				if (top == 0 && len + max_linkhdr <= m->m_len)
647					m->m_data += max_linkhdr;
648				m->m_len = len;
649			} else
650				len = m->m_len;
651		}
652		if (copy)
653			copy(cp, mtod(m, caddr_t), (unsigned)len);
654		else
655			bcopy(cp, mtod(m, caddr_t), (unsigned)len);
656		cp += len;
657		*mp = m;
658		mp = &m->m_next;
659		totlen -= len;
660		if (cp == epkt)
661			cp = buf;
662	}
663	return (top);
664}
665