uipc_mbuf.c revision 3451 
1/*
2 * Copyright (c) 1982, 1986, 1988, 1991, 1993
3 *	The Regents of the University of California.  All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 * 3. All advertising materials mentioning features or use of this software
14 *    must display the following acknowledgement:
15 *	This product includes software developed by the University of
16 *	California, Berkeley and its contributors.
17 * 4. Neither the name of the University nor the names of its contributors
18 *    may be used to endorse or promote products derived from this software
19 *    without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
32 *
33 *	@(#)uipc_mbuf.c	8.2 (Berkeley) 1/4/94
34 * $Id: uipc_mbuf.c,v 1.5 1994/10/04 06:50:01 phk Exp $
35 */
36
37#include <sys/param.h>
38#include <sys/systm.h>
39#include <sys/proc.h>
40#include <sys/malloc.h>
41#define MBTYPES
42#include <sys/mbuf.h>
43#include <sys/kernel.h>
44#include <sys/syslog.h>
45#include <sys/domain.h>
46#include <sys/protosw.h>
47
48#include <vm/vm.h>
49
50extern	vm_map_t mb_map;
51struct	mbuf *mbutl;
52char	*mclrefcnt;
53
54void
55mbinit()
56{
57	int s;
58
59#if CLBYTES < 4096
60#define NCL_INIT	(4096/CLBYTES)
61#else
62#define NCL_INIT	1
63#endif
64	s = splimp();
65	if (m_clalloc(NCL_INIT, M_DONTWAIT) == 0)
66		goto bad;
67	splx(s);
68	return;
69bad:
70	panic("mbinit");
71}
72
73/*
74 * Allocate some number of mbuf clusters
75 * and place on cluster free list.
76 * Must be called at splimp.
77 */
78/* ARGSUSED */
79int
80m_clalloc(ncl, nowait)
81	register int ncl;
82	int nowait;
83{
84	static int logged;
85	register caddr_t p;
86	register int i;
87	int npg;
88
89	npg = ncl * CLSIZE;
90	p = (caddr_t)kmem_malloc(mb_map, ctob(npg), !nowait);
91	if (p == NULL) {
92		if (logged == 0) {
93			logged++;
94			log(LOG_ERR, "mb_map full\n");
95		}
96		return (0);
97	}
98	ncl = ncl * CLBYTES / MCLBYTES;
99	for (i = 0; i < ncl; i++) {
100		((union mcluster *)p)->mcl_next = mclfree;
101		mclfree = (union mcluster *)p;
102		p += MCLBYTES;
103		mbstat.m_clfree++;
104	}
105	mbstat.m_clusters += ncl;
106	return (1);
107}
108
109/*
110 * When MGET failes, ask protocols to free space when short of memory,
111 * then re-attempt to allocate an mbuf.
112 */
113struct mbuf *
114m_retry(i, t)
115	int i, t;
116{
117	register struct mbuf *m;
118
119	m_reclaim();
120#define m_retry(i, t)	(struct mbuf *)0
121	MGET(m, i, t);
122#undef m_retry
123	return (m);
124}
125
126/*
127 * As above; retry an MGETHDR.
128 */
129struct mbuf *
130m_retryhdr(i, t)
131	int i, t;
132{
133	register struct mbuf *m;
134
135	m_reclaim();
136#define m_retryhdr(i, t) (struct mbuf *)0
137	MGETHDR(m, i, t);
138#undef m_retryhdr
139	return (m);
140}
141
142void
143m_reclaim()
144{
145	register struct domain *dp;
146	register struct protosw *pr;
147	int s = splimp();
148
149	for (dp = domains; dp; dp = dp->dom_next)
150		for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
151			if (pr->pr_drain)
152				(*pr->pr_drain)();
153	splx(s);
154	mbstat.m_drain++;
155}
156
157/*
158 * Space allocation routines.
159 * These are also available as macros
160 * for critical paths.
161 */
162struct mbuf *
163m_get(nowait, type)
164	int nowait, type;
165{
166	register struct mbuf *m;
167
168	MGET(m, nowait, type);
169	return (m);
170}
171
172struct mbuf *
173m_gethdr(nowait, type)
174	int nowait, type;
175{
176	register struct mbuf *m;
177
178	MGETHDR(m, nowait, type);
179	return (m);
180}
181
182struct mbuf *
183m_getclr(nowait, type)
184	int nowait, type;
185{
186	register struct mbuf *m;
187
188	MGET(m, nowait, type);
189	if (m == 0)
190		return (0);
191	bzero(mtod(m, caddr_t), MLEN);
192	return (m);
193}
194
195struct mbuf *
196m_free(m)
197	struct mbuf *m;
198{
199	register struct mbuf *n;
200
201	MFREE(m, n);
202	return (n);
203}
204
205void
206m_freem(m)
207	register struct mbuf *m;
208{
209	register struct mbuf *n;
210
211	if (m == NULL)
212		return;
213	do {
214		MFREE(m, n);
215		m = n;
216	} while (m);
217}
218
219/*
220 * Mbuffer utility routines.
221 */
222
223/*
224 * Lesser-used path for M_PREPEND:
225 * allocate new mbuf to prepend to chain,
226 * copy junk along.
227 */
228struct mbuf *
229m_prepend(m, len, how)
230	register struct mbuf *m;
231	int len, how;
232{
233	struct mbuf *mn;
234
235	MGET(mn, how, m->m_type);
236	if (mn == (struct mbuf *)NULL) {
237		m_freem(m);
238		return ((struct mbuf *)NULL);
239	}
240	if (m->m_flags & M_PKTHDR) {
241		M_COPY_PKTHDR(mn, m);
242		m->m_flags &= ~M_PKTHDR;
243	}
244	mn->m_next = m;
245	m = mn;
246	if (len < MHLEN)
247		MH_ALIGN(m, len);
248	m->m_len = len;
249	return (m);
250}
251
252/*
253 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
254 * continuing for "len" bytes.  If len is M_COPYALL, copy to end of mbuf.
255 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
256 */
257int MCFail;
258
259struct mbuf *
260m_copym(m, off0, len, wait)
261	register struct mbuf *m;
262	int off0, wait;
263	register int len;
264{
265	register struct mbuf *n, **np;
266	register int off = off0;
267	struct mbuf *top;
268	int copyhdr = 0;
269
270	if (off < 0 || len < 0)
271		panic("m_copym");
272	if (off == 0 && m->m_flags & M_PKTHDR)
273		copyhdr = 1;
274	while (off > 0) {
275		if (m == 0)
276			panic("m_copym");
277		if (off < m->m_len)
278			break;
279		off -= m->m_len;
280		m = m->m_next;
281	}
282	np = &top;
283	top = 0;
284	while (len > 0) {
285		if (m == 0) {
286			if (len != M_COPYALL)
287				panic("m_copym");
288			break;
289		}
290		MGET(n, wait, m->m_type);
291		*np = n;
292		if (n == 0)
293			goto nospace;
294		if (copyhdr) {
295			M_COPY_PKTHDR(n, m);
296			if (len == M_COPYALL)
297				n->m_pkthdr.len -= off0;
298			else
299				n->m_pkthdr.len = len;
300			copyhdr = 0;
301		}
302		n->m_len = min(len, m->m_len - off);
303		if (m->m_flags & M_EXT) {
304			n->m_data = m->m_data + off;
305			mclrefcnt[mtocl(m->m_ext.ext_buf)]++;
306			n->m_ext = m->m_ext;
307			n->m_flags |= M_EXT;
308		} else
309			bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
310			    (unsigned)n->m_len);
311		if (len != M_COPYALL)
312			len -= n->m_len;
313		off = 0;
314		m = m->m_next;
315		np = &n->m_next;
316	}
317	if (top == 0)
318		MCFail++;
319	return (top);
320nospace:
321	m_freem(top);
322	MCFail++;
323	return (0);
324}
325
326/*
327 * Copy data from an mbuf chain starting "off" bytes from the beginning,
328 * continuing for "len" bytes, into the indicated buffer.
329 */
330void
331m_copydata(m, off, len, cp)
332	register struct mbuf *m;
333	register int off;
334	register int len;
335	caddr_t cp;
336{
337	register unsigned count;
338
339	if (off < 0 || len < 0)
340		panic("m_copydata");
341	while (off > 0) {
342		if (m == 0)
343			panic("m_copydata");
344		if (off < m->m_len)
345			break;
346		off -= m->m_len;
347		m = m->m_next;
348	}
349	while (len > 0) {
350		if (m == 0)
351			panic("m_copydata");
352		count = min(m->m_len - off, len);
353		bcopy(mtod(m, caddr_t) + off, cp, count);
354		len -= count;
355		cp += count;
356		off = 0;
357		m = m->m_next;
358	}
359}
360
361/*
362 * Concatenate mbuf chain n to m.
363 * Both chains must be of the same type (e.g. MT_DATA).
364 * Any m_pkthdr is not updated.
365 */
366void
367m_cat(m, n)
368	register struct mbuf *m, *n;
369{
370	while (m->m_next)
371		m = m->m_next;
372	while (n) {
373		if (m->m_flags & M_EXT ||
374		    m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
375			/* just join the two chains */
376			m->m_next = n;
377			return;
378		}
379		/* splat the data from one into the other */
380		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
381		    (u_int)n->m_len);
382		m->m_len += n->m_len;
383		n = m_free(n);
384	}
385}
386
387void
388m_adj(mp, req_len)
389	struct mbuf *mp;
390	int req_len;
391{
392	register int len = req_len;
393	register struct mbuf *m;
394	register count;
395
396	if ((m = mp) == NULL)
397		return;
398	if (len >= 0) {
399		/*
400		 * Trim from head.
401		 */
402		while (m != NULL && len > 0) {
403			if (m->m_len <= len) {
404				len -= m->m_len;
405				m->m_len = 0;
406				m = m->m_next;
407			} else {
408				m->m_len -= len;
409				m->m_data += len;
410				len = 0;
411			}
412		}
413		m = mp;
414		if (mp->m_flags & M_PKTHDR)
415			m->m_pkthdr.len -= (req_len - len);
416	} else {
417		/*
418		 * Trim from tail.  Scan the mbuf chain,
419		 * calculating its length and finding the last mbuf.
420		 * If the adjustment only affects this mbuf, then just
421		 * adjust and return.  Otherwise, rescan and truncate
422		 * after the remaining size.
423		 */
424		len = -len;
425		count = 0;
426		for (;;) {
427			count += m->m_len;
428			if (m->m_next == (struct mbuf *)0)
429				break;
430			m = m->m_next;
431		}
432		if (m->m_len >= len) {
433			m->m_len -= len;
434			if (mp->m_flags & M_PKTHDR)
435				mp->m_pkthdr.len -= len;
436			return;
437		}
438		count -= len;
439		if (count < 0)
440			count = 0;
441		/*
442		 * Correct length for chain is "count".
443		 * Find the mbuf with last data, adjust its length,
444		 * and toss data from remaining mbufs on chain.
445		 */
446		m = mp;
447		if (m->m_flags & M_PKTHDR)
448			m->m_pkthdr.len = count;
449		for (; m; m = m->m_next) {
450			if (m->m_len >= count) {
451				m->m_len = count;
452				break;
453			}
454			count -= m->m_len;
455		}
456		while (m->m_next)
457			(m = m->m_next) ->m_len = 0;
458	}
459}
460
461/*
462 * Rearange an mbuf chain so that len bytes are contiguous
463 * and in the data area of an mbuf (so that mtod and dtom
464 * will work for a structure of size len).  Returns the resulting
465 * mbuf chain on success, frees it and returns null on failure.
466 * If there is room, it will add up to max_protohdr-len extra bytes to the
467 * contiguous region in an attempt to avoid being called next time.
468 */
469int MPFail;
470
471struct mbuf *
472m_pullup(n, len)
473	register struct mbuf *n;
474	int len;
475{
476	register struct mbuf *m;
477	register int count;
478	int space;
479
480	/*
481	 * If first mbuf has no cluster, and has room for len bytes
482	 * without shifting current data, pullup into it,
483	 * otherwise allocate a new mbuf to prepend to the chain.
484	 */
485	if ((n->m_flags & M_EXT) == 0 &&
486	    n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
487		if (n->m_len >= len)
488			return (n);
489		m = n;
490		n = n->m_next;
491		len -= m->m_len;
492	} else {
493		if (len > MHLEN)
494			goto bad;
495		MGET(m, M_DONTWAIT, n->m_type);
496		if (m == 0)
497			goto bad;
498		m->m_len = 0;
499		if (n->m_flags & M_PKTHDR) {
500			M_COPY_PKTHDR(m, n);
501			n->m_flags &= ~M_PKTHDR;
502		}
503	}
504	space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
505	do {
506		count = min(min(max(len, max_protohdr), space), n->m_len);
507		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
508		  (unsigned)count);
509		len -= count;
510		m->m_len += count;
511		n->m_len -= count;
512		space -= count;
513		if (n->m_len)
514			n->m_data += count;
515		else
516			n = m_free(n);
517	} while (len > 0 && n);
518	if (len > 0) {
519		(void) m_free(m);
520		goto bad;
521	}
522	m->m_next = n;
523	return (m);
524bad:
525	m_freem(n);
526	MPFail++;
527	return (0);
528}
529
530/*
531 * Partition an mbuf chain in two pieces, returning the tail --
532 * all but the first len0 bytes.  In case of failure, it returns NULL and
533 * attempts to restore the chain to its original state.
534 */
535struct mbuf *
536m_split(m0, len0, wait)
537	register struct mbuf *m0;
538	int len0, wait;
539{
540	register struct mbuf *m, *n;
541	unsigned len = len0, remain;
542
543	for (m = m0; m && len > m->m_len; m = m->m_next)
544		len -= m->m_len;
545	if (m == 0)
546		return (0);
547	remain = m->m_len - len;
548	if (m0->m_flags & M_PKTHDR) {
549		MGETHDR(n, wait, m0->m_type);
550		if (n == 0)
551			return (0);
552		n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
553		n->m_pkthdr.len = m0->m_pkthdr.len - len0;
554		m0->m_pkthdr.len = len0;
555		if (m->m_flags & M_EXT)
556			goto extpacket;
557		if (remain > MHLEN) {
558			/* m can't be the lead packet */
559			MH_ALIGN(n, 0);
560			n->m_next = m_split(m, len, wait);
561			if (n->m_next == 0) {
562				(void) m_free(n);
563				return (0);
564			} else
565				return (n);
566		} else
567			MH_ALIGN(n, remain);
568	} else if (remain == 0) {
569		n = m->m_next;
570		m->m_next = 0;
571		return (n);
572	} else {
573		MGET(n, wait, m->m_type);
574		if (n == 0)
575			return (0);
576		M_ALIGN(n, remain);
577	}
578extpacket:
579	if (m->m_flags & M_EXT) {
580		n->m_flags |= M_EXT;
581		n->m_ext = m->m_ext;
582		mclrefcnt[mtocl(m->m_ext.ext_buf)]++;
583		m->m_ext.ext_size = 0; /* For Accounting XXXXXX danger */
584		n->m_data = m->m_data + len;
585	} else {
586		bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
587	}
588	n->m_len = remain;
589	m->m_len = len;
590	n->m_next = m->m_next;
591	m->m_next = 0;
592	return (n);
593}
594/*
595 * Routine to copy from device local memory into mbufs.
596 */
597struct mbuf *
598m_devget(buf, totlen, off0, ifp, copy)
599	char *buf;
600	int totlen, off0;
601	struct ifnet *ifp;
602	void (*copy)();
603{
604	register struct mbuf *m;
605	struct mbuf *top = 0, **mp = &top;
606	register int off = off0, len;
607	register char *cp;
608	char *epkt;
609
610	cp = buf;
611	epkt = cp + totlen;
612	if (off) {
613		cp += off + 2 * sizeof(u_short);
614		totlen -= 2 * sizeof(u_short);
615	}
616	MGETHDR(m, M_DONTWAIT, MT_DATA);
617	if (m == 0)
618		return (0);
619	m->m_pkthdr.rcvif = ifp;
620	m->m_pkthdr.len = totlen;
621	m->m_len = MHLEN;
622
623	while (totlen > 0) {
624		if (top) {
625			MGET(m, M_DONTWAIT, MT_DATA);
626			if (m == 0) {
627				m_freem(top);
628				return (0);
629			}
630			m->m_len = MLEN;
631		}
632		len = min(totlen, epkt - cp);
633		if (len >= MINCLSIZE) {
634			MCLGET(m, M_DONTWAIT);
635			if (m->m_flags & M_EXT)
636				m->m_len = len = min(len, MCLBYTES);
637			else
638				len = m->m_len;
639		} else {
640			/*
641			 * Place initial small packet/header at end of mbuf.
642			 */
643			if (len < m->m_len) {
644				if (top == 0 && len + max_linkhdr <= m->m_len)
645					m->m_data += max_linkhdr;
646				m->m_len = len;
647			} else
648				len = m->m_len;
649		}
650		if (copy)
651			copy(cp, mtod(m, caddr_t), (unsigned)len);
652		else
653			bcopy(cp, mtod(m, caddr_t), (unsigned)len);
654		cp += len;
655		*mp = m;
656		mp = &m->m_next;
657		totlen -= len;
658		if (cp == epkt)
659			cp = buf;
660	}
661	return (top);
662}
663
664/*
665 * Copy data from a buffer back into the indicated mbuf chain,
666 * starting "off" bytes from the beginning, extending the mbuf
667 * chain if necessary.
668 */
669void
670m_copyback(m0, off, len, cp)
671	struct	mbuf *m0;
672	register int off;
673	register int len;
674	caddr_t cp;
675{
676	register int mlen;
677	register struct mbuf *m = m0, *n;
678	int totlen = 0;
679
680	if (m0 == 0)
681		return;
682	while (off > (mlen = m->m_len)) {
683		off -= mlen;
684		totlen += mlen;
685		if (m->m_next == 0) {
686			n = m_getclr(M_DONTWAIT, m->m_type);
687			if (n == 0)
688				goto out;
689			n->m_len = min(MLEN, len + off);
690			m->m_next = n;
691		}
692		m = m->m_next;
693	}
694	while (len > 0) {
695		mlen = min (m->m_len - off, len);
696		bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen);
697		cp += mlen;
698		len -= mlen;
699		mlen += off;
700		off = 0;
701		totlen += mlen;
702		if (len == 0)
703			break;
704		if (m->m_next == 0) {
705			n = m_get(M_DONTWAIT, m->m_type);
706			if (n == 0)
707				break;
708			n->m_len = min(MLEN, len);
709			m->m_next = n;
710		}
711		m = m->m_next;
712	}
713out:	if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
714		m->m_pkthdr.len = totlen;
715}
716