1/*
2 * Copyright (c) 1982, 1986, 1988, 1991, 1993
3 * The Regents of the University of California. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. All advertising materials mentioning features or use of this software
14 * must display the following acknowledgement:
15 * This product includes software developed by the University of
16 * California, Berkeley and its contributors.
17 * 4. Neither the name of the University nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
32 *
33 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94
34 * $FreeBSD: head/sys/kern/uipc_mbuf.c 77843 2001-06-06 22:17:08Z peter $
35 */
36
37#include "opt_param.h"
38#include <sys/param.h>
39#include <sys/systm.h>
40#include <sys/condvar.h>
41#include <sys/kernel.h>
42#include <sys/lock.h>
43#include <sys/malloc.h>
44#include <sys/mbuf.h>
45#include <sys/mutex.h>
46#include <sys/sysctl.h>
47#include <sys/domain.h>
48#include <sys/protosw.h>
49
50#include <vm/vm.h>
51#include <vm/vm_kern.h>
52#include <vm/vm_extern.h>
53
54#ifndef NMBCLUSTERS
55#define NMBCLUSTERS (512 + MAXUSERS * 16)
56#endif
57
58static void mbinit(void *);
59SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbinit, NULL)
60
61struct mbuf *mbutl;
62struct mbstat mbstat;
63u_long mbtypes[MT_NTYPES];
64int max_linkhdr;
65int max_protohdr;
66int max_hdr;
67int max_datalen;
68int nmbclusters = NMBCLUSTERS;
69int nmbufs = NMBCLUSTERS * 4;
70int nmbcnt;
71u_long m_mballoc_wid = 0;
72u_long m_clalloc_wid = 0;
73
74/*
75 * freelist header structures...
76 * mbffree_lst, mclfree_lst, mcntfree_lst
77 */
78struct mbffree_lst mmbfree;
79struct mclfree_lst mclfree;
80struct mcntfree_lst mcntfree;
81struct mtx mbuf_mtx;
82
83/*
84 * sysctl(8) exported objects
85 */
86SYSCTL_DECL(_kern_ipc);
87SYSCTL_INT(_kern_ipc, KIPC_MAX_LINKHDR, max_linkhdr, CTLFLAG_RW,
88 &max_linkhdr, 0, "");
89SYSCTL_INT(_kern_ipc, KIPC_MAX_PROTOHDR, max_protohdr, CTLFLAG_RW,
90 &max_protohdr, 0, "");
91SYSCTL_INT(_kern_ipc, KIPC_MAX_HDR, max_hdr, CTLFLAG_RW, &max_hdr, 0, "");
92SYSCTL_INT(_kern_ipc, KIPC_MAX_DATALEN, max_datalen, CTLFLAG_RW,
93 &max_datalen, 0, "");
94SYSCTL_INT(_kern_ipc, OID_AUTO, mbuf_wait, CTLFLAG_RW,
95 &mbuf_wait, 0, "");
96SYSCTL_STRUCT(_kern_ipc, KIPC_MBSTAT, mbstat, CTLFLAG_RD, &mbstat, mbstat, "");
97SYSCTL_OPAQUE(_kern_ipc, OID_AUTO, mbtypes, CTLFLAG_RD, mbtypes,
98 sizeof(mbtypes), "LU", "");
99SYSCTL_INT(_kern_ipc, KIPC_NMBCLUSTERS, nmbclusters, CTLFLAG_RD,
100 &nmbclusters, 0, "Maximum number of mbuf clusters available");
101SYSCTL_INT(_kern_ipc, OID_AUTO, nmbufs, CTLFLAG_RD, &nmbufs, 0,
102 "Maximum number of mbufs available");
103SYSCTL_INT(_kern_ipc, OID_AUTO, nmbcnt, CTLFLAG_RD, &nmbcnt, 0,
104 "Maximum number of ext_buf counters available");
105
106TUNABLE_INT("kern.ipc.nmbclusters", &nmbclusters);
107TUNABLE_INT("kern.ipc.nmbufs", &nmbufs);
108TUNABLE_INT("kern.ipc.nmbcnt", &nmbcnt);
109
110static void m_reclaim(void);
111
112/* Initial allocation numbers */
113#define NCL_INIT 2
114#define NMB_INIT 16
115#define REF_INIT NMBCLUSTERS
116
117/*
118 * Full mbuf subsystem initialization done here.
119 *
120 * XXX: If ever we have system specific map setups to do, then move them to
121 * machdep.c - for now, there is no reason for this stuff to go there.
122 */
123static void
124mbinit(void *dummy)
125{
126 vm_offset_t maxaddr;
127 vm_size_t mb_map_size;
128
129 /* Sanity checks and pre-initialization for non-constants */
130 if (nmbufs < nmbclusters * 2)
131 nmbufs = nmbclusters * 2;
132 if (nmbcnt == 0)
133 nmbcnt = EXT_COUNTERS;
134
135 /*
136 * Setup the mb_map, allocate requested VM space.
137 */
138 mb_map_size = (vm_size_t)(nmbufs * MSIZE + nmbclusters * MCLBYTES +
139 nmbcnt * sizeof(union mext_refcnt));
140 mb_map_size = rounddown(mb_map_size, PAGE_SIZE);
141 mb_map = kmem_suballoc(kmem_map, (vm_offset_t *)&mbutl, &maxaddr,
142 mb_map_size);
143 /* XXX XXX XXX: mb_map->system_map = 1; */
144
145 /*
146 * Initialize the free list headers, and setup locks for lists.
147 */
148 mmbfree.m_head = NULL;
149 mclfree.m_head = NULL;
150 mcntfree.m_head = NULL;
151 mtx_init(&mbuf_mtx, "mbuf free list lock", MTX_DEF);
152 cv_init(&mmbfree.m_starved, "mbuf free list starved cv");
153 cv_init(&mclfree.m_starved, "mbuf cluster free list starved cv");
154
155 /*
156 * Initialize mbuf subsystem (sysctl exported) statistics structure.
157 */
158 mbstat.m_msize = MSIZE;
159 mbstat.m_mclbytes = MCLBYTES;
160 mbstat.m_minclsize = MINCLSIZE;
161 mbstat.m_mlen = MLEN;
162 mbstat.m_mhlen = MHLEN;
163
164 /*
165 * Perform some initial allocations.
166 */
167 mtx_lock(&mbuf_mtx);
168 if (m_alloc_ref(REF_INIT, M_DONTWAIT) == 0)
169 goto bad;
170 if (m_mballoc(NMB_INIT, M_DONTWAIT) == 0)
171 goto bad;
172 if (m_clalloc(NCL_INIT, M_DONTWAIT) == 0)
173 goto bad;
174 mtx_unlock(&mbuf_mtx);
175
176 return;
177bad:
178 panic("mbinit: failed to initialize mbuf subsystem!");
179}
180
181/*
182 * Allocate at least nmb reference count structs and place them
183 * on the ref cnt free list.
184 *
185 * Must be called with the mcntfree lock held.
186 */
187int
188m_alloc_ref(u_int nmb, int how)
189{
190 caddr_t p;
191 u_int nbytes;
192 int i;
193
194 /*
195 * We don't cap the amount of memory that can be used
196 * by the reference counters, like we do for mbufs and
197 * mbuf clusters. In fact, we're absolutely sure that we
198 * won't ever be going over our allocated space. We keep enough
199 * space in mb_map to accomodate maximum values of allocatable
200 * external buffers including, but not limited to, clusters.
201 * (That's also why we won't have to have wait routines for
202 * counters).
203 *
204 * If we're in here, we're absolutely certain to be returning
205 * succesfully, as long as there is physical memory to accomodate
206 * us. And if there isn't, but we're willing to wait, then
207 * kmem_malloc() will do the only waiting needed.
208 */
209
210 nbytes = round_page(nmb * sizeof(union mext_refcnt));
211 if (1 /* XXX: how == M_TRYWAIT */)
212 mtx_unlock(&mbuf_mtx);
213 if ((p = (caddr_t)kmem_malloc(mb_map, nbytes, how == M_TRYWAIT ?
214 M_WAITOK : M_NOWAIT)) == NULL) {
215 if (1 /* XXX: how == M_TRYWAIT */)
216 mtx_lock(&mbuf_mtx);
217 return (0);
218 }
219 nmb = nbytes / sizeof(union mext_refcnt);
220
221 /*
222 * We don't let go of the mutex in order to avoid a race.
223 * It is up to the caller to let go of the mutex.
224 */
225 if (1 /* XXX: how == M_TRYWAIT */)
226 mtx_lock(&mbuf_mtx);
227 for (i = 0; i < nmb; i++) {
228 ((union mext_refcnt *)p)->next_ref = mcntfree.m_head;
229 mcntfree.m_head = (union mext_refcnt *)p;
230 p += sizeof(union mext_refcnt);
231 mbstat.m_refree++;
232 }
233 mbstat.m_refcnt += nmb;
234
235 return (1);
236}
237
238/*
239 * Allocate at least nmb mbufs and place on mbuf free list.
240 *
241 * Must be called with the mmbfree lock held.
242 */
243int
244m_mballoc(int nmb, int how)
245{
246 caddr_t p;
247 int i;
248 int nbytes;
249
250 nbytes = round_page(nmb * MSIZE);
251 nmb = nbytes / MSIZE;
252
253 /*
254 * If we've hit the mbuf limit, stop allocating from mb_map.
255 * Also, once we run out of map space, it will be impossible to
256 * get any more (nothing is ever freed back to the map).
257 */
258 if (mb_map_full || ((nmb + mbstat.m_mbufs) > nmbufs))
259 return (0);
260
261 if (1 /* XXX: how == M_TRYWAIT */)
262 mtx_unlock(&mbuf_mtx);
263 p = (caddr_t)kmem_malloc(mb_map, nbytes, how == M_TRYWAIT ?
264 M_WAITOK : M_NOWAIT);
265 if (1 /* XXX: how == M_TRYWAIT */) {
266 mtx_lock(&mbuf_mtx);
267 if (p == NULL)
268 mbstat.m_wait++;
269 }
270
271 /*
272 * Either the map is now full, or `how' is M_DONTWAIT and there
273 * are no pages left.
274 */
275 if (p == NULL)
276 return (0);
277
278 /*
279 * We don't let go of the mutex in order to avoid a race.
280 * It is up to the caller to let go of the mutex when done
281 * with grabbing the mbuf from the free list.
282 */
283 for (i = 0; i < nmb; i++) {
284 ((struct mbuf *)p)->m_next = mmbfree.m_head;
285 mmbfree.m_head = (struct mbuf *)p;
286 p += MSIZE;
287 }
288 mbstat.m_mbufs += nmb;
289 mbtypes[MT_FREE] += nmb;
290 return (1);
291}
292
293/*
294 * Once the mb_map has been exhausted and if the call to the allocation macros
295 * (or, in some cases, functions) is with M_TRYWAIT, then it is necessary to
296 * rely solely on reclaimed mbufs.
297 *
298 * Here we request for the protocols to free up some resources and, if we
299 * still cannot get anything, then we wait for an mbuf to be freed for a
300 * designated (mbuf_wait) time, at most.
301 *
302 * Must be called with the mmbfree mutex held.
303 */
304struct mbuf *
305m_mballoc_wait(void)
306{
307 struct mbuf *p = NULL;
308
309 /*
310 * See if we can drain some resources out of the protocols.
311 * We drop the mmbfree mutex to avoid recursing into it in some of
312 * the drain routines. Clearly, we're faced with a race here because
313 * once something is freed during the drain, it may be grabbed right
314 * from under us by some other thread. But we accept this possibility
315 * in order to avoid a potentially large lock recursion and, more
316 * importantly, to avoid a potential lock order reversal which may
317 * result in deadlock (See comment above m_reclaim()).
318 */
319 mtx_unlock(&mbuf_mtx);
320 m_reclaim();
321
322 mtx_lock(&mbuf_mtx);
323 _MGET(p, M_DONTWAIT);
324
325 if (p == NULL) {
326 int retval;
327
328 m_mballoc_wid++;
329 retval = cv_timedwait(&mmbfree.m_starved, &mbuf_mtx,
330 mbuf_wait);
331 m_mballoc_wid--;
332
333 /*
334 * If we got signaled (i.e. didn't time out), allocate.
335 */
336 if (retval == 0)
337 _MGET(p, M_DONTWAIT);
338 }
339
340 if (p != NULL) {
341 mbstat.m_wait++;
342 if (mmbfree.m_head != NULL)
343 MBWAKEUP(m_mballoc_wid, &mmbfree.m_starved);
344 }
345
346 return (p);
347}
348
349/*
350 * Allocate some number of mbuf clusters
351 * and place on cluster free list.
352 *
353 * Must be called with the mclfree lock held.
354 */
355int
356m_clalloc(int ncl, int how)
357{
358 caddr_t p;
359 int i;
360 int npg_sz;
361
362 npg_sz = round_page(ncl * MCLBYTES);
363 ncl = npg_sz / MCLBYTES;
364
365 /*
366 * If the map is now full (nothing will ever be freed to it).
367 * If we've hit the mcluster number limit, stop allocating from
368 * mb_map.
369 */
370 if (mb_map_full || ((ncl + mbstat.m_clusters) > nmbclusters))
371 return (0);
372
373 if (1 /* XXX: how == M_TRYWAIT */)
374 mtx_unlock(&mbuf_mtx);
375 p = (caddr_t)kmem_malloc(mb_map, npg_sz,
376 how == M_TRYWAIT ? M_WAITOK : M_NOWAIT);
377 if (1 /* XXX: how == M_TRYWAIT */)
378 mtx_lock(&mbuf_mtx);
379
380 /*
381 * Either the map is now full, or `how' is M_DONTWAIT and there
382 * are no pages left.
383 */
384 if (p == NULL)
385 return (0);
386
387 for (i = 0; i < ncl; i++) {
388 ((union mcluster *)p)->mcl_next = mclfree.m_head;
389 mclfree.m_head = (union mcluster *)p;
390 p += MCLBYTES;
391 mbstat.m_clfree++;
392 }
393 mbstat.m_clusters += ncl;
394 return (1);
395}
396
397/*
398 * Once the mb_map submap has been exhausted and the allocation is called with
399 * M_TRYWAIT, we rely on the mclfree list. If nothing is free, we will
400 * block on a cv for a designated amount of time (mbuf_wait) or until we're
401 * signaled due to sudden mcluster availability.
402 *
403 * Must be called with the mclfree lock held.
404 */
405caddr_t
406m_clalloc_wait(void)
407{
408 caddr_t p = NULL;
409 int retval;
410
411 m_clalloc_wid++;
412 retval = cv_timedwait(&mclfree.m_starved, &mbuf_mtx, mbuf_wait);
413 m_clalloc_wid--;
414
415 /*
416 * Now that we (think) that we've got something, try again.
417 */
418 if (retval == 0)
419 _MCLALLOC(p, M_DONTWAIT);
420
421 if (p != NULL) {
422 mbstat.m_wait++;
423 if (mclfree.m_head != NULL)
424 MBWAKEUP(m_clalloc_wid, &mclfree.m_starved);
425 }
426
427 return (p);
428}
429
430/*
431 * m_reclaim: drain protocols in hopes to free up some resources...
432 *
433 * XXX: No locks should be held going in here. The drain routines have
434 * to presently acquire some locks which raises the possibility of lock
435 * order violation if we're holding any mutex if that mutex is acquired in
436 * reverse order relative to one of the locks in the drain routines.
437 */
438static void
439m_reclaim(void)
440{
441 struct domain *dp;
442 struct protosw *pr;
443
444#ifdef WITNESS
445 KASSERT(witness_list(curproc) == 0,
446 ("m_reclaim called with locks held"));
447#endif
448
449 for (dp = domains; dp; dp = dp->dom_next)
450 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
451 if (pr->pr_drain)
452 (*pr->pr_drain)();
453 mbstat.m_drain++;
454}
455
456/*
457 * Space allocation routines.
458 * Some of these are also available as macros
459 * for critical paths.
460 */
461struct mbuf *
462m_get(int how, int type)
463{
464 struct mbuf *m;
465
466 MGET(m, how, type);
467 return (m);
468}
469
470struct mbuf *
471m_gethdr(int how, int type)
472{
473 struct mbuf *m;
474
475 MGETHDR(m, how, type);
476 return (m);
477}
478
479struct mbuf *
480m_getclr(int how, int type)
481{
482 struct mbuf *m;
483
484 MGET(m, how, type);
485 if (m != NULL)
486 bzero(mtod(m, caddr_t), MLEN);
487 return (m);
488}
489
490struct mbuf *
491m_free(struct mbuf *m)
492{
493 struct mbuf *n;
494
495 MFREE(m, n);
496 return (n);
497}
498
499/*
500 * struct mbuf *
501 * m_getm(m, len, how, type)
502 *
503 * This will allocate len-worth of mbufs and/or mbuf clusters (whatever fits
504 * best) and return a pointer to the top of the allocated chain. If m is
505 * non-null, then we assume that it is a single mbuf or an mbuf chain to
506 * which we want len bytes worth of mbufs and/or clusters attached, and so
507 * if we succeed in allocating it, we will just return a pointer to m.
508 *
509 * If we happen to fail at any point during the allocation, we will free
510 * up everything we have already allocated and return NULL.
511 *
512 */
513struct mbuf *
514m_getm(struct mbuf *m, int len, int how, int type)
515{
516 struct mbuf *top, *tail, *mp, *mtail = NULL;
517
518 KASSERT(len >= 0, ("len is < 0 in m_getm"));
519
520 MGET(mp, how, type);
521 if (mp == NULL)
522 return (NULL);
523 else if (len > MINCLSIZE) {
524 MCLGET(mp, how);
525 if ((mp->m_flags & M_EXT) == 0) {
526 m_free(mp);
527 return (NULL);
528 }
529 }
530 mp->m_len = 0;
531 len -= M_TRAILINGSPACE(mp);
532
533 if (m != NULL)
534 for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next);
535 else
536 m = mp;
537
538 top = tail = mp;
539 while (len > 0) {
540 MGET(mp, how, type);
541 if (mp == NULL)
542 goto failed;
543
544 tail->m_next = mp;
545 tail = mp;
546 if (len > MINCLSIZE) {
547 MCLGET(mp, how);
548 if ((mp->m_flags & M_EXT) == 0)
549 goto failed;
550 }
551
552 mp->m_len = 0;
553 len -= M_TRAILINGSPACE(mp);
554 }
555
556 if (mtail != NULL)
557 mtail->m_next = top;
558 return (m);
559
560failed:
561 m_freem(top);
562 return (NULL);
563}
564
565void
566m_freem(struct mbuf *m)
567{
568 struct mbuf *n;
569
570 if (m == NULL)
571 return;
572 do {
573 /*
574 * we do need to check non-first mbuf, since some of existing
575 * code does not call M_PREPEND properly.
576 * (example: call to bpf_mtap from drivers)
577 */
578 if ((m->m_flags & M_PKTHDR) != 0 && m->m_pkthdr.aux) {
579 m_freem(m->m_pkthdr.aux);
580 m->m_pkthdr.aux = NULL;
581 }
582 MFREE(m, n);
583 m = n;
584 } while (m);
585}
586
587/*
588 * Lesser-used path for M_PREPEND:
589 * allocate new mbuf to prepend to chain,
590 * copy junk along.
591 */
592struct mbuf *
593m_prepend(struct mbuf *m, int len, int how)
594{
595 struct mbuf *mn;
596
597 MGET(mn, how, m->m_type);
598 if (mn == NULL) {
599 m_freem(m);
600 return (NULL);
601 }
602 if (m->m_flags & M_PKTHDR) {
603 M_COPY_PKTHDR(mn, m);
604 m->m_flags &= ~M_PKTHDR;
605 }
606 mn->m_next = m;
607 m = mn;
608 if (len < MHLEN)
609 MH_ALIGN(m, len);
610 m->m_len = len;
611 return (m);
612}
613
614/*
615 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
616 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
617 * The wait parameter is a choice of M_TRYWAIT/M_DONTWAIT from caller.
618 * Note that the copy is read-only, because clusters are not copied,
619 * only their reference counts are incremented.
620 */
621struct mbuf *
622m_copym(struct mbuf *m, int off0, int len, int wait)
623{
624 struct mbuf *n, **np;
625 int off = off0;
626 struct mbuf *top;
627 int copyhdr = 0;
628
629 KASSERT(off >= 0, ("m_copym, negative off %d", off));
630 KASSERT(len >= 0, ("m_copym, negative len %d", len));
631 if (off == 0 && m->m_flags & M_PKTHDR)
632 copyhdr = 1;
633 while (off > 0) {
634 KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain"));
635 if (off < m->m_len)
636 break;
637 off -= m->m_len;
638 m = m->m_next;
639 }
640 np = ⊤
641 top = 0;
642 while (len > 0) {
643 if (m == NULL) {
644 KASSERT(len == M_COPYALL,
645 ("m_copym, length > size of mbuf chain"));
646 break;
647 }
648 MGET(n, wait, m->m_type);
649 *np = n;
650 if (n == NULL)
651 goto nospace;
652 if (copyhdr) {
653 M_COPY_PKTHDR(n, m);
654 if (len == M_COPYALL)
655 n->m_pkthdr.len -= off0;
656 else
657 n->m_pkthdr.len = len;
658 copyhdr = 0;
659 }
660 n->m_len = min(len, m->m_len - off);
661 if (m->m_flags & M_EXT) {
662 n->m_data = m->m_data + off;
663 n->m_ext = m->m_ext;
664 n->m_flags |= M_EXT;
665 MEXT_ADD_REF(m);
666 } else
667 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
668 (unsigned)n->m_len);
669 if (len != M_COPYALL)
670 len -= n->m_len;
671 off = 0;
672 m = m->m_next;
673 np = &n->m_next;
674 }
675 if (top == NULL) {
676 mtx_lock(&mbuf_mtx);
677 mbstat.m_mcfail++;
678 mtx_unlock(&mbuf_mtx);
679 }
680 return (top);
681nospace:
682 m_freem(top);
683 mtx_lock(&mbuf_mtx);
684 mbstat.m_mcfail++;
685 mtx_unlock(&mbuf_mtx);
686 return (NULL);
687}
688
689/*
690 * Copy an entire packet, including header (which must be present).
691 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'.
692 * Note that the copy is read-only, because clusters are not copied,
693 * only their reference counts are incremented.
694 * Preserve alignment of the first mbuf so if the creator has left
695 * some room at the beginning (e.g. for inserting protocol headers)
696 * the copies still have the room available.
697 */
698struct mbuf *
699m_copypacket(struct mbuf *m, int how)
700{
701 struct mbuf *top, *n, *o;
702
703 MGET(n, how, m->m_type);
704 top = n;
705 if (n == NULL)
706 goto nospace;
707
708 M_COPY_PKTHDR(n, m);
709 n->m_len = m->m_len;
710 if (m->m_flags & M_EXT) {
711 n->m_data = m->m_data;
712 n->m_ext = m->m_ext;
713 n->m_flags |= M_EXT;
714 MEXT_ADD_REF(m);
715 } else {
716 n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat );
717 bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
718 }
719
720 m = m->m_next;
721 while (m) {
722 MGET(o, how, m->m_type);
723 if (o == NULL)
724 goto nospace;
725
726 n->m_next = o;
727 n = n->m_next;
728
729 n->m_len = m->m_len;
730 if (m->m_flags & M_EXT) {
731 n->m_data = m->m_data;
732 n->m_ext = m->m_ext;
733 n->m_flags |= M_EXT;
734 MEXT_ADD_REF(m);
735 } else {
736 bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
737 }
738
739 m = m->m_next;
740 }
741 return top;
742nospace:
743 m_freem(top);
744 mtx_lock(&mbuf_mtx);
745 mbstat.m_mcfail++;
746 mtx_unlock(&mbuf_mtx);
747 return (NULL);
748}
749
750/*
751 * Copy data from an mbuf chain starting "off" bytes from the beginning,
752 * continuing for "len" bytes, into the indicated buffer.
753 */
754void
755m_copydata(struct mbuf *m, int off, int len, caddr_t cp)
756{
757 unsigned count;
758
759 KASSERT(off >= 0, ("m_copydata, negative off %d", off));
760 KASSERT(len >= 0, ("m_copydata, negative len %d", len));
761 while (off > 0) {
762 KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain"));
763 if (off < m->m_len)
764 break;
765 off -= m->m_len;
766 m = m->m_next;
767 }
768 while (len > 0) {
769 KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain"));
770 count = min(m->m_len - off, len);
771 bcopy(mtod(m, caddr_t) + off, cp, count);
772 len -= count;
773 cp += count;
774 off = 0;
775 m = m->m_next;
776 }
777}
778
779/*
780 * Copy a packet header mbuf chain into a completely new chain, including
781 * copying any mbuf clusters. Use this instead of m_copypacket() when
782 * you need a writable copy of an mbuf chain.
783 */
784struct mbuf *
785m_dup(struct mbuf *m, int how)
786{
787 struct mbuf **p, *top = NULL;
788 int remain, moff, nsize;
789
790 /* Sanity check */
791 if (m == NULL)
792 return (NULL);
793 KASSERT((m->m_flags & M_PKTHDR) != 0, ("%s: !PKTHDR", __FUNCTION__));
794
795 /* While there's more data, get a new mbuf, tack it on, and fill it */
796 remain = m->m_pkthdr.len;
797 moff = 0;
798 p = ⊤
799 while (remain > 0 || top == NULL) { /* allow m->m_pkthdr.len == 0 */
800 struct mbuf *n;
801
802 /* Get the next new mbuf */
803 MGET(n, how, m->m_type);
804 if (n == NULL)
805 goto nospace;
806 if (top == NULL) { /* first one, must be PKTHDR */
807 M_COPY_PKTHDR(n, m);
808 nsize = MHLEN;
809 } else /* not the first one */
810 nsize = MLEN;
811 if (remain >= MINCLSIZE) {
812 MCLGET(n, how);
813 if ((n->m_flags & M_EXT) == 0) {
814 (void)m_free(n);
815 goto nospace;
816 }
817 nsize = MCLBYTES;
818 }
819 n->m_len = 0;
820
821 /* Link it into the new chain */
822 *p = n;
823 p = &n->m_next;
824
825 /* Copy data from original mbuf(s) into new mbuf */
826 while (n->m_len < nsize && m != NULL) {
827 int chunk = min(nsize - n->m_len, m->m_len - moff);
828
829 bcopy(m->m_data + moff, n->m_data + n->m_len, chunk);
830 moff += chunk;
831 n->m_len += chunk;
832 remain -= chunk;
833 if (moff == m->m_len) {
834 m = m->m_next;
835 moff = 0;
836 }
837 }
838
839 /* Check correct total mbuf length */
840 KASSERT((remain > 0 && m != NULL) || (remain == 0 && m == NULL),
841 ("%s: bogus m_pkthdr.len", __FUNCTION__));
842 }
843 return (top);
844
845nospace:
846 m_freem(top);
847 mtx_lock(&mbuf_mtx);
848 mbstat.m_mcfail++;
849 mtx_unlock(&mbuf_mtx);
850 return (NULL);
851}
852
853/*
854 * Concatenate mbuf chain n to m.
855 * Both chains must be of the same type (e.g. MT_DATA).
856 * Any m_pkthdr is not updated.
857 */
858void
859m_cat(struct mbuf *m, struct mbuf *n)
860{
861 while (m->m_next)
862 m = m->m_next;
863 while (n) {
864 if (m->m_flags & M_EXT ||
865 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
866 /* just join the two chains */
867 m->m_next = n;
868 return;
869 }
870 /* splat the data from one into the other */
871 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
872 (u_int)n->m_len);
873 m->m_len += n->m_len;
874 n = m_free(n);
875 }
876}
877
878void
879m_adj(struct mbuf *mp, int req_len)
880{
881 int len = req_len;
882 struct mbuf *m;
883 int count;
884
885 if ((m = mp) == NULL)
886 return;
887 if (len >= 0) {
888 /*
889 * Trim from head.
890 */
891 while (m != NULL && len > 0) {
892 if (m->m_len <= len) {
893 len -= m->m_len;
894 m->m_len = 0;
895 m = m->m_next;
896 } else {
897 m->m_len -= len;
898 m->m_data += len;
899 len = 0;
900 }
901 }
902 m = mp;
903 if (mp->m_flags & M_PKTHDR)
904 m->m_pkthdr.len -= (req_len - len);
905 } else {
906 /*
907 * Trim from tail. Scan the mbuf chain,
908 * calculating its length and finding the last mbuf.
909 * If the adjustment only affects this mbuf, then just
910 * adjust and return. Otherwise, rescan and truncate
911 * after the remaining size.
912 */
913 len = -len;
914 count = 0;
915 for (;;) {
916 count += m->m_len;
917 if (m->m_next == (struct mbuf *)0)
918 break;
919 m = m->m_next;
920 }
921 if (m->m_len >= len) {
922 m->m_len -= len;
923 if (mp->m_flags & M_PKTHDR)
924 mp->m_pkthdr.len -= len;
925 return;
926 }
927 count -= len;
928 if (count < 0)
929 count = 0;
930 /*
931 * Correct length for chain is "count".
932 * Find the mbuf with last data, adjust its length,
933 * and toss data from remaining mbufs on chain.
934 */
935 m = mp;
936 if (m->m_flags & M_PKTHDR)
937 m->m_pkthdr.len = count;
938 for (; m; m = m->m_next) {
939 if (m->m_len >= count) {
940 m->m_len = count;
941 break;
942 }
943 count -= m->m_len;
944 }
945 while (m->m_next)
946 (m = m->m_next) ->m_len = 0;
947 }
948}
949
950/*
951 * Rearange an mbuf chain so that len bytes are contiguous
952 * and in the data area of an mbuf (so that mtod and dtom
953 * will work for a structure of size len). Returns the resulting
954 * mbuf chain on success, frees it and returns null on failure.
955 * If there is room, it will add up to max_protohdr-len extra bytes to the
956 * contiguous region in an attempt to avoid being called next time.
957 */
958struct mbuf *
959m_pullup(struct mbuf *n, int len)
960{
961 struct mbuf *m;
962 int count;
963 int space;
964
965 /*
966 * If first mbuf has no cluster, and has room for len bytes
967 * without shifting current data, pullup into it,
968 * otherwise allocate a new mbuf to prepend to the chain.
969 */
970 if ((n->m_flags & M_EXT) == 0 &&
971 n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
972 if (n->m_len >= len)
973 return (n);
974 m = n;
975 n = n->m_next;
976 len -= m->m_len;
977 } else {
978 if (len > MHLEN)
979 goto bad;
980 MGET(m, M_DONTWAIT, n->m_type);
981 if (m == NULL)
982 goto bad;
983 m->m_len = 0;
984 if (n->m_flags & M_PKTHDR) {
985 M_COPY_PKTHDR(m, n);
986 n->m_flags &= ~M_PKTHDR;
987 }
988 }
989 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
990 do {
991 count = min(min(max(len, max_protohdr), space), n->m_len);
992 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
993 (unsigned)count);
994 len -= count;
995 m->m_len += count;
996 n->m_len -= count;
997 space -= count;
998 if (n->m_len)
999 n->m_data += count;
1000 else
1001 n = m_free(n);
1002 } while (len > 0 && n);
1003 if (len > 0) {
1004 (void) m_free(m);
1005 goto bad;
1006 }
1007 m->m_next = n;
1008 return (m);
1009bad:
1010 m_freem(n);
1011 mtx_lock(&mbuf_mtx);
1012 mbstat.m_mpfail++;
1013 mtx_unlock(&mbuf_mtx);
1014 return (NULL);
1015}
1016
1017/*
1018 * Partition an mbuf chain in two pieces, returning the tail --
1019 * all but the first len0 bytes. In case of failure, it returns NULL and
1020 * attempts to restore the chain to its original state.
1021 */
1022struct mbuf *
1023m_split(struct mbuf *m0, int len0, int wait)
1024{
1025 struct mbuf *m, *n;
1026 unsigned len = len0, remain;
1027
1028 for (m = m0; m && len > m->m_len; m = m->m_next)
1029 len -= m->m_len;
1030 if (m == NULL)
1031 return (NULL);
1032 remain = m->m_len - len;
1033 if (m0->m_flags & M_PKTHDR) {
1034 MGETHDR(n, wait, m0->m_type);
1035 if (n == NULL)
1036 return (NULL);
1037 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1038 n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1039 m0->m_pkthdr.len = len0;
1040 if (m->m_flags & M_EXT)
1041 goto extpacket;
1042 if (remain > MHLEN) {
1043 /* m can't be the lead packet */
1044 MH_ALIGN(n, 0);
1045 n->m_next = m_split(m, len, wait);
1046 if (n->m_next == NULL) {
1047 (void) m_free(n);
1048 return (NULL);
1049 } else
1050 return (n);
1051 } else
1052 MH_ALIGN(n, remain);
1053 } else if (remain == 0) {
1054 n = m->m_next;
1055 m->m_next = NULL;
1056 return (n);
1057 } else {
1058 MGET(n, wait, m->m_type);
1059 if (n == NULL)
1060 return (NULL);
1061 M_ALIGN(n, remain);
1062 }
1063extpacket:
1064 if (m->m_flags & M_EXT) {
1065 n->m_flags |= M_EXT;
1066 n->m_ext = m->m_ext;
1067 MEXT_ADD_REF(m);
1068 m->m_ext.ext_size = 0; /* For Accounting XXXXXX danger */
1069 n->m_data = m->m_data + len;
1070 } else {
1071 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
1072 }
1073 n->m_len = remain;
1074 m->m_len = len;
1075 n->m_next = m->m_next;
1076 m->m_next = NULL;
1077 return (n);
1078}
1079/*
1080 * Routine to copy from device local memory into mbufs.
1081 */
1082struct mbuf *
1083m_devget(char *buf, int totlen, int off0, struct ifnet *ifp,
1084 void (*copy)(char *from, caddr_t to, u_int len))
1085{
1086 struct mbuf *m;
1087 struct mbuf *top = 0, **mp = ⊤
1088 int off = off0, len;
1089 char *cp;
1090 char *epkt;
1091
1092 cp = buf;
1093 epkt = cp + totlen;
1094 if (off) {
1095 cp += off + 2 * sizeof(u_short);
1096 totlen -= 2 * sizeof(u_short);
1097 }
1098 MGETHDR(m, M_DONTWAIT, MT_DATA);
1099 if (m == NULL)
1100 return (NULL);
1101 m->m_pkthdr.rcvif = ifp;
1102 m->m_pkthdr.len = totlen;
1103 m->m_len = MHLEN;
1104
1105 while (totlen > 0) {
1106 if (top) {
1107 MGET(m, M_DONTWAIT, MT_DATA);
1108 if (m == NULL) {
1109 m_freem(top);
1110 return (NULL);
1111 }
1112 m->m_len = MLEN;
1113 }
1114 len = min(totlen, epkt - cp);
1115 if (len >= MINCLSIZE) {
1116 MCLGET(m, M_DONTWAIT);
1117 if (m->m_flags & M_EXT)
1118 m->m_len = len = min(len, MCLBYTES);
1119 else
1120 len = m->m_len;
1121 } else {
1122 /*
1123 * Place initial small packet/header at end of mbuf.
1124 */
1125 if (len < m->m_len) {
1126 if (top == NULL && len +
1127 max_linkhdr <= m->m_len)
1128 m->m_data += max_linkhdr;
1129 m->m_len = len;
1130 } else
1131 len = m->m_len;
1132 }
1133 if (copy)
1134 copy(cp, mtod(m, caddr_t), (unsigned)len);
1135 else
1136 bcopy(cp, mtod(m, caddr_t), (unsigned)len);
1137 cp += len;
1138 *mp = m;
1139 mp = &m->m_next;
1140 totlen -= len;
1141 if (cp == epkt)
1142 cp = buf;
1143 }
1144 return (top);
1145}
1146
1147/*
1148 * Copy data from a buffer back into the indicated mbuf chain,
1149 * starting "off" bytes from the beginning, extending the mbuf
1150 * chain if necessary.
1151 */
1152void
1153m_copyback(struct mbuf *m0, int off, int len, caddr_t cp)
1154{
1155 int mlen;
1156 struct mbuf *m = m0, *n;
1157 int totlen = 0;
1158
1159 if (m0 == NULL)
1160 return;
1161 while (off > (mlen = m->m_len)) {
1162 off -= mlen;
1163 totlen += mlen;
1164 if (m->m_next == NULL) {
1165 n = m_getclr(M_DONTWAIT, m->m_type);
1166 if (n == NULL)
1167 goto out;
1168 n->m_len = min(MLEN, len + off);
1169 m->m_next = n;
1170 }
1171 m = m->m_next;
1172 }
1173 while (len > 0) {
1174 mlen = min (m->m_len - off, len);
1175 bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen);
1176 cp += mlen;
1177 len -= mlen;
1178 mlen += off;
1179 off = 0;
1180 totlen += mlen;
1181 if (len == 0)
1182 break;
1183 if (m->m_next == NULL) {
1184 n = m_get(M_DONTWAIT, m->m_type);
1185 if (n == NULL)
1186 break;
1187 n->m_len = min(MLEN, len);
1188 m->m_next = n;
1189 }
1190 m = m->m_next;
1191 }
1192out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
1193 m->m_pkthdr.len = totlen;
1194}
1195
1196void
1197m_print(const struct mbuf *m)
1198{
1199 int len;
1200 const struct mbuf *m2;
1201
1202 len = m->m_pkthdr.len;
1203 m2 = m;
1204 while (len) {
1205 printf("%p %*D\n", m2, m2->m_len, (u_char *)m2->m_data, "-");
1206 len -= m2->m_len;
1207 m2 = m2->m_next;
1208 }
1209 return;
1210}
2 * Copyright (c) 1982, 1986, 1988, 1991, 1993
3 * The Regents of the University of California. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. All advertising materials mentioning features or use of this software
14 * must display the following acknowledgement:
15 * This product includes software developed by the University of
16 * California, Berkeley and its contributors.
17 * 4. Neither the name of the University nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
32 *
33 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94
34 * $FreeBSD: head/sys/kern/uipc_mbuf.c 77843 2001-06-06 22:17:08Z peter $
35 */
36
37#include "opt_param.h"
38#include <sys/param.h>
39#include <sys/systm.h>
40#include <sys/condvar.h>
41#include <sys/kernel.h>
42#include <sys/lock.h>
43#include <sys/malloc.h>
44#include <sys/mbuf.h>
45#include <sys/mutex.h>
46#include <sys/sysctl.h>
47#include <sys/domain.h>
48#include <sys/protosw.h>
49
50#include <vm/vm.h>
51#include <vm/vm_kern.h>
52#include <vm/vm_extern.h>
53
54#ifndef NMBCLUSTERS
55#define NMBCLUSTERS (512 + MAXUSERS * 16)
56#endif
57
58static void mbinit(void *);
59SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbinit, NULL)
60
61struct mbuf *mbutl;
62struct mbstat mbstat;
63u_long mbtypes[MT_NTYPES];
64int max_linkhdr;
65int max_protohdr;
66int max_hdr;
67int max_datalen;
68int nmbclusters = NMBCLUSTERS;
69int nmbufs = NMBCLUSTERS * 4;
70int nmbcnt;
71u_long m_mballoc_wid = 0;
72u_long m_clalloc_wid = 0;
73
74/*
75 * freelist header structures...
76 * mbffree_lst, mclfree_lst, mcntfree_lst
77 */
78struct mbffree_lst mmbfree;
79struct mclfree_lst mclfree;
80struct mcntfree_lst mcntfree;
81struct mtx mbuf_mtx;
82
83/*
84 * sysctl(8) exported objects
85 */
86SYSCTL_DECL(_kern_ipc);
87SYSCTL_INT(_kern_ipc, KIPC_MAX_LINKHDR, max_linkhdr, CTLFLAG_RW,
88 &max_linkhdr, 0, "");
89SYSCTL_INT(_kern_ipc, KIPC_MAX_PROTOHDR, max_protohdr, CTLFLAG_RW,
90 &max_protohdr, 0, "");
91SYSCTL_INT(_kern_ipc, KIPC_MAX_HDR, max_hdr, CTLFLAG_RW, &max_hdr, 0, "");
92SYSCTL_INT(_kern_ipc, KIPC_MAX_DATALEN, max_datalen, CTLFLAG_RW,
93 &max_datalen, 0, "");
94SYSCTL_INT(_kern_ipc, OID_AUTO, mbuf_wait, CTLFLAG_RW,
95 &mbuf_wait, 0, "");
96SYSCTL_STRUCT(_kern_ipc, KIPC_MBSTAT, mbstat, CTLFLAG_RD, &mbstat, mbstat, "");
97SYSCTL_OPAQUE(_kern_ipc, OID_AUTO, mbtypes, CTLFLAG_RD, mbtypes,
98 sizeof(mbtypes), "LU", "");
99SYSCTL_INT(_kern_ipc, KIPC_NMBCLUSTERS, nmbclusters, CTLFLAG_RD,
100 &nmbclusters, 0, "Maximum number of mbuf clusters available");
101SYSCTL_INT(_kern_ipc, OID_AUTO, nmbufs, CTLFLAG_RD, &nmbufs, 0,
102 "Maximum number of mbufs available");
103SYSCTL_INT(_kern_ipc, OID_AUTO, nmbcnt, CTLFLAG_RD, &nmbcnt, 0,
104 "Maximum number of ext_buf counters available");
105
106TUNABLE_INT("kern.ipc.nmbclusters", &nmbclusters);
107TUNABLE_INT("kern.ipc.nmbufs", &nmbufs);
108TUNABLE_INT("kern.ipc.nmbcnt", &nmbcnt);
109
110static void m_reclaim(void);
111
112/* Initial allocation numbers */
113#define NCL_INIT 2
114#define NMB_INIT 16
115#define REF_INIT NMBCLUSTERS
116
117/*
118 * Full mbuf subsystem initialization done here.
119 *
120 * XXX: If ever we have system specific map setups to do, then move them to
121 * machdep.c - for now, there is no reason for this stuff to go there.
122 */
123static void
124mbinit(void *dummy)
125{
126 vm_offset_t maxaddr;
127 vm_size_t mb_map_size;
128
129 /* Sanity checks and pre-initialization for non-constants */
130 if (nmbufs < nmbclusters * 2)
131 nmbufs = nmbclusters * 2;
132 if (nmbcnt == 0)
133 nmbcnt = EXT_COUNTERS;
134
135 /*
136 * Setup the mb_map, allocate requested VM space.
137 */
138 mb_map_size = (vm_size_t)(nmbufs * MSIZE + nmbclusters * MCLBYTES +
139 nmbcnt * sizeof(union mext_refcnt));
140 mb_map_size = rounddown(mb_map_size, PAGE_SIZE);
141 mb_map = kmem_suballoc(kmem_map, (vm_offset_t *)&mbutl, &maxaddr,
142 mb_map_size);
143 /* XXX XXX XXX: mb_map->system_map = 1; */
144
145 /*
146 * Initialize the free list headers, and setup locks for lists.
147 */
148 mmbfree.m_head = NULL;
149 mclfree.m_head = NULL;
150 mcntfree.m_head = NULL;
151 mtx_init(&mbuf_mtx, "mbuf free list lock", MTX_DEF);
152 cv_init(&mmbfree.m_starved, "mbuf free list starved cv");
153 cv_init(&mclfree.m_starved, "mbuf cluster free list starved cv");
154
155 /*
156 * Initialize mbuf subsystem (sysctl exported) statistics structure.
157 */
158 mbstat.m_msize = MSIZE;
159 mbstat.m_mclbytes = MCLBYTES;
160 mbstat.m_minclsize = MINCLSIZE;
161 mbstat.m_mlen = MLEN;
162 mbstat.m_mhlen = MHLEN;
163
164 /*
165 * Perform some initial allocations.
166 */
167 mtx_lock(&mbuf_mtx);
168 if (m_alloc_ref(REF_INIT, M_DONTWAIT) == 0)
169 goto bad;
170 if (m_mballoc(NMB_INIT, M_DONTWAIT) == 0)
171 goto bad;
172 if (m_clalloc(NCL_INIT, M_DONTWAIT) == 0)
173 goto bad;
174 mtx_unlock(&mbuf_mtx);
175
176 return;
177bad:
178 panic("mbinit: failed to initialize mbuf subsystem!");
179}
180
181/*
182 * Allocate at least nmb reference count structs and place them
183 * on the ref cnt free list.
184 *
185 * Must be called with the mcntfree lock held.
186 */
187int
188m_alloc_ref(u_int nmb, int how)
189{
190 caddr_t p;
191 u_int nbytes;
192 int i;
193
194 /*
195 * We don't cap the amount of memory that can be used
196 * by the reference counters, like we do for mbufs and
197 * mbuf clusters. In fact, we're absolutely sure that we
198 * won't ever be going over our allocated space. We keep enough
199 * space in mb_map to accomodate maximum values of allocatable
200 * external buffers including, but not limited to, clusters.
201 * (That's also why we won't have to have wait routines for
202 * counters).
203 *
204 * If we're in here, we're absolutely certain to be returning
205 * succesfully, as long as there is physical memory to accomodate
206 * us. And if there isn't, but we're willing to wait, then
207 * kmem_malloc() will do the only waiting needed.
208 */
209
210 nbytes = round_page(nmb * sizeof(union mext_refcnt));
211 if (1 /* XXX: how == M_TRYWAIT */)
212 mtx_unlock(&mbuf_mtx);
213 if ((p = (caddr_t)kmem_malloc(mb_map, nbytes, how == M_TRYWAIT ?
214 M_WAITOK : M_NOWAIT)) == NULL) {
215 if (1 /* XXX: how == M_TRYWAIT */)
216 mtx_lock(&mbuf_mtx);
217 return (0);
218 }
219 nmb = nbytes / sizeof(union mext_refcnt);
220
221 /*
222 * We don't let go of the mutex in order to avoid a race.
223 * It is up to the caller to let go of the mutex.
224 */
225 if (1 /* XXX: how == M_TRYWAIT */)
226 mtx_lock(&mbuf_mtx);
227 for (i = 0; i < nmb; i++) {
228 ((union mext_refcnt *)p)->next_ref = mcntfree.m_head;
229 mcntfree.m_head = (union mext_refcnt *)p;
230 p += sizeof(union mext_refcnt);
231 mbstat.m_refree++;
232 }
233 mbstat.m_refcnt += nmb;
234
235 return (1);
236}
237
238/*
239 * Allocate at least nmb mbufs and place on mbuf free list.
240 *
241 * Must be called with the mmbfree lock held.
242 */
243int
244m_mballoc(int nmb, int how)
245{
246 caddr_t p;
247 int i;
248 int nbytes;
249
250 nbytes = round_page(nmb * MSIZE);
251 nmb = nbytes / MSIZE;
252
253 /*
254 * If we've hit the mbuf limit, stop allocating from mb_map.
255 * Also, once we run out of map space, it will be impossible to
256 * get any more (nothing is ever freed back to the map).
257 */
258 if (mb_map_full || ((nmb + mbstat.m_mbufs) > nmbufs))
259 return (0);
260
261 if (1 /* XXX: how == M_TRYWAIT */)
262 mtx_unlock(&mbuf_mtx);
263 p = (caddr_t)kmem_malloc(mb_map, nbytes, how == M_TRYWAIT ?
264 M_WAITOK : M_NOWAIT);
265 if (1 /* XXX: how == M_TRYWAIT */) {
266 mtx_lock(&mbuf_mtx);
267 if (p == NULL)
268 mbstat.m_wait++;
269 }
270
271 /*
272 * Either the map is now full, or `how' is M_DONTWAIT and there
273 * are no pages left.
274 */
275 if (p == NULL)
276 return (0);
277
278 /*
279 * We don't let go of the mutex in order to avoid a race.
280 * It is up to the caller to let go of the mutex when done
281 * with grabbing the mbuf from the free list.
282 */
283 for (i = 0; i < nmb; i++) {
284 ((struct mbuf *)p)->m_next = mmbfree.m_head;
285 mmbfree.m_head = (struct mbuf *)p;
286 p += MSIZE;
287 }
288 mbstat.m_mbufs += nmb;
289 mbtypes[MT_FREE] += nmb;
290 return (1);
291}
292
293/*
294 * Once the mb_map has been exhausted and if the call to the allocation macros
295 * (or, in some cases, functions) is with M_TRYWAIT, then it is necessary to
296 * rely solely on reclaimed mbufs.
297 *
298 * Here we request for the protocols to free up some resources and, if we
299 * still cannot get anything, then we wait for an mbuf to be freed for a
300 * designated (mbuf_wait) time, at most.
301 *
302 * Must be called with the mmbfree mutex held.
303 */
304struct mbuf *
305m_mballoc_wait(void)
306{
307 struct mbuf *p = NULL;
308
309 /*
310 * See if we can drain some resources out of the protocols.
311 * We drop the mmbfree mutex to avoid recursing into it in some of
312 * the drain routines. Clearly, we're faced with a race here because
313 * once something is freed during the drain, it may be grabbed right
314 * from under us by some other thread. But we accept this possibility
315 * in order to avoid a potentially large lock recursion and, more
316 * importantly, to avoid a potential lock order reversal which may
317 * result in deadlock (See comment above m_reclaim()).
318 */
319 mtx_unlock(&mbuf_mtx);
320 m_reclaim();
321
322 mtx_lock(&mbuf_mtx);
323 _MGET(p, M_DONTWAIT);
324
325 if (p == NULL) {
326 int retval;
327
328 m_mballoc_wid++;
329 retval = cv_timedwait(&mmbfree.m_starved, &mbuf_mtx,
330 mbuf_wait);
331 m_mballoc_wid--;
332
333 /*
334 * If we got signaled (i.e. didn't time out), allocate.
335 */
336 if (retval == 0)
337 _MGET(p, M_DONTWAIT);
338 }
339
340 if (p != NULL) {
341 mbstat.m_wait++;
342 if (mmbfree.m_head != NULL)
343 MBWAKEUP(m_mballoc_wid, &mmbfree.m_starved);
344 }
345
346 return (p);
347}
348
349/*
350 * Allocate some number of mbuf clusters
351 * and place on cluster free list.
352 *
353 * Must be called with the mclfree lock held.
354 */
355int
356m_clalloc(int ncl, int how)
357{
358 caddr_t p;
359 int i;
360 int npg_sz;
361
362 npg_sz = round_page(ncl * MCLBYTES);
363 ncl = npg_sz / MCLBYTES;
364
365 /*
366 * If the map is now full (nothing will ever be freed to it).
367 * If we've hit the mcluster number limit, stop allocating from
368 * mb_map.
369 */
370 if (mb_map_full || ((ncl + mbstat.m_clusters) > nmbclusters))
371 return (0);
372
373 if (1 /* XXX: how == M_TRYWAIT */)
374 mtx_unlock(&mbuf_mtx);
375 p = (caddr_t)kmem_malloc(mb_map, npg_sz,
376 how == M_TRYWAIT ? M_WAITOK : M_NOWAIT);
377 if (1 /* XXX: how == M_TRYWAIT */)
378 mtx_lock(&mbuf_mtx);
379
380 /*
381 * Either the map is now full, or `how' is M_DONTWAIT and there
382 * are no pages left.
383 */
384 if (p == NULL)
385 return (0);
386
387 for (i = 0; i < ncl; i++) {
388 ((union mcluster *)p)->mcl_next = mclfree.m_head;
389 mclfree.m_head = (union mcluster *)p;
390 p += MCLBYTES;
391 mbstat.m_clfree++;
392 }
393 mbstat.m_clusters += ncl;
394 return (1);
395}
396
397/*
398 * Once the mb_map submap has been exhausted and the allocation is called with
399 * M_TRYWAIT, we rely on the mclfree list. If nothing is free, we will
400 * block on a cv for a designated amount of time (mbuf_wait) or until we're
401 * signaled due to sudden mcluster availability.
402 *
403 * Must be called with the mclfree lock held.
404 */
405caddr_t
406m_clalloc_wait(void)
407{
408 caddr_t p = NULL;
409 int retval;
410
411 m_clalloc_wid++;
412 retval = cv_timedwait(&mclfree.m_starved, &mbuf_mtx, mbuf_wait);
413 m_clalloc_wid--;
414
415 /*
416 * Now that we (think) that we've got something, try again.
417 */
418 if (retval == 0)
419 _MCLALLOC(p, M_DONTWAIT);
420
421 if (p != NULL) {
422 mbstat.m_wait++;
423 if (mclfree.m_head != NULL)
424 MBWAKEUP(m_clalloc_wid, &mclfree.m_starved);
425 }
426
427 return (p);
428}
429
430/*
431 * m_reclaim: drain protocols in hopes to free up some resources...
432 *
433 * XXX: No locks should be held going in here. The drain routines have
434 * to presently acquire some locks which raises the possibility of lock
435 * order violation if we're holding any mutex if that mutex is acquired in
436 * reverse order relative to one of the locks in the drain routines.
437 */
438static void
439m_reclaim(void)
440{
441 struct domain *dp;
442 struct protosw *pr;
443
444#ifdef WITNESS
445 KASSERT(witness_list(curproc) == 0,
446 ("m_reclaim called with locks held"));
447#endif
448
449 for (dp = domains; dp; dp = dp->dom_next)
450 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
451 if (pr->pr_drain)
452 (*pr->pr_drain)();
453 mbstat.m_drain++;
454}
455
456/*
457 * Space allocation routines.
458 * Some of these are also available as macros
459 * for critical paths.
460 */
461struct mbuf *
462m_get(int how, int type)
463{
464 struct mbuf *m;
465
466 MGET(m, how, type);
467 return (m);
468}
469
470struct mbuf *
471m_gethdr(int how, int type)
472{
473 struct mbuf *m;
474
475 MGETHDR(m, how, type);
476 return (m);
477}
478
479struct mbuf *
480m_getclr(int how, int type)
481{
482 struct mbuf *m;
483
484 MGET(m, how, type);
485 if (m != NULL)
486 bzero(mtod(m, caddr_t), MLEN);
487 return (m);
488}
489
490struct mbuf *
491m_free(struct mbuf *m)
492{
493 struct mbuf *n;
494
495 MFREE(m, n);
496 return (n);
497}
498
499/*
500 * struct mbuf *
501 * m_getm(m, len, how, type)
502 *
503 * This will allocate len-worth of mbufs and/or mbuf clusters (whatever fits
504 * best) and return a pointer to the top of the allocated chain. If m is
505 * non-null, then we assume that it is a single mbuf or an mbuf chain to
506 * which we want len bytes worth of mbufs and/or clusters attached, and so
507 * if we succeed in allocating it, we will just return a pointer to m.
508 *
509 * If we happen to fail at any point during the allocation, we will free
510 * up everything we have already allocated and return NULL.
511 *
512 */
513struct mbuf *
514m_getm(struct mbuf *m, int len, int how, int type)
515{
516 struct mbuf *top, *tail, *mp, *mtail = NULL;
517
518 KASSERT(len >= 0, ("len is < 0 in m_getm"));
519
520 MGET(mp, how, type);
521 if (mp == NULL)
522 return (NULL);
523 else if (len > MINCLSIZE) {
524 MCLGET(mp, how);
525 if ((mp->m_flags & M_EXT) == 0) {
526 m_free(mp);
527 return (NULL);
528 }
529 }
530 mp->m_len = 0;
531 len -= M_TRAILINGSPACE(mp);
532
533 if (m != NULL)
534 for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next);
535 else
536 m = mp;
537
538 top = tail = mp;
539 while (len > 0) {
540 MGET(mp, how, type);
541 if (mp == NULL)
542 goto failed;
543
544 tail->m_next = mp;
545 tail = mp;
546 if (len > MINCLSIZE) {
547 MCLGET(mp, how);
548 if ((mp->m_flags & M_EXT) == 0)
549 goto failed;
550 }
551
552 mp->m_len = 0;
553 len -= M_TRAILINGSPACE(mp);
554 }
555
556 if (mtail != NULL)
557 mtail->m_next = top;
558 return (m);
559
560failed:
561 m_freem(top);
562 return (NULL);
563}
564
565void
566m_freem(struct mbuf *m)
567{
568 struct mbuf *n;
569
570 if (m == NULL)
571 return;
572 do {
573 /*
574 * we do need to check non-first mbuf, since some of existing
575 * code does not call M_PREPEND properly.
576 * (example: call to bpf_mtap from drivers)
577 */
578 if ((m->m_flags & M_PKTHDR) != 0 && m->m_pkthdr.aux) {
579 m_freem(m->m_pkthdr.aux);
580 m->m_pkthdr.aux = NULL;
581 }
582 MFREE(m, n);
583 m = n;
584 } while (m);
585}
586
587/*
588 * Lesser-used path for M_PREPEND:
589 * allocate new mbuf to prepend to chain,
590 * copy junk along.
591 */
592struct mbuf *
593m_prepend(struct mbuf *m, int len, int how)
594{
595 struct mbuf *mn;
596
597 MGET(mn, how, m->m_type);
598 if (mn == NULL) {
599 m_freem(m);
600 return (NULL);
601 }
602 if (m->m_flags & M_PKTHDR) {
603 M_COPY_PKTHDR(mn, m);
604 m->m_flags &= ~M_PKTHDR;
605 }
606 mn->m_next = m;
607 m = mn;
608 if (len < MHLEN)
609 MH_ALIGN(m, len);
610 m->m_len = len;
611 return (m);
612}
613
614/*
615 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
616 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
617 * The wait parameter is a choice of M_TRYWAIT/M_DONTWAIT from caller.
618 * Note that the copy is read-only, because clusters are not copied,
619 * only their reference counts are incremented.
620 */
621struct mbuf *
622m_copym(struct mbuf *m, int off0, int len, int wait)
623{
624 struct mbuf *n, **np;
625 int off = off0;
626 struct mbuf *top;
627 int copyhdr = 0;
628
629 KASSERT(off >= 0, ("m_copym, negative off %d", off));
630 KASSERT(len >= 0, ("m_copym, negative len %d", len));
631 if (off == 0 && m->m_flags & M_PKTHDR)
632 copyhdr = 1;
633 while (off > 0) {
634 KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain"));
635 if (off < m->m_len)
636 break;
637 off -= m->m_len;
638 m = m->m_next;
639 }
640 np = ⊤
641 top = 0;
642 while (len > 0) {
643 if (m == NULL) {
644 KASSERT(len == M_COPYALL,
645 ("m_copym, length > size of mbuf chain"));
646 break;
647 }
648 MGET(n, wait, m->m_type);
649 *np = n;
650 if (n == NULL)
651 goto nospace;
652 if (copyhdr) {
653 M_COPY_PKTHDR(n, m);
654 if (len == M_COPYALL)
655 n->m_pkthdr.len -= off0;
656 else
657 n->m_pkthdr.len = len;
658 copyhdr = 0;
659 }
660 n->m_len = min(len, m->m_len - off);
661 if (m->m_flags & M_EXT) {
662 n->m_data = m->m_data + off;
663 n->m_ext = m->m_ext;
664 n->m_flags |= M_EXT;
665 MEXT_ADD_REF(m);
666 } else
667 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
668 (unsigned)n->m_len);
669 if (len != M_COPYALL)
670 len -= n->m_len;
671 off = 0;
672 m = m->m_next;
673 np = &n->m_next;
674 }
675 if (top == NULL) {
676 mtx_lock(&mbuf_mtx);
677 mbstat.m_mcfail++;
678 mtx_unlock(&mbuf_mtx);
679 }
680 return (top);
681nospace:
682 m_freem(top);
683 mtx_lock(&mbuf_mtx);
684 mbstat.m_mcfail++;
685 mtx_unlock(&mbuf_mtx);
686 return (NULL);
687}
688
689/*
690 * Copy an entire packet, including header (which must be present).
691 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'.
692 * Note that the copy is read-only, because clusters are not copied,
693 * only their reference counts are incremented.
694 * Preserve alignment of the first mbuf so if the creator has left
695 * some room at the beginning (e.g. for inserting protocol headers)
696 * the copies still have the room available.
697 */
698struct mbuf *
699m_copypacket(struct mbuf *m, int how)
700{
701 struct mbuf *top, *n, *o;
702
703 MGET(n, how, m->m_type);
704 top = n;
705 if (n == NULL)
706 goto nospace;
707
708 M_COPY_PKTHDR(n, m);
709 n->m_len = m->m_len;
710 if (m->m_flags & M_EXT) {
711 n->m_data = m->m_data;
712 n->m_ext = m->m_ext;
713 n->m_flags |= M_EXT;
714 MEXT_ADD_REF(m);
715 } else {
716 n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat );
717 bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
718 }
719
720 m = m->m_next;
721 while (m) {
722 MGET(o, how, m->m_type);
723 if (o == NULL)
724 goto nospace;
725
726 n->m_next = o;
727 n = n->m_next;
728
729 n->m_len = m->m_len;
730 if (m->m_flags & M_EXT) {
731 n->m_data = m->m_data;
732 n->m_ext = m->m_ext;
733 n->m_flags |= M_EXT;
734 MEXT_ADD_REF(m);
735 } else {
736 bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
737 }
738
739 m = m->m_next;
740 }
741 return top;
742nospace:
743 m_freem(top);
744 mtx_lock(&mbuf_mtx);
745 mbstat.m_mcfail++;
746 mtx_unlock(&mbuf_mtx);
747 return (NULL);
748}
749
750/*
751 * Copy data from an mbuf chain starting "off" bytes from the beginning,
752 * continuing for "len" bytes, into the indicated buffer.
753 */
754void
755m_copydata(struct mbuf *m, int off, int len, caddr_t cp)
756{
757 unsigned count;
758
759 KASSERT(off >= 0, ("m_copydata, negative off %d", off));
760 KASSERT(len >= 0, ("m_copydata, negative len %d", len));
761 while (off > 0) {
762 KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain"));
763 if (off < m->m_len)
764 break;
765 off -= m->m_len;
766 m = m->m_next;
767 }
768 while (len > 0) {
769 KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain"));
770 count = min(m->m_len - off, len);
771 bcopy(mtod(m, caddr_t) + off, cp, count);
772 len -= count;
773 cp += count;
774 off = 0;
775 m = m->m_next;
776 }
777}
778
779/*
780 * Copy a packet header mbuf chain into a completely new chain, including
781 * copying any mbuf clusters. Use this instead of m_copypacket() when
782 * you need a writable copy of an mbuf chain.
783 */
784struct mbuf *
785m_dup(struct mbuf *m, int how)
786{
787 struct mbuf **p, *top = NULL;
788 int remain, moff, nsize;
789
790 /* Sanity check */
791 if (m == NULL)
792 return (NULL);
793 KASSERT((m->m_flags & M_PKTHDR) != 0, ("%s: !PKTHDR", __FUNCTION__));
794
795 /* While there's more data, get a new mbuf, tack it on, and fill it */
796 remain = m->m_pkthdr.len;
797 moff = 0;
798 p = ⊤
799 while (remain > 0 || top == NULL) { /* allow m->m_pkthdr.len == 0 */
800 struct mbuf *n;
801
802 /* Get the next new mbuf */
803 MGET(n, how, m->m_type);
804 if (n == NULL)
805 goto nospace;
806 if (top == NULL) { /* first one, must be PKTHDR */
807 M_COPY_PKTHDR(n, m);
808 nsize = MHLEN;
809 } else /* not the first one */
810 nsize = MLEN;
811 if (remain >= MINCLSIZE) {
812 MCLGET(n, how);
813 if ((n->m_flags & M_EXT) == 0) {
814 (void)m_free(n);
815 goto nospace;
816 }
817 nsize = MCLBYTES;
818 }
819 n->m_len = 0;
820
821 /* Link it into the new chain */
822 *p = n;
823 p = &n->m_next;
824
825 /* Copy data from original mbuf(s) into new mbuf */
826 while (n->m_len < nsize && m != NULL) {
827 int chunk = min(nsize - n->m_len, m->m_len - moff);
828
829 bcopy(m->m_data + moff, n->m_data + n->m_len, chunk);
830 moff += chunk;
831 n->m_len += chunk;
832 remain -= chunk;
833 if (moff == m->m_len) {
834 m = m->m_next;
835 moff = 0;
836 }
837 }
838
839 /* Check correct total mbuf length */
840 KASSERT((remain > 0 && m != NULL) || (remain == 0 && m == NULL),
841 ("%s: bogus m_pkthdr.len", __FUNCTION__));
842 }
843 return (top);
844
845nospace:
846 m_freem(top);
847 mtx_lock(&mbuf_mtx);
848 mbstat.m_mcfail++;
849 mtx_unlock(&mbuf_mtx);
850 return (NULL);
851}
852
853/*
854 * Concatenate mbuf chain n to m.
855 * Both chains must be of the same type (e.g. MT_DATA).
856 * Any m_pkthdr is not updated.
857 */
858void
859m_cat(struct mbuf *m, struct mbuf *n)
860{
861 while (m->m_next)
862 m = m->m_next;
863 while (n) {
864 if (m->m_flags & M_EXT ||
865 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
866 /* just join the two chains */
867 m->m_next = n;
868 return;
869 }
870 /* splat the data from one into the other */
871 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
872 (u_int)n->m_len);
873 m->m_len += n->m_len;
874 n = m_free(n);
875 }
876}
877
878void
879m_adj(struct mbuf *mp, int req_len)
880{
881 int len = req_len;
882 struct mbuf *m;
883 int count;
884
885 if ((m = mp) == NULL)
886 return;
887 if (len >= 0) {
888 /*
889 * Trim from head.
890 */
891 while (m != NULL && len > 0) {
892 if (m->m_len <= len) {
893 len -= m->m_len;
894 m->m_len = 0;
895 m = m->m_next;
896 } else {
897 m->m_len -= len;
898 m->m_data += len;
899 len = 0;
900 }
901 }
902 m = mp;
903 if (mp->m_flags & M_PKTHDR)
904 m->m_pkthdr.len -= (req_len - len);
905 } else {
906 /*
907 * Trim from tail. Scan the mbuf chain,
908 * calculating its length and finding the last mbuf.
909 * If the adjustment only affects this mbuf, then just
910 * adjust and return. Otherwise, rescan and truncate
911 * after the remaining size.
912 */
913 len = -len;
914 count = 0;
915 for (;;) {
916 count += m->m_len;
917 if (m->m_next == (struct mbuf *)0)
918 break;
919 m = m->m_next;
920 }
921 if (m->m_len >= len) {
922 m->m_len -= len;
923 if (mp->m_flags & M_PKTHDR)
924 mp->m_pkthdr.len -= len;
925 return;
926 }
927 count -= len;
928 if (count < 0)
929 count = 0;
930 /*
931 * Correct length for chain is "count".
932 * Find the mbuf with last data, adjust its length,
933 * and toss data from remaining mbufs on chain.
934 */
935 m = mp;
936 if (m->m_flags & M_PKTHDR)
937 m->m_pkthdr.len = count;
938 for (; m; m = m->m_next) {
939 if (m->m_len >= count) {
940 m->m_len = count;
941 break;
942 }
943 count -= m->m_len;
944 }
945 while (m->m_next)
946 (m = m->m_next) ->m_len = 0;
947 }
948}
949
950/*
951 * Rearange an mbuf chain so that len bytes are contiguous
952 * and in the data area of an mbuf (so that mtod and dtom
953 * will work for a structure of size len). Returns the resulting
954 * mbuf chain on success, frees it and returns null on failure.
955 * If there is room, it will add up to max_protohdr-len extra bytes to the
956 * contiguous region in an attempt to avoid being called next time.
957 */
958struct mbuf *
959m_pullup(struct mbuf *n, int len)
960{
961 struct mbuf *m;
962 int count;
963 int space;
964
965 /*
966 * If first mbuf has no cluster, and has room for len bytes
967 * without shifting current data, pullup into it,
968 * otherwise allocate a new mbuf to prepend to the chain.
969 */
970 if ((n->m_flags & M_EXT) == 0 &&
971 n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
972 if (n->m_len >= len)
973 return (n);
974 m = n;
975 n = n->m_next;
976 len -= m->m_len;
977 } else {
978 if (len > MHLEN)
979 goto bad;
980 MGET(m, M_DONTWAIT, n->m_type);
981 if (m == NULL)
982 goto bad;
983 m->m_len = 0;
984 if (n->m_flags & M_PKTHDR) {
985 M_COPY_PKTHDR(m, n);
986 n->m_flags &= ~M_PKTHDR;
987 }
988 }
989 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
990 do {
991 count = min(min(max(len, max_protohdr), space), n->m_len);
992 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
993 (unsigned)count);
994 len -= count;
995 m->m_len += count;
996 n->m_len -= count;
997 space -= count;
998 if (n->m_len)
999 n->m_data += count;
1000 else
1001 n = m_free(n);
1002 } while (len > 0 && n);
1003 if (len > 0) {
1004 (void) m_free(m);
1005 goto bad;
1006 }
1007 m->m_next = n;
1008 return (m);
1009bad:
1010 m_freem(n);
1011 mtx_lock(&mbuf_mtx);
1012 mbstat.m_mpfail++;
1013 mtx_unlock(&mbuf_mtx);
1014 return (NULL);
1015}
1016
1017/*
1018 * Partition an mbuf chain in two pieces, returning the tail --
1019 * all but the first len0 bytes. In case of failure, it returns NULL and
1020 * attempts to restore the chain to its original state.
1021 */
1022struct mbuf *
1023m_split(struct mbuf *m0, int len0, int wait)
1024{
1025 struct mbuf *m, *n;
1026 unsigned len = len0, remain;
1027
1028 for (m = m0; m && len > m->m_len; m = m->m_next)
1029 len -= m->m_len;
1030 if (m == NULL)
1031 return (NULL);
1032 remain = m->m_len - len;
1033 if (m0->m_flags & M_PKTHDR) {
1034 MGETHDR(n, wait, m0->m_type);
1035 if (n == NULL)
1036 return (NULL);
1037 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1038 n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1039 m0->m_pkthdr.len = len0;
1040 if (m->m_flags & M_EXT)
1041 goto extpacket;
1042 if (remain > MHLEN) {
1043 /* m can't be the lead packet */
1044 MH_ALIGN(n, 0);
1045 n->m_next = m_split(m, len, wait);
1046 if (n->m_next == NULL) {
1047 (void) m_free(n);
1048 return (NULL);
1049 } else
1050 return (n);
1051 } else
1052 MH_ALIGN(n, remain);
1053 } else if (remain == 0) {
1054 n = m->m_next;
1055 m->m_next = NULL;
1056 return (n);
1057 } else {
1058 MGET(n, wait, m->m_type);
1059 if (n == NULL)
1060 return (NULL);
1061 M_ALIGN(n, remain);
1062 }
1063extpacket:
1064 if (m->m_flags & M_EXT) {
1065 n->m_flags |= M_EXT;
1066 n->m_ext = m->m_ext;
1067 MEXT_ADD_REF(m);
1068 m->m_ext.ext_size = 0; /* For Accounting XXXXXX danger */
1069 n->m_data = m->m_data + len;
1070 } else {
1071 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
1072 }
1073 n->m_len = remain;
1074 m->m_len = len;
1075 n->m_next = m->m_next;
1076 m->m_next = NULL;
1077 return (n);
1078}
1079/*
1080 * Routine to copy from device local memory into mbufs.
1081 */
1082struct mbuf *
1083m_devget(char *buf, int totlen, int off0, struct ifnet *ifp,
1084 void (*copy)(char *from, caddr_t to, u_int len))
1085{
1086 struct mbuf *m;
1087 struct mbuf *top = 0, **mp = ⊤
1088 int off = off0, len;
1089 char *cp;
1090 char *epkt;
1091
1092 cp = buf;
1093 epkt = cp + totlen;
1094 if (off) {
1095 cp += off + 2 * sizeof(u_short);
1096 totlen -= 2 * sizeof(u_short);
1097 }
1098 MGETHDR(m, M_DONTWAIT, MT_DATA);
1099 if (m == NULL)
1100 return (NULL);
1101 m->m_pkthdr.rcvif = ifp;
1102 m->m_pkthdr.len = totlen;
1103 m->m_len = MHLEN;
1104
1105 while (totlen > 0) {
1106 if (top) {
1107 MGET(m, M_DONTWAIT, MT_DATA);
1108 if (m == NULL) {
1109 m_freem(top);
1110 return (NULL);
1111 }
1112 m->m_len = MLEN;
1113 }
1114 len = min(totlen, epkt - cp);
1115 if (len >= MINCLSIZE) {
1116 MCLGET(m, M_DONTWAIT);
1117 if (m->m_flags & M_EXT)
1118 m->m_len = len = min(len, MCLBYTES);
1119 else
1120 len = m->m_len;
1121 } else {
1122 /*
1123 * Place initial small packet/header at end of mbuf.
1124 */
1125 if (len < m->m_len) {
1126 if (top == NULL && len +
1127 max_linkhdr <= m->m_len)
1128 m->m_data += max_linkhdr;
1129 m->m_len = len;
1130 } else
1131 len = m->m_len;
1132 }
1133 if (copy)
1134 copy(cp, mtod(m, caddr_t), (unsigned)len);
1135 else
1136 bcopy(cp, mtod(m, caddr_t), (unsigned)len);
1137 cp += len;
1138 *mp = m;
1139 mp = &m->m_next;
1140 totlen -= len;
1141 if (cp == epkt)
1142 cp = buf;
1143 }
1144 return (top);
1145}
1146
1147/*
1148 * Copy data from a buffer back into the indicated mbuf chain,
1149 * starting "off" bytes from the beginning, extending the mbuf
1150 * chain if necessary.
1151 */
1152void
1153m_copyback(struct mbuf *m0, int off, int len, caddr_t cp)
1154{
1155 int mlen;
1156 struct mbuf *m = m0, *n;
1157 int totlen = 0;
1158
1159 if (m0 == NULL)
1160 return;
1161 while (off > (mlen = m->m_len)) {
1162 off -= mlen;
1163 totlen += mlen;
1164 if (m->m_next == NULL) {
1165 n = m_getclr(M_DONTWAIT, m->m_type);
1166 if (n == NULL)
1167 goto out;
1168 n->m_len = min(MLEN, len + off);
1169 m->m_next = n;
1170 }
1171 m = m->m_next;
1172 }
1173 while (len > 0) {
1174 mlen = min (m->m_len - off, len);
1175 bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen);
1176 cp += mlen;
1177 len -= mlen;
1178 mlen += off;
1179 off = 0;
1180 totlen += mlen;
1181 if (len == 0)
1182 break;
1183 if (m->m_next == NULL) {
1184 n = m_get(M_DONTWAIT, m->m_type);
1185 if (n == NULL)
1186 break;
1187 n->m_len = min(MLEN, len);
1188 m->m_next = n;
1189 }
1190 m = m->m_next;
1191 }
1192out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
1193 m->m_pkthdr.len = totlen;
1194}
1195
1196void
1197m_print(const struct mbuf *m)
1198{
1199 int len;
1200 const struct mbuf *m2;
1201
1202 len = m->m_pkthdr.len;
1203 m2 = m;
1204 while (len) {
1205 printf("%p %*D\n", m2, m2->m_len, (u_char *)m2->m_data, "-");
1206 len -= m2->m_len;
1207 m2 = m2->m_next;
1208 }
1209 return;
1210}