uipc_mbuf.c revision 54478
1251876Speter/* 2251876Speter * Copyright (c) 1982, 1986, 1988, 1991, 1993 3251876Speter * The Regents of the University of California. All rights reserved. 4251876Speter * 5251876Speter * Redistribution and use in source and binary forms, with or without 6251876Speter * modification, are permitted provided that the following conditions 7251876Speter * are met: 8251876Speter * 1. Redistributions of source code must retain the above copyright 9251876Speter * notice, this list of conditions and the following disclaimer. 10251876Speter * 2. Redistributions in binary form must reproduce the above copyright 11251876Speter * notice, this list of conditions and the following disclaimer in the 12251876Speter * documentation and/or other materials provided with the distribution. 13251876Speter * 3. All advertising materials mentioning features or use of this software 14251876Speter * must display the following acknowledgement: 15251876Speter * This product includes software developed by the University of 16251876Speter * California, Berkeley and its contributors. 17251876Speter * 4. Neither the name of the University nor the names of its contributors 18251876Speter * may be used to endorse or promote products derived from this software 19251876Speter * without specific prior written permission. 20251876Speter * 21251876Speter * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 22251876Speter * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23251876Speter * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24251876Speter * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 25251876Speter * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26251876Speter * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27251876Speter * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28251876Speter * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29251876Speter * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30251876Speter * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31251876Speter * SUCH DAMAGE. 32251876Speter * 33251876Speter * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94 34251876Speter * $FreeBSD: head/sys/kern/uipc_mbuf.c 54478 1999-12-12 05:52:51Z green $ 35251876Speter */ 36251876Speter 37251876Speter#include "opt_param.h" 38251876Speter#include <sys/param.h> 39251876Speter#include <sys/systm.h> 40251876Speter#include <sys/malloc.h> 41251876Speter#include <sys/mbuf.h> 42251876Speter#include <sys/kernel.h> 43251876Speter#include <sys/sysctl.h> 44251876Speter#include <sys/domain.h> 45251876Speter#include <sys/protosw.h> 46251876Speter 47251876Speter#include <vm/vm.h> 48251876Speter#include <vm/vm_kern.h> 49251876Speter#include <vm/vm_extern.h> 50251876Speter 51251876Speter#ifdef INVARIANTS 52251876Speter#include <machine/cpu.h> 53251876Speter#endif 54251876Speter 55251876Speterstatic void mbinit __P((void *)); 56251876SpeterSYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbinit, NULL) 57251876Speter 58251876Speterstruct mbuf *mbutl; 59251876Speterchar *mclrefcnt; 60251876Speterstruct mbstat mbstat; 61251876Speterstruct mbuf *mmbfree; 62251876Speterunion mcluster *mclfree; 63251876Speterint max_linkhdr; 64251876Speterint max_protohdr; 65251876Speterint max_hdr; 66251876Speterint max_datalen; 67251876Speterint nmbclusters; 68251876Speterint nmbufs; 69251876Speter 70251876SpeterSYSCTL_DECL(_kern_ipc); 71251876SpeterSYSCTL_INT(_kern_ipc, KIPC_MAX_LINKHDR, max_linkhdr, CTLFLAG_RW, 72251876Speter &max_linkhdr, 0, ""); 73251876SpeterSYSCTL_INT(_kern_ipc, KIPC_MAX_PROTOHDR, max_protohdr, CTLFLAG_RW, 74251876Speter &max_protohdr, 0, ""); 75251876SpeterSYSCTL_INT(_kern_ipc, KIPC_MAX_HDR, max_hdr, CTLFLAG_RW, &max_hdr, 0, ""); 76251876SpeterSYSCTL_INT(_kern_ipc, KIPC_MAX_DATALEN, max_datalen, CTLFLAG_RW, 77251876Speter &max_datalen, 0, ""); 78251876SpeterSYSCTL_INT(_kern_ipc, OID_AUTO, mbuf_wait, CTLFLAG_RW, 79251876Speter &mbuf_wait, 0, ""); 80251876SpeterSYSCTL_STRUCT(_kern_ipc, KIPC_MBSTAT, mbstat, CTLFLAG_RW, &mbstat, mbstat, ""); 81251876SpeterSYSCTL_INT(_kern_ipc, KIPC_NMBCLUSTERS, nmbclusters, CTLFLAG_RD, 82251876Speter &nmbclusters, 0, "Maximum number of mbuf clusters avaliable"); 83251876Speter#ifndef NMBCLUSTERS 84251876Speter#define NMBCLUSTERS (512 + MAXUSERS * 16) 85251876Speter#endif 86251876SpeterTUNABLE_INT_DECL("kern.ipc.nmbclusters", NMBCLUSTERS, nmbclusters); 87251876SpeterTUNABLE_INT_DECL("kern.ipc.nmbufs", NMBCLUSTERS * 4, nmbufs); /* XXX fixup? */ 88251876Speter 89251876Speterstatic void m_reclaim __P((void)); 90251876Speter 91251876Speter/* "number of clusters of pages" */ 92251876Speter#define NCL_INIT 1 93251876Speter 94251876Speter#define NMB_INIT 16 95251876Speter 96251876Speter/* ARGSUSED*/ 97static void 98mbinit(dummy) 99 void *dummy; 100{ 101 int s; 102 103 mmbfree = NULL; mclfree = NULL; 104 mbstat.m_msize = MSIZE; 105 mbstat.m_mclbytes = MCLBYTES; 106 mbstat.m_minclsize = MINCLSIZE; 107 mbstat.m_mlen = MLEN; 108 mbstat.m_mhlen = MHLEN; 109 110 s = splimp(); 111 if (m_mballoc(NMB_INIT, M_DONTWAIT) == 0) 112 goto bad; 113#if MCLBYTES <= PAGE_SIZE 114 if (m_clalloc(NCL_INIT, M_DONTWAIT) == 0) 115 goto bad; 116#else 117 /* It's OK to call contigmalloc in this context. */ 118 if (m_clalloc(16, M_WAIT) == 0) 119 goto bad; 120#endif 121 splx(s); 122 return; 123bad: 124 panic("mbinit"); 125} 126 127/* 128 * Allocate at least nmb mbufs and place on mbuf free list. 129 * Must be called at splimp. 130 */ 131/* ARGSUSED */ 132int 133m_mballoc(nmb, how) 134 register int nmb; 135 int how; 136{ 137 register caddr_t p; 138 register int i; 139 int nbytes; 140 141 /* 142 * Once we run out of map space, it will be impossible to get 143 * any more (nothing is ever freed back to the map) 144 * -- however you are not dead as m_reclaim might 145 * still be able to free a substantial amount of space. 146 * 147 * XXX Furthermore, we can also work with "recycled" mbufs (when 148 * we're calling with M_WAIT the sleep procedure will be woken 149 * up when an mbuf is freed. See m_mballoc_wait()). 150 */ 151 if (mb_map_full) 152 return (0); 153 154 nbytes = round_page(nmb * MSIZE); 155 p = (caddr_t)kmem_malloc(mb_map, nbytes, M_NOWAIT); 156 if (p == 0 && how == M_WAIT) { 157 mbstat.m_wait++; 158 p = (caddr_t)kmem_malloc(mb_map, nbytes, M_WAITOK); 159 } 160 161 /* 162 * Either the map is now full, or `how' is M_NOWAIT and there 163 * are no pages left. 164 */ 165 if (p == NULL) 166 return (0); 167 168 nmb = nbytes / MSIZE; 169 for (i = 0; i < nmb; i++) { 170 ((struct mbuf *)p)->m_next = mmbfree; 171 mmbfree = (struct mbuf *)p; 172 p += MSIZE; 173 } 174 mbstat.m_mbufs += nmb; 175 return (1); 176} 177 178/* 179 * Once the mb_map has been exhausted and if the call to the allocation macros 180 * (or, in some cases, functions) is with M_WAIT, then it is necessary to rely 181 * solely on reclaimed mbufs. Here we wait for an mbuf to be freed for a 182 * designated (mbuf_wait) time. 183 */ 184struct mbuf * 185m_mballoc_wait(int caller, int type) 186{ 187 struct mbuf *p; 188 int s; 189 190 m_mballoc_wid++; 191 if ((tsleep(&m_mballoc_wid, PVM, "mballc", mbuf_wait)) == EWOULDBLOCK) 192 m_mballoc_wid--; 193 194 /* 195 * Now that we (think) that we've got something, we will redo an 196 * MGET, but avoid getting into another instance of m_mballoc_wait() 197 * XXX: We retry to fetch _even_ if the sleep timed out. This is left 198 * this way, purposely, in the [unlikely] case that an mbuf was 199 * freed but the sleep was not awakened in time. 200 */ 201 p = NULL; 202 switch (caller) { 203 case MGET_C: 204 MGET(p, M_DONTWAIT, type); 205 break; 206 case MGETHDR_C: 207 MGETHDR(p, M_DONTWAIT, type); 208 break; 209 default: 210 panic("m_mballoc_wait: invalid caller (%d)", caller); 211 } 212 213 s = splimp(); 214 if (p != NULL) { /* We waited and got something... */ 215 mbstat.m_wait++; 216 /* Wake up another if we have more free. */ 217 if (mmbfree != NULL) 218 m_mballoc_wakeup(); 219 } 220 splx(s); 221 return (p); 222} 223 224#if MCLBYTES > PAGE_SIZE 225static int i_want_my_mcl; 226 227static void 228kproc_mclalloc(void) 229{ 230 int status; 231 232 while (1) { 233 tsleep(&i_want_my_mcl, PVM, "mclalloc", 0); 234 235 for (; i_want_my_mcl; i_want_my_mcl--) { 236 if (m_clalloc(1, M_WAIT) == 0) 237 printf("m_clalloc failed even in process context!\n"); 238 } 239 } 240} 241 242static struct proc *mclallocproc; 243static struct kproc_desc mclalloc_kp = { 244 "mclalloc", 245 kproc_mclalloc, 246 &mclallocproc 247}; 248SYSINIT(mclallocproc, SI_SUB_KTHREAD_UPDATE, SI_ORDER_ANY, kproc_start, 249 &mclalloc_kp); 250#endif 251 252/* 253 * Allocate some number of mbuf clusters 254 * and place on cluster free list. 255 * Must be called at splimp. 256 */ 257/* ARGSUSED */ 258int 259m_clalloc(ncl, how) 260 register int ncl; 261 int how; 262{ 263 register caddr_t p; 264 register int i; 265 int npg; 266 267 /* 268 * Once we run out of map space, it will be impossible 269 * to get any more (nothing is ever freed back to the 270 * map). From this point on, we solely rely on freed 271 * mclusters. 272 */ 273 if (mb_map_full) { 274 mbstat.m_drops++; 275 return (0); 276 } 277 278#if MCLBYTES > PAGE_SIZE 279 if (how != M_WAIT) { 280 i_want_my_mcl += ncl; 281 wakeup(&i_want_my_mcl); 282 mbstat.m_wait++; 283 p = 0; 284 } else { 285 p = contigmalloc1(MCLBYTES * ncl, M_DEVBUF, M_WAITOK, 0ul, 286 ~0ul, PAGE_SIZE, 0, mb_map); 287 } 288#else 289 npg = ncl; 290 p = (caddr_t)kmem_malloc(mb_map, ctob(npg), 291 how != M_WAIT ? M_NOWAIT : M_WAITOK); 292 ncl = ncl * PAGE_SIZE / MCLBYTES; 293#endif 294 /* 295 * Either the map is now full, or `how' is M_NOWAIT and there 296 * are no pages left. 297 */ 298 if (p == NULL) { 299 mbstat.m_drops++; 300 return (0); 301 } 302 303 for (i = 0; i < ncl; i++) { 304 ((union mcluster *)p)->mcl_next = mclfree; 305 mclfree = (union mcluster *)p; 306 p += MCLBYTES; 307 mbstat.m_clfree++; 308 } 309 mbstat.m_clusters += ncl; 310 return (1); 311} 312 313/* 314 * Once the mb_map submap has been exhausted and the allocation is called with 315 * M_WAIT, we rely on the mclfree union pointers. If nothing is free, we will 316 * sleep for a designated amount of time (mbuf_wait) or until we're woken up 317 * due to sudden mcluster availability. 318 */ 319caddr_t 320m_clalloc_wait(void) 321{ 322 caddr_t p; 323 int s; 324 325#ifdef __i386__ 326 /* If in interrupt context, and INVARIANTS, maintain sanity and die. */ 327 KASSERT(intr_nesting_level == 0, ("CLALLOC: CANNOT WAIT IN INTERRUPT")); 328#endif 329 330 /* Sleep until something's available or until we expire. */ 331 m_clalloc_wid++; 332 if ((tsleep(&m_clalloc_wid, PVM, "mclalc", mbuf_wait)) == EWOULDBLOCK) 333 m_clalloc_wid--; 334 335 /* 336 * Now that we (think) that we've got something, we will redo and 337 * MGET, but avoid getting into another instance of m_clalloc_wait() 338 */ 339 p = NULL; 340 MCLALLOC(p, M_DONTWAIT); 341 342 s = splimp(); 343 if (p != NULL) { /* We waited and got something... */ 344 mbstat.m_wait++; 345 /* Wake up another if we have more free. */ 346 if (mclfree != NULL) 347 m_clalloc_wakeup(); 348 } 349 350 splx(s); 351 return (p); 352} 353 354/* 355 * When MGET fails, ask protocols to free space when short of memory, 356 * then re-attempt to allocate an mbuf. 357 */ 358struct mbuf * 359m_retry(i, t) 360 int i, t; 361{ 362 register struct mbuf *m; 363 364 /* 365 * Must only do the reclaim if not in an interrupt context. 366 */ 367 if (i == M_WAIT) { 368#ifdef __i386__ 369 KASSERT(intr_nesting_level == 0, 370 ("MBALLOC: CANNOT WAIT IN INTERRUPT")); 371#endif 372 m_reclaim(); 373 } 374 375 /* 376 * Both m_mballoc_wait and m_retry must be nulled because 377 * when the MGET macro is run from here, we deffinately do _not_ 378 * want to enter an instance of m_mballoc_wait() or m_retry() (again!) 379 */ 380#define m_mballoc_wait(caller,type) (struct mbuf *)0 381#define m_retry(i, t) (struct mbuf *)0 382 MGET(m, i, t); 383#undef m_retry 384#undef m_mballoc_wait 385 386 if (m != NULL) 387 mbstat.m_wait++; 388 else 389 mbstat.m_drops++; 390 391 return (m); 392} 393 394/* 395 * As above; retry an MGETHDR. 396 */ 397struct mbuf * 398m_retryhdr(i, t) 399 int i, t; 400{ 401 register struct mbuf *m; 402 403 /* 404 * Must only do the reclaim if not in an interrupt context. 405 */ 406 if (i == M_WAIT) { 407#ifdef __i386__ 408 KASSERT(intr_nesting_level == 0, 409 ("MBALLOC: CANNOT WAIT IN INTERRUPT")); 410#endif 411 m_reclaim(); 412 } 413 414#define m_mballoc_wait(caller,type) (struct mbuf *)0 415#define m_retryhdr(i, t) (struct mbuf *)0 416 MGETHDR(m, i, t); 417#undef m_retryhdr 418#undef m_mballoc_wait 419 420 if (m != NULL) 421 mbstat.m_wait++; 422 else 423 mbstat.m_drops++; 424 425 return (m); 426} 427 428static void 429m_reclaim() 430{ 431 register struct domain *dp; 432 register struct protosw *pr; 433 int s = splimp(); 434 435 for (dp = domains; dp; dp = dp->dom_next) 436 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) 437 if (pr->pr_drain) 438 (*pr->pr_drain)(); 439 splx(s); 440 mbstat.m_drain++; 441} 442 443/* 444 * Space allocation routines. 445 * These are also available as macros 446 * for critical paths. 447 */ 448struct mbuf * 449m_get(how, type) 450 int how, type; 451{ 452 register struct mbuf *m; 453 454 MGET(m, how, type); 455 return (m); 456} 457 458struct mbuf * 459m_gethdr(how, type) 460 int how, type; 461{ 462 register struct mbuf *m; 463 464 MGETHDR(m, how, type); 465 return (m); 466} 467 468struct mbuf * 469m_getclr(how, type) 470 int how, type; 471{ 472 register struct mbuf *m; 473 474 MGET(m, how, type); 475 if (m == 0) 476 return (0); 477 bzero(mtod(m, caddr_t), MLEN); 478 return (m); 479} 480 481struct mbuf * 482m_free(m) 483 struct mbuf *m; 484{ 485 register struct mbuf *n; 486 487 MFREE(m, n); 488 return (n); 489} 490 491void 492m_freem(m) 493 register struct mbuf *m; 494{ 495 register struct mbuf *n; 496 497 if (m == NULL) 498 return; 499 do { 500 MFREE(m, n); 501 m = n; 502 } while (m); 503} 504 505/* 506 * Mbuffer utility routines. 507 */ 508 509/* 510 * Lesser-used path for M_PREPEND: 511 * allocate new mbuf to prepend to chain, 512 * copy junk along. 513 */ 514struct mbuf * 515m_prepend(m, len, how) 516 register struct mbuf *m; 517 int len, how; 518{ 519 struct mbuf *mn; 520 521 MGET(mn, how, m->m_type); 522 if (mn == (struct mbuf *)NULL) { 523 m_freem(m); 524 return ((struct mbuf *)NULL); 525 } 526 if (m->m_flags & M_PKTHDR) { 527 M_COPY_PKTHDR(mn, m); 528 m->m_flags &= ~M_PKTHDR; 529 } 530 mn->m_next = m; 531 m = mn; 532 if (len < MHLEN) 533 MH_ALIGN(m, len); 534 m->m_len = len; 535 return (m); 536} 537 538/* 539 * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 540 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 541 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. 542 * Note that the copy is read-only, because clusters are not copied, 543 * only their reference counts are incremented. 544 */ 545#define MCFail (mbstat.m_mcfail) 546 547struct mbuf * 548m_copym(m, off0, len, wait) 549 register struct mbuf *m; 550 int off0, wait; 551 register int len; 552{ 553 register struct mbuf *n, **np; 554 register int off = off0; 555 struct mbuf *top; 556 int copyhdr = 0; 557 558 KASSERT(off >= 0, ("m_copym, negative off %d", off)); 559 KASSERT(len >= 0, ("m_copym, negative len %d", len)); 560 if (off == 0 && m->m_flags & M_PKTHDR) 561 copyhdr = 1; 562 while (off > 0) { 563 KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain")); 564 if (off < m->m_len) 565 break; 566 off -= m->m_len; 567 m = m->m_next; 568 } 569 np = ⊤ 570 top = 0; 571 while (len > 0) { 572 if (m == 0) { 573 KASSERT(len == M_COPYALL, 574 ("m_copym, length > size of mbuf chain")); 575 break; 576 } 577 MGET(n, wait, m->m_type); 578 *np = n; 579 if (n == 0) 580 goto nospace; 581 if (copyhdr) { 582 M_COPY_PKTHDR(n, m); 583 if (len == M_COPYALL) 584 n->m_pkthdr.len -= off0; 585 else 586 n->m_pkthdr.len = len; 587 copyhdr = 0; 588 } 589 n->m_len = min(len, m->m_len - off); 590 if (m->m_flags & M_EXT) { 591 n->m_data = m->m_data + off; 592 if(!m->m_ext.ext_ref) 593 mclrefcnt[mtocl(m->m_ext.ext_buf)]++; 594 else 595 (*(m->m_ext.ext_ref))(m->m_ext.ext_buf, 596 m->m_ext.ext_size); 597 n->m_ext = m->m_ext; 598 n->m_flags |= M_EXT; 599 } else 600 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), 601 (unsigned)n->m_len); 602 if (len != M_COPYALL) 603 len -= n->m_len; 604 off = 0; 605 m = m->m_next; 606 np = &n->m_next; 607 } 608 if (top == 0) 609 MCFail++; 610 return (top); 611nospace: 612 m_freem(top); 613 MCFail++; 614 return (0); 615} 616 617/* 618 * Copy an entire packet, including header (which must be present). 619 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'. 620 * Note that the copy is read-only, because clusters are not copied, 621 * only their reference counts are incremented. 622 */ 623struct mbuf * 624m_copypacket(m, how) 625 struct mbuf *m; 626 int how; 627{ 628 struct mbuf *top, *n, *o; 629 630 MGET(n, how, m->m_type); 631 top = n; 632 if (!n) 633 goto nospace; 634 635 M_COPY_PKTHDR(n, m); 636 n->m_len = m->m_len; 637 if (m->m_flags & M_EXT) { 638 n->m_data = m->m_data; 639 if(!m->m_ext.ext_ref) 640 mclrefcnt[mtocl(m->m_ext.ext_buf)]++; 641 else 642 (*(m->m_ext.ext_ref))(m->m_ext.ext_buf, 643 m->m_ext.ext_size); 644 n->m_ext = m->m_ext; 645 n->m_flags |= M_EXT; 646 } else { 647 bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 648 } 649 650 m = m->m_next; 651 while (m) { 652 MGET(o, how, m->m_type); 653 if (!o) 654 goto nospace; 655 656 n->m_next = o; 657 n = n->m_next; 658 659 n->m_len = m->m_len; 660 if (m->m_flags & M_EXT) { 661 n->m_data = m->m_data; 662 if(!m->m_ext.ext_ref) 663 mclrefcnt[mtocl(m->m_ext.ext_buf)]++; 664 else 665 (*(m->m_ext.ext_ref))(m->m_ext.ext_buf, 666 m->m_ext.ext_size); 667 n->m_ext = m->m_ext; 668 n->m_flags |= M_EXT; 669 } else { 670 bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 671 } 672 673 m = m->m_next; 674 } 675 return top; 676nospace: 677 m_freem(top); 678 MCFail++; 679 return 0; 680} 681 682/* 683 * Copy data from an mbuf chain starting "off" bytes from the beginning, 684 * continuing for "len" bytes, into the indicated buffer. 685 */ 686void 687m_copydata(m, off, len, cp) 688 register struct mbuf *m; 689 register int off; 690 register int len; 691 caddr_t cp; 692{ 693 register unsigned count; 694 695 KASSERT(off >= 0, ("m_copydata, negative off %d", off)); 696 KASSERT(len >= 0, ("m_copydata, negative len %d", len)); 697 while (off > 0) { 698 KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain")); 699 if (off < m->m_len) 700 break; 701 off -= m->m_len; 702 m = m->m_next; 703 } 704 while (len > 0) { 705 KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain")); 706 count = min(m->m_len - off, len); 707 bcopy(mtod(m, caddr_t) + off, cp, count); 708 len -= count; 709 cp += count; 710 off = 0; 711 m = m->m_next; 712 } 713} 714 715/* 716 * Copy a packet header mbuf chain into a completely new chain, including 717 * copying any mbuf clusters. Use this instead of m_copypacket() when 718 * you need a writable copy of an mbuf chain. 719 */ 720struct mbuf * 721m_dup(m, how) 722 struct mbuf *m; 723 int how; 724{ 725 struct mbuf **p, *top = NULL; 726 int remain, moff, nsize; 727 728 /* Sanity check */ 729 if (m == NULL) 730 return (0); 731 KASSERT((m->m_flags & M_PKTHDR) != 0, ("%s: !PKTHDR", __FUNCTION__)); 732 733 /* While there's more data, get a new mbuf, tack it on, and fill it */ 734 remain = m->m_pkthdr.len; 735 moff = 0; 736 p = ⊤ 737 while (remain > 0 || top == NULL) { /* allow m->m_pkthdr.len == 0 */ 738 struct mbuf *n; 739 740 /* Get the next new mbuf */ 741 MGET(n, how, m->m_type); 742 if (n == NULL) 743 goto nospace; 744 if (top == NULL) { /* first one, must be PKTHDR */ 745 M_COPY_PKTHDR(n, m); 746 nsize = MHLEN; 747 } else /* not the first one */ 748 nsize = MLEN; 749 if (remain >= MINCLSIZE) { 750 MCLGET(n, how); 751 if ((n->m_flags & M_EXT) == 0) { 752 (void)m_free(n); 753 goto nospace; 754 } 755 nsize = MCLBYTES; 756 } 757 n->m_len = 0; 758 759 /* Link it into the new chain */ 760 *p = n; 761 p = &n->m_next; 762 763 /* Copy data from original mbuf(s) into new mbuf */ 764 while (n->m_len < nsize && m != NULL) { 765 int chunk = min(nsize - n->m_len, m->m_len - moff); 766 767 bcopy(m->m_data + moff, n->m_data + n->m_len, chunk); 768 moff += chunk; 769 n->m_len += chunk; 770 remain -= chunk; 771 if (moff == m->m_len) { 772 m = m->m_next; 773 moff = 0; 774 } 775 } 776 777 /* Check correct total mbuf length */ 778 KASSERT((remain > 0 && m != NULL) || (remain == 0 && m == NULL), 779 ("%s: bogus m_pkthdr.len", __FUNCTION__)); 780 } 781 return (top); 782 783nospace: 784 m_freem(top); 785 MCFail++; 786 return (0); 787} 788 789/* 790 * Concatenate mbuf chain n to m. 791 * Both chains must be of the same type (e.g. MT_DATA). 792 * Any m_pkthdr is not updated. 793 */ 794void 795m_cat(m, n) 796 register struct mbuf *m, *n; 797{ 798 while (m->m_next) 799 m = m->m_next; 800 while (n) { 801 if (m->m_flags & M_EXT || 802 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { 803 /* just join the two chains */ 804 m->m_next = n; 805 return; 806 } 807 /* splat the data from one into the other */ 808 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 809 (u_int)n->m_len); 810 m->m_len += n->m_len; 811 n = m_free(n); 812 } 813} 814 815void 816m_adj(mp, req_len) 817 struct mbuf *mp; 818 int req_len; 819{ 820 register int len = req_len; 821 register struct mbuf *m; 822 register int count; 823 824 if ((m = mp) == NULL) 825 return; 826 if (len >= 0) { 827 /* 828 * Trim from head. 829 */ 830 while (m != NULL && len > 0) { 831 if (m->m_len <= len) { 832 len -= m->m_len; 833 m->m_len = 0; 834 m = m->m_next; 835 } else { 836 m->m_len -= len; 837 m->m_data += len; 838 len = 0; 839 } 840 } 841 m = mp; 842 if (mp->m_flags & M_PKTHDR) 843 m->m_pkthdr.len -= (req_len - len); 844 } else { 845 /* 846 * Trim from tail. Scan the mbuf chain, 847 * calculating its length and finding the last mbuf. 848 * If the adjustment only affects this mbuf, then just 849 * adjust and return. Otherwise, rescan and truncate 850 * after the remaining size. 851 */ 852 len = -len; 853 count = 0; 854 for (;;) { 855 count += m->m_len; 856 if (m->m_next == (struct mbuf *)0) 857 break; 858 m = m->m_next; 859 } 860 if (m->m_len >= len) { 861 m->m_len -= len; 862 if (mp->m_flags & M_PKTHDR) 863 mp->m_pkthdr.len -= len; 864 return; 865 } 866 count -= len; 867 if (count < 0) 868 count = 0; 869 /* 870 * Correct length for chain is "count". 871 * Find the mbuf with last data, adjust its length, 872 * and toss data from remaining mbufs on chain. 873 */ 874 m = mp; 875 if (m->m_flags & M_PKTHDR) 876 m->m_pkthdr.len = count; 877 for (; m; m = m->m_next) { 878 if (m->m_len >= count) { 879 m->m_len = count; 880 break; 881 } 882 count -= m->m_len; 883 } 884 while (m->m_next) 885 (m = m->m_next) ->m_len = 0; 886 } 887} 888 889/* 890 * Rearange an mbuf chain so that len bytes are contiguous 891 * and in the data area of an mbuf (so that mtod and dtom 892 * will work for a structure of size len). Returns the resulting 893 * mbuf chain on success, frees it and returns null on failure. 894 * If there is room, it will add up to max_protohdr-len extra bytes to the 895 * contiguous region in an attempt to avoid being called next time. 896 */ 897#define MPFail (mbstat.m_mpfail) 898 899struct mbuf * 900m_pullup(n, len) 901 register struct mbuf *n; 902 int len; 903{ 904 register struct mbuf *m; 905 register int count; 906 int space; 907 908 /* 909 * If first mbuf has no cluster, and has room for len bytes 910 * without shifting current data, pullup into it, 911 * otherwise allocate a new mbuf to prepend to the chain. 912 */ 913 if ((n->m_flags & M_EXT) == 0 && 914 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 915 if (n->m_len >= len) 916 return (n); 917 m = n; 918 n = n->m_next; 919 len -= m->m_len; 920 } else { 921 if (len > MHLEN) 922 goto bad; 923 MGET(m, M_DONTWAIT, n->m_type); 924 if (m == 0) 925 goto bad; 926 m->m_len = 0; 927 if (n->m_flags & M_PKTHDR) { 928 M_COPY_PKTHDR(m, n); 929 n->m_flags &= ~M_PKTHDR; 930 } 931 } 932 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 933 do { 934 count = min(min(max(len, max_protohdr), space), n->m_len); 935 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 936 (unsigned)count); 937 len -= count; 938 m->m_len += count; 939 n->m_len -= count; 940 space -= count; 941 if (n->m_len) 942 n->m_data += count; 943 else 944 n = m_free(n); 945 } while (len > 0 && n); 946 if (len > 0) { 947 (void) m_free(m); 948 goto bad; 949 } 950 m->m_next = n; 951 return (m); 952bad: 953 m_freem(n); 954 MPFail++; 955 return (0); 956} 957 958/* 959 * Partition an mbuf chain in two pieces, returning the tail -- 960 * all but the first len0 bytes. In case of failure, it returns NULL and 961 * attempts to restore the chain to its original state. 962 */ 963struct mbuf * 964m_split(m0, len0, wait) 965 register struct mbuf *m0; 966 int len0, wait; 967{ 968 register struct mbuf *m, *n; 969 unsigned len = len0, remain; 970 971 for (m = m0; m && len > m->m_len; m = m->m_next) 972 len -= m->m_len; 973 if (m == 0) 974 return (0); 975 remain = m->m_len - len; 976 if (m0->m_flags & M_PKTHDR) { 977 MGETHDR(n, wait, m0->m_type); 978 if (n == 0) 979 return (0); 980 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 981 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 982 m0->m_pkthdr.len = len0; 983 if (m->m_flags & M_EXT) 984 goto extpacket; 985 if (remain > MHLEN) { 986 /* m can't be the lead packet */ 987 MH_ALIGN(n, 0); 988 n->m_next = m_split(m, len, wait); 989 if (n->m_next == 0) { 990 (void) m_free(n); 991 return (0); 992 } else 993 return (n); 994 } else 995 MH_ALIGN(n, remain); 996 } else if (remain == 0) { 997 n = m->m_next; 998 m->m_next = 0; 999 return (n); 1000 } else { 1001 MGET(n, wait, m->m_type); 1002 if (n == 0) 1003 return (0); 1004 M_ALIGN(n, remain); 1005 } 1006extpacket: 1007 if (m->m_flags & M_EXT) { 1008 n->m_flags |= M_EXT; 1009 n->m_ext = m->m_ext; 1010 if(!m->m_ext.ext_ref) 1011 mclrefcnt[mtocl(m->m_ext.ext_buf)]++; 1012 else 1013 (*(m->m_ext.ext_ref))(m->m_ext.ext_buf, 1014 m->m_ext.ext_size); 1015 m->m_ext.ext_size = 0; /* For Accounting XXXXXX danger */ 1016 n->m_data = m->m_data + len; 1017 } else { 1018 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); 1019 } 1020 n->m_len = remain; 1021 m->m_len = len; 1022 n->m_next = m->m_next; 1023 m->m_next = 0; 1024 return (n); 1025} 1026/* 1027 * Routine to copy from device local memory into mbufs. 1028 */ 1029struct mbuf * 1030m_devget(buf, totlen, off0, ifp, copy) 1031 char *buf; 1032 int totlen, off0; 1033 struct ifnet *ifp; 1034 void (*copy) __P((char *from, caddr_t to, u_int len)); 1035{ 1036 register struct mbuf *m; 1037 struct mbuf *top = 0, **mp = ⊤ 1038 register int off = off0, len; 1039 register char *cp; 1040 char *epkt; 1041 1042 cp = buf; 1043 epkt = cp + totlen; 1044 if (off) { 1045 cp += off + 2 * sizeof(u_short); 1046 totlen -= 2 * sizeof(u_short); 1047 } 1048 MGETHDR(m, M_DONTWAIT, MT_DATA); 1049 if (m == 0) 1050 return (0); 1051 m->m_pkthdr.rcvif = ifp; 1052 m->m_pkthdr.len = totlen; 1053 m->m_len = MHLEN; 1054 1055 while (totlen > 0) { 1056 if (top) { 1057 MGET(m, M_DONTWAIT, MT_DATA); 1058 if (m == 0) { 1059 m_freem(top); 1060 return (0); 1061 } 1062 m->m_len = MLEN; 1063 } 1064 len = min(totlen, epkt - cp); 1065 if (len >= MINCLSIZE) { 1066 MCLGET(m, M_DONTWAIT); 1067 if (m->m_flags & M_EXT) 1068 m->m_len = len = min(len, MCLBYTES); 1069 else 1070 len = m->m_len; 1071 } else { 1072 /* 1073 * Place initial small packet/header at end of mbuf. 1074 */ 1075 if (len < m->m_len) { 1076 if (top == 0 && len + max_linkhdr <= m->m_len) 1077 m->m_data += max_linkhdr; 1078 m->m_len = len; 1079 } else 1080 len = m->m_len; 1081 } 1082 if (copy) 1083 copy(cp, mtod(m, caddr_t), (unsigned)len); 1084 else 1085 bcopy(cp, mtod(m, caddr_t), (unsigned)len); 1086 cp += len; 1087 *mp = m; 1088 mp = &m->m_next; 1089 totlen -= len; 1090 if (cp == epkt) 1091 cp = buf; 1092 } 1093 return (top); 1094} 1095 1096/* 1097 * Copy data from a buffer back into the indicated mbuf chain, 1098 * starting "off" bytes from the beginning, extending the mbuf 1099 * chain if necessary. 1100 */ 1101void 1102m_copyback(m0, off, len, cp) 1103 struct mbuf *m0; 1104 register int off; 1105 register int len; 1106 caddr_t cp; 1107{ 1108 register int mlen; 1109 register struct mbuf *m = m0, *n; 1110 int totlen = 0; 1111 1112 if (m0 == 0) 1113 return; 1114 while (off > (mlen = m->m_len)) { 1115 off -= mlen; 1116 totlen += mlen; 1117 if (m->m_next == 0) { 1118 n = m_getclr(M_DONTWAIT, m->m_type); 1119 if (n == 0) 1120 goto out; 1121 n->m_len = min(MLEN, len + off); 1122 m->m_next = n; 1123 } 1124 m = m->m_next; 1125 } 1126 while (len > 0) { 1127 mlen = min (m->m_len - off, len); 1128 bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen); 1129 cp += mlen; 1130 len -= mlen; 1131 mlen += off; 1132 off = 0; 1133 totlen += mlen; 1134 if (len == 0) 1135 break; 1136 if (m->m_next == 0) { 1137 n = m_get(M_DONTWAIT, m->m_type); 1138 if (n == 0) 1139 break; 1140 n->m_len = min(MLEN, len); 1141 m->m_next = n; 1142 } 1143 m = m->m_next; 1144 } 1145out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) 1146 m->m_pkthdr.len = totlen; 1147} 1148 1149void 1150m_print(const struct mbuf *m) 1151{ 1152 int len; 1153 struct mbuf *m2; 1154 1155 len = m->m_pkthdr.len; 1156 m2 = m; 1157 while (len) { 1158 printf("%p %*D\n", m2, m2->m_len, (u_char *)m2->m_data, "-"); 1159 len -= m2->m_len; 1160 m2 = m2->m_next; 1161 } 1162 return; 1163} 1164