uipc_mbuf.c revision 6669
1/* 2 * Copyright (c) 1982, 1986, 1988, 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. All advertising materials mentioning features or use of this software 14 * must display the following acknowledgement: 15 * This product includes software developed by the University of 16 * California, Berkeley and its contributors. 17 * 4. Neither the name of the University nor the names of its contributors 18 * may be used to endorse or promote products derived from this software 19 * without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 * 33 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94 34 * $Id: uipc_mbuf.c,v 1.7 1995/02/05 07:08:27 bde Exp $ 35 */ 36 37#include <sys/param.h> 38#include <sys/systm.h> 39#include <sys/proc.h> 40#include <sys/malloc.h> 41#define MBTYPES 42#include <sys/mbuf.h> 43#include <sys/kernel.h> 44#include <sys/syslog.h> 45#include <sys/domain.h> 46#include <sys/protosw.h> 47 48#include <vm/vm.h> 49 50extern vm_map_t mb_map; 51struct mbuf *mbutl; 52char *mclrefcnt; 53 54void 55mbinit() 56{ 57 int s; 58 59#if CLBYTES < 4096 60#define NCL_INIT (4096/CLBYTES) 61#else 62#define NCL_INIT 1 63#endif 64 s = splimp(); 65 if (m_clalloc(NCL_INIT, M_DONTWAIT) == 0) 66 goto bad; 67 splx(s); 68 return; 69bad: 70 panic("mbinit"); 71} 72 73/* 74 * Allocate some number of mbuf clusters 75 * and place on cluster free list. 76 * Must be called at splimp. 77 */ 78/* ARGSUSED */ 79int 80m_clalloc(ncl, nowait) 81 register int ncl; 82 int nowait; 83{ 84 static int logged; 85 register caddr_t p; 86 register int i; 87 int npg; 88 89 npg = ncl * CLSIZE; 90 p = (caddr_t)kmem_malloc(mb_map, ctob(npg), 91 nowait ? M_NOWAIT : M_WAITOK); 92 if (p == NULL) { 93 if (logged == 0) { 94 logged++; 95 log(LOG_ERR, "mb_map full\n"); 96 } 97 return (0); 98 } 99 ncl = ncl * CLBYTES / MCLBYTES; 100 for (i = 0; i < ncl; i++) { 101 ((union mcluster *)p)->mcl_next = mclfree; 102 mclfree = (union mcluster *)p; 103 p += MCLBYTES; 104 mbstat.m_clfree++; 105 } 106 mbstat.m_clusters += ncl; 107 return (1); 108} 109 110/* 111 * When MGET failes, ask protocols to free space when short of memory, 112 * then re-attempt to allocate an mbuf. 113 */ 114struct mbuf * 115m_retry(i, t) 116 int i, t; 117{ 118 register struct mbuf *m; 119 120 m_reclaim(); 121#define m_retry(i, t) (struct mbuf *)0 122 MGET(m, i, t); 123#undef m_retry 124 if (m != NULL) 125 mbstat.m_wait++; 126 else 127 mbstat.m_drops++; 128 return (m); 129} 130 131/* 132 * As above; retry an MGETHDR. 133 */ 134struct mbuf * 135m_retryhdr(i, t) 136 int i, t; 137{ 138 register struct mbuf *m; 139 140 m_reclaim(); 141#define m_retryhdr(i, t) (struct mbuf *)0 142 MGETHDR(m, i, t); 143#undef m_retryhdr 144 if (m != NULL) 145 mbstat.m_wait++; 146 else 147 mbstat.m_drops++; 148 return (m); 149} 150 151void 152m_reclaim() 153{ 154 register struct domain *dp; 155 register struct protosw *pr; 156 int s = splimp(); 157 158 for (dp = domains; dp; dp = dp->dom_next) 159 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) 160 if (pr->pr_drain) 161 (*pr->pr_drain)(); 162 splx(s); 163 mbstat.m_drain++; 164} 165 166/* 167 * Space allocation routines. 168 * These are also available as macros 169 * for critical paths. 170 */ 171struct mbuf * 172m_get(nowait, type) 173 int nowait, type; 174{ 175 register struct mbuf *m; 176 177 MGET(m, nowait, type); 178 return (m); 179} 180 181struct mbuf * 182m_gethdr(nowait, type) 183 int nowait, type; 184{ 185 register struct mbuf *m; 186 187 MGETHDR(m, nowait, type); 188 return (m); 189} 190 191struct mbuf * 192m_getclr(nowait, type) 193 int nowait, type; 194{ 195 register struct mbuf *m; 196 197 MGET(m, nowait, type); 198 if (m == 0) 199 return (0); 200 bzero(mtod(m, caddr_t), MLEN); 201 return (m); 202} 203 204struct mbuf * 205m_free(m) 206 struct mbuf *m; 207{ 208 register struct mbuf *n; 209 210 MFREE(m, n); 211 return (n); 212} 213 214void 215m_freem(m) 216 register struct mbuf *m; 217{ 218 register struct mbuf *n; 219 220 if (m == NULL) 221 return; 222 do { 223 MFREE(m, n); 224 m = n; 225 } while (m); 226} 227 228/* 229 * Mbuffer utility routines. 230 */ 231 232/* 233 * Lesser-used path for M_PREPEND: 234 * allocate new mbuf to prepend to chain, 235 * copy junk along. 236 */ 237struct mbuf * 238m_prepend(m, len, how) 239 register struct mbuf *m; 240 int len, how; 241{ 242 struct mbuf *mn; 243 244 MGET(mn, how, m->m_type); 245 if (mn == (struct mbuf *)NULL) { 246 m_freem(m); 247 return ((struct mbuf *)NULL); 248 } 249 if (m->m_flags & M_PKTHDR) { 250 M_COPY_PKTHDR(mn, m); 251 m->m_flags &= ~M_PKTHDR; 252 } 253 mn->m_next = m; 254 m = mn; 255 if (len < MHLEN) 256 MH_ALIGN(m, len); 257 m->m_len = len; 258 return (m); 259} 260 261/* 262 * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 263 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 264 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. 265 */ 266int MCFail; 267 268struct mbuf * 269m_copym(m, off0, len, wait) 270 register struct mbuf *m; 271 int off0, wait; 272 register int len; 273{ 274 register struct mbuf *n, **np; 275 register int off = off0; 276 struct mbuf *top; 277 int copyhdr = 0; 278 279 if (off < 0 || len < 0) 280 panic("m_copym"); 281 if (off == 0 && m->m_flags & M_PKTHDR) 282 copyhdr = 1; 283 while (off > 0) { 284 if (m == 0) 285 panic("m_copym"); 286 if (off < m->m_len) 287 break; 288 off -= m->m_len; 289 m = m->m_next; 290 } 291 np = ⊤ 292 top = 0; 293 while (len > 0) { 294 if (m == 0) { 295 if (len != M_COPYALL) 296 panic("m_copym"); 297 break; 298 } 299 MGET(n, wait, m->m_type); 300 *np = n; 301 if (n == 0) 302 goto nospace; 303 if (copyhdr) { 304 M_COPY_PKTHDR(n, m); 305 if (len == M_COPYALL) 306 n->m_pkthdr.len -= off0; 307 else 308 n->m_pkthdr.len = len; 309 copyhdr = 0; 310 } 311 n->m_len = min(len, m->m_len - off); 312 if (m->m_flags & M_EXT) { 313 n->m_data = m->m_data + off; 314 mclrefcnt[mtocl(m->m_ext.ext_buf)]++; 315 n->m_ext = m->m_ext; 316 n->m_flags |= M_EXT; 317 } else 318 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), 319 (unsigned)n->m_len); 320 if (len != M_COPYALL) 321 len -= n->m_len; 322 off = 0; 323 m = m->m_next; 324 np = &n->m_next; 325 } 326 if (top == 0) 327 MCFail++; 328 return (top); 329nospace: 330 m_freem(top); 331 MCFail++; 332 return (0); 333} 334 335/* 336 * Copy data from an mbuf chain starting "off" bytes from the beginning, 337 * continuing for "len" bytes, into the indicated buffer. 338 */ 339void 340m_copydata(m, off, len, cp) 341 register struct mbuf *m; 342 register int off; 343 register int len; 344 caddr_t cp; 345{ 346 register unsigned count; 347 348 if (off < 0 || len < 0) 349 panic("m_copydata"); 350 while (off > 0) { 351 if (m == 0) 352 panic("m_copydata"); 353 if (off < m->m_len) 354 break; 355 off -= m->m_len; 356 m = m->m_next; 357 } 358 while (len > 0) { 359 if (m == 0) 360 panic("m_copydata"); 361 count = min(m->m_len - off, len); 362 bcopy(mtod(m, caddr_t) + off, cp, count); 363 len -= count; 364 cp += count; 365 off = 0; 366 m = m->m_next; 367 } 368} 369 370/* 371 * Concatenate mbuf chain n to m. 372 * Both chains must be of the same type (e.g. MT_DATA). 373 * Any m_pkthdr is not updated. 374 */ 375void 376m_cat(m, n) 377 register struct mbuf *m, *n; 378{ 379 while (m->m_next) 380 m = m->m_next; 381 while (n) { 382 if (m->m_flags & M_EXT || 383 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { 384 /* just join the two chains */ 385 m->m_next = n; 386 return; 387 } 388 /* splat the data from one into the other */ 389 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 390 (u_int)n->m_len); 391 m->m_len += n->m_len; 392 n = m_free(n); 393 } 394} 395 396void 397m_adj(mp, req_len) 398 struct mbuf *mp; 399 int req_len; 400{ 401 register int len = req_len; 402 register struct mbuf *m; 403 register count; 404 405 if ((m = mp) == NULL) 406 return; 407 if (len >= 0) { 408 /* 409 * Trim from head. 410 */ 411 while (m != NULL && len > 0) { 412 if (m->m_len <= len) { 413 len -= m->m_len; 414 m->m_len = 0; 415 m = m->m_next; 416 } else { 417 m->m_len -= len; 418 m->m_data += len; 419 len = 0; 420 } 421 } 422 m = mp; 423 if (mp->m_flags & M_PKTHDR) 424 m->m_pkthdr.len -= (req_len - len); 425 } else { 426 /* 427 * Trim from tail. Scan the mbuf chain, 428 * calculating its length and finding the last mbuf. 429 * If the adjustment only affects this mbuf, then just 430 * adjust and return. Otherwise, rescan and truncate 431 * after the remaining size. 432 */ 433 len = -len; 434 count = 0; 435 for (;;) { 436 count += m->m_len; 437 if (m->m_next == (struct mbuf *)0) 438 break; 439 m = m->m_next; 440 } 441 if (m->m_len >= len) { 442 m->m_len -= len; 443 if (mp->m_flags & M_PKTHDR) 444 mp->m_pkthdr.len -= len; 445 return; 446 } 447 count -= len; 448 if (count < 0) 449 count = 0; 450 /* 451 * Correct length for chain is "count". 452 * Find the mbuf with last data, adjust its length, 453 * and toss data from remaining mbufs on chain. 454 */ 455 m = mp; 456 if (m->m_flags & M_PKTHDR) 457 m->m_pkthdr.len = count; 458 for (; m; m = m->m_next) { 459 if (m->m_len >= count) { 460 m->m_len = count; 461 break; 462 } 463 count -= m->m_len; 464 } 465 while (m->m_next) 466 (m = m->m_next) ->m_len = 0; 467 } 468} 469 470/* 471 * Rearange an mbuf chain so that len bytes are contiguous 472 * and in the data area of an mbuf (so that mtod and dtom 473 * will work for a structure of size len). Returns the resulting 474 * mbuf chain on success, frees it and returns null on failure. 475 * If there is room, it will add up to max_protohdr-len extra bytes to the 476 * contiguous region in an attempt to avoid being called next time. 477 */ 478int MPFail; 479 480struct mbuf * 481m_pullup(n, len) 482 register struct mbuf *n; 483 int len; 484{ 485 register struct mbuf *m; 486 register int count; 487 int space; 488 489 /* 490 * If first mbuf has no cluster, and has room for len bytes 491 * without shifting current data, pullup into it, 492 * otherwise allocate a new mbuf to prepend to the chain. 493 */ 494 if ((n->m_flags & M_EXT) == 0 && 495 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 496 if (n->m_len >= len) 497 return (n); 498 m = n; 499 n = n->m_next; 500 len -= m->m_len; 501 } else { 502 if (len > MHLEN) 503 goto bad; 504 MGET(m, M_DONTWAIT, n->m_type); 505 if (m == 0) 506 goto bad; 507 m->m_len = 0; 508 if (n->m_flags & M_PKTHDR) { 509 M_COPY_PKTHDR(m, n); 510 n->m_flags &= ~M_PKTHDR; 511 } 512 } 513 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 514 do { 515 count = min(min(max(len, max_protohdr), space), n->m_len); 516 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 517 (unsigned)count); 518 len -= count; 519 m->m_len += count; 520 n->m_len -= count; 521 space -= count; 522 if (n->m_len) 523 n->m_data += count; 524 else 525 n = m_free(n); 526 } while (len > 0 && n); 527 if (len > 0) { 528 (void) m_free(m); 529 goto bad; 530 } 531 m->m_next = n; 532 return (m); 533bad: 534 m_freem(n); 535 MPFail++; 536 return (0); 537} 538 539/* 540 * Partition an mbuf chain in two pieces, returning the tail -- 541 * all but the first len0 bytes. In case of failure, it returns NULL and 542 * attempts to restore the chain to its original state. 543 */ 544struct mbuf * 545m_split(m0, len0, wait) 546 register struct mbuf *m0; 547 int len0, wait; 548{ 549 register struct mbuf *m, *n; 550 unsigned len = len0, remain; 551 552 for (m = m0; m && len > m->m_len; m = m->m_next) 553 len -= m->m_len; 554 if (m == 0) 555 return (0); 556 remain = m->m_len - len; 557 if (m0->m_flags & M_PKTHDR) { 558 MGETHDR(n, wait, m0->m_type); 559 if (n == 0) 560 return (0); 561 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 562 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 563 m0->m_pkthdr.len = len0; 564 if (m->m_flags & M_EXT) 565 goto extpacket; 566 if (remain > MHLEN) { 567 /* m can't be the lead packet */ 568 MH_ALIGN(n, 0); 569 n->m_next = m_split(m, len, wait); 570 if (n->m_next == 0) { 571 (void) m_free(n); 572 return (0); 573 } else 574 return (n); 575 } else 576 MH_ALIGN(n, remain); 577 } else if (remain == 0) { 578 n = m->m_next; 579 m->m_next = 0; 580 return (n); 581 } else { 582 MGET(n, wait, m->m_type); 583 if (n == 0) 584 return (0); 585 M_ALIGN(n, remain); 586 } 587extpacket: 588 if (m->m_flags & M_EXT) { 589 n->m_flags |= M_EXT; 590 n->m_ext = m->m_ext; 591 mclrefcnt[mtocl(m->m_ext.ext_buf)]++; 592 m->m_ext.ext_size = 0; /* For Accounting XXXXXX danger */ 593 n->m_data = m->m_data + len; 594 } else { 595 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); 596 } 597 n->m_len = remain; 598 m->m_len = len; 599 n->m_next = m->m_next; 600 m->m_next = 0; 601 return (n); 602} 603/* 604 * Routine to copy from device local memory into mbufs. 605 */ 606struct mbuf * 607m_devget(buf, totlen, off0, ifp, copy) 608 char *buf; 609 int totlen, off0; 610 struct ifnet *ifp; 611 void (*copy)(); 612{ 613 register struct mbuf *m; 614 struct mbuf *top = 0, **mp = ⊤ 615 register int off = off0, len; 616 register char *cp; 617 char *epkt; 618 619 cp = buf; 620 epkt = cp + totlen; 621 if (off) { 622 cp += off + 2 * sizeof(u_short); 623 totlen -= 2 * sizeof(u_short); 624 } 625 MGETHDR(m, M_DONTWAIT, MT_DATA); 626 if (m == 0) 627 return (0); 628 m->m_pkthdr.rcvif = ifp; 629 m->m_pkthdr.len = totlen; 630 m->m_len = MHLEN; 631 632 while (totlen > 0) { 633 if (top) { 634 MGET(m, M_DONTWAIT, MT_DATA); 635 if (m == 0) { 636 m_freem(top); 637 return (0); 638 } 639 m->m_len = MLEN; 640 } 641 len = min(totlen, epkt - cp); 642 if (len >= MINCLSIZE) { 643 MCLGET(m, M_DONTWAIT); 644 if (m->m_flags & M_EXT) 645 m->m_len = len = min(len, MCLBYTES); 646 else 647 len = m->m_len; 648 } else { 649 /* 650 * Place initial small packet/header at end of mbuf. 651 */ 652 if (len < m->m_len) { 653 if (top == 0 && len + max_linkhdr <= m->m_len) 654 m->m_data += max_linkhdr; 655 m->m_len = len; 656 } else 657 len = m->m_len; 658 } 659 if (copy) 660 copy(cp, mtod(m, caddr_t), (unsigned)len); 661 else 662 bcopy(cp, mtod(m, caddr_t), (unsigned)len); 663 cp += len; 664 *mp = m; 665 mp = &m->m_next; 666 totlen -= len; 667 if (cp == epkt) 668 cp = buf; 669 } 670 return (top); 671} 672 673/* 674 * Copy data from a buffer back into the indicated mbuf chain, 675 * starting "off" bytes from the beginning, extending the mbuf 676 * chain if necessary. 677 */ 678void 679m_copyback(m0, off, len, cp) 680 struct mbuf *m0; 681 register int off; 682 register int len; 683 caddr_t cp; 684{ 685 register int mlen; 686 register struct mbuf *m = m0, *n; 687 int totlen = 0; 688 689 if (m0 == 0) 690 return; 691 while (off > (mlen = m->m_len)) { 692 off -= mlen; 693 totlen += mlen; 694 if (m->m_next == 0) { 695 n = m_getclr(M_DONTWAIT, m->m_type); 696 if (n == 0) 697 goto out; 698 n->m_len = min(MLEN, len + off); 699 m->m_next = n; 700 } 701 m = m->m_next; 702 } 703 while (len > 0) { 704 mlen = min (m->m_len - off, len); 705 bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen); 706 cp += mlen; 707 len -= mlen; 708 mlen += off; 709 off = 0; 710 totlen += mlen; 711 if (len == 0) 712 break; 713 if (m->m_next == 0) { 714 n = m_get(M_DONTWAIT, m->m_type); 715 if (n == 0) 716 break; 717 n->m_len = min(MLEN, len); 718 m->m_next = n; 719 } 720 m = m->m_next; 721 } 722out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) 723 m->m_pkthdr.len = totlen; 724} 725