70int nmbcnt; 71u_long m_mballoc_wid = 0; 72u_long m_clalloc_wid = 0; 73 74/* 75 * freelist header structures... 76 * mbffree_lst, mclfree_lst, mcntfree_lst 77 */ 78struct mbffree_lst mmbfree; 79struct mclfree_lst mclfree; 80struct mcntfree_lst mcntfree; 81struct mtx mbuf_mtx; 82 83/* 84 * sysctl(8) exported objects 85 */ 86SYSCTL_DECL(_kern_ipc); 87SYSCTL_INT(_kern_ipc, KIPC_MAX_LINKHDR, max_linkhdr, CTLFLAG_RW, 88 &max_linkhdr, 0, ""); 89SYSCTL_INT(_kern_ipc, KIPC_MAX_PROTOHDR, max_protohdr, CTLFLAG_RW, 90 &max_protohdr, 0, ""); 91SYSCTL_INT(_kern_ipc, KIPC_MAX_HDR, max_hdr, CTLFLAG_RW, &max_hdr, 0, ""); 92SYSCTL_INT(_kern_ipc, KIPC_MAX_DATALEN, max_datalen, CTLFLAG_RW, 93 &max_datalen, 0, ""); 94SYSCTL_INT(_kern_ipc, OID_AUTO, mbuf_wait, CTLFLAG_RW, 95 &mbuf_wait, 0, ""); 96SYSCTL_STRUCT(_kern_ipc, KIPC_MBSTAT, mbstat, CTLFLAG_RD, &mbstat, mbstat, ""); 97SYSCTL_OPAQUE(_kern_ipc, OID_AUTO, mbtypes, CTLFLAG_RD, mbtypes, 98 sizeof(mbtypes), "LU", ""); 99SYSCTL_INT(_kern_ipc, KIPC_NMBCLUSTERS, nmbclusters, CTLFLAG_RD, 100 &nmbclusters, 0, "Maximum number of mbuf clusters available"); 101SYSCTL_INT(_kern_ipc, OID_AUTO, nmbufs, CTLFLAG_RD, &nmbufs, 0, 102 "Maximum number of mbufs available"); 103SYSCTL_INT(_kern_ipc, OID_AUTO, nmbcnt, CTLFLAG_RD, &nmbcnt, 0, 104 "Maximum number of ext_buf counters available"); 105
|
135 /* 136 * Setup the mb_map, allocate requested VM space. 137 */ 138 mb_map_size = (vm_size_t)(nmbufs * MSIZE + nmbclusters * MCLBYTES + 139 nmbcnt * sizeof(union mext_refcnt)); 140 mb_map_size = rounddown(mb_map_size, PAGE_SIZE); 141 mb_map = kmem_suballoc(kmem_map, (vm_offset_t *)&mbutl, &maxaddr, 142 mb_map_size); 143 /* XXX XXX XXX: mb_map->system_map = 1; */ 144 145 /* 146 * Initialize the free list headers, and setup locks for lists. 147 */ 148 mmbfree.m_head = NULL; 149 mclfree.m_head = NULL; 150 mcntfree.m_head = NULL; 151 mtx_init(&mbuf_mtx, "mbuf free list lock", MTX_DEF); 152 cv_init(&mmbfree.m_starved, "mbuf free list starved cv"); 153 cv_init(&mclfree.m_starved, "mbuf cluster free list starved cv"); 154 155 /* 156 * Initialize mbuf subsystem (sysctl exported) statistics structure. 157 */ 158 mbstat.m_msize = MSIZE; 159 mbstat.m_mclbytes = MCLBYTES; 160 mbstat.m_minclsize = MINCLSIZE; 161 mbstat.m_mlen = MLEN; 162 mbstat.m_mhlen = MHLEN; 163 164 /* 165 * Perform some initial allocations. 166 */ 167 mtx_lock(&mbuf_mtx); 168 if (m_alloc_ref(REF_INIT, M_DONTWAIT) == 0) 169 goto bad; 170 if (m_mballoc(NMB_INIT, M_DONTWAIT) == 0) 171 goto bad; 172 if (m_clalloc(NCL_INIT, M_DONTWAIT) == 0) 173 goto bad; 174 mtx_unlock(&mbuf_mtx); 175 176 return; 177bad: 178 panic("mbinit: failed to initialize mbuf subsystem!"); 179} 180 181/* 182 * Allocate at least nmb reference count structs and place them 183 * on the ref cnt free list. 184 * 185 * Must be called with the mcntfree lock held. 186 */ 187int 188m_alloc_ref(u_int nmb, int how) 189{ 190 caddr_t p; 191 u_int nbytes; 192 int i; 193 194 /* 195 * We don't cap the amount of memory that can be used 196 * by the reference counters, like we do for mbufs and 197 * mbuf clusters. In fact, we're absolutely sure that we 198 * won't ever be going over our allocated space. We keep enough 199 * space in mb_map to accomodate maximum values of allocatable 200 * external buffers including, but not limited to, clusters. 201 * (That's also why we won't have to have wait routines for 202 * counters). 203 * 204 * If we're in here, we're absolutely certain to be returning 205 * succesfully, as long as there is physical memory to accomodate 206 * us. And if there isn't, but we're willing to wait, then 207 * kmem_malloc() will do the only waiting needed. 208 */ 209 210 nbytes = round_page(nmb * sizeof(union mext_refcnt)); 211 if (1 /* XXX: how == M_TRYWAIT */) 212 mtx_unlock(&mbuf_mtx); 213 if ((p = (caddr_t)kmem_malloc(mb_map, nbytes, how == M_TRYWAIT ? 214 M_WAITOK : M_NOWAIT)) == NULL) { 215 if (1 /* XXX: how == M_TRYWAIT */) 216 mtx_lock(&mbuf_mtx); 217 return (0); 218 } 219 nmb = nbytes / sizeof(union mext_refcnt); 220 221 /* 222 * We don't let go of the mutex in order to avoid a race. 223 * It is up to the caller to let go of the mutex. 224 */ 225 if (1 /* XXX: how == M_TRYWAIT */) 226 mtx_lock(&mbuf_mtx); 227 for (i = 0; i < nmb; i++) { 228 ((union mext_refcnt *)p)->next_ref = mcntfree.m_head; 229 mcntfree.m_head = (union mext_refcnt *)p; 230 p += sizeof(union mext_refcnt); 231 mbstat.m_refree++; 232 } 233 mbstat.m_refcnt += nmb; 234 235 return (1); 236} 237 238/* 239 * Allocate at least nmb mbufs and place on mbuf free list. 240 * 241 * Must be called with the mmbfree lock held. 242 */ 243int 244m_mballoc(int nmb, int how) 245{ 246 caddr_t p; 247 int i; 248 int nbytes; 249 250 nbytes = round_page(nmb * MSIZE); 251 nmb = nbytes / MSIZE; 252 253 /* 254 * If we've hit the mbuf limit, stop allocating from mb_map. 255 * Also, once we run out of map space, it will be impossible to 256 * get any more (nothing is ever freed back to the map). 257 */ 258 if (mb_map_full || ((nmb + mbstat.m_mbufs) > nmbufs)) 259 return (0); 260 261 if (1 /* XXX: how == M_TRYWAIT */) 262 mtx_unlock(&mbuf_mtx); 263 p = (caddr_t)kmem_malloc(mb_map, nbytes, how == M_TRYWAIT ? 264 M_WAITOK : M_NOWAIT); 265 if (1 /* XXX: how == M_TRYWAIT */) { 266 mtx_lock(&mbuf_mtx); 267 if (p == NULL) 268 mbstat.m_wait++; 269 } 270 271 /* 272 * Either the map is now full, or `how' is M_DONTWAIT and there 273 * are no pages left. 274 */ 275 if (p == NULL) 276 return (0); 277 278 /* 279 * We don't let go of the mutex in order to avoid a race. 280 * It is up to the caller to let go of the mutex when done 281 * with grabbing the mbuf from the free list. 282 */ 283 for (i = 0; i < nmb; i++) { 284 ((struct mbuf *)p)->m_next = mmbfree.m_head; 285 mmbfree.m_head = (struct mbuf *)p; 286 p += MSIZE; 287 } 288 mbstat.m_mbufs += nmb; 289 mbtypes[MT_FREE] += nmb; 290 return (1); 291} 292 293/* 294 * Once the mb_map has been exhausted and if the call to the allocation macros 295 * (or, in some cases, functions) is with M_TRYWAIT, then it is necessary to 296 * rely solely on reclaimed mbufs. 297 * 298 * Here we request for the protocols to free up some resources and, if we 299 * still cannot get anything, then we wait for an mbuf to be freed for a 300 * designated (mbuf_wait) time, at most. 301 * 302 * Must be called with the mmbfree mutex held. 303 */ 304struct mbuf * 305m_mballoc_wait(void) 306{ 307 struct mbuf *p = NULL; 308 309 /* 310 * See if we can drain some resources out of the protocols. 311 * We drop the mmbfree mutex to avoid recursing into it in some of 312 * the drain routines. Clearly, we're faced with a race here because 313 * once something is freed during the drain, it may be grabbed right 314 * from under us by some other thread. But we accept this possibility 315 * in order to avoid a potentially large lock recursion and, more 316 * importantly, to avoid a potential lock order reversal which may 317 * result in deadlock (See comment above m_reclaim()). 318 */ 319 mtx_unlock(&mbuf_mtx); 320 m_reclaim(); 321 322 mtx_lock(&mbuf_mtx); 323 _MGET(p, M_DONTWAIT); 324 325 if (p == NULL) { 326 int retval; 327 328 m_mballoc_wid++; 329 retval = cv_timedwait(&mmbfree.m_starved, &mbuf_mtx, 330 mbuf_wait); 331 m_mballoc_wid--; 332 333 /* 334 * If we got signaled (i.e. didn't time out), allocate. 335 */ 336 if (retval == 0) 337 _MGET(p, M_DONTWAIT); 338 } 339 340 if (p != NULL) { 341 mbstat.m_wait++; 342 if (mmbfree.m_head != NULL) 343 MBWAKEUP(m_mballoc_wid, &mmbfree.m_starved); 344 } 345 346 return (p); 347} 348 349/* 350 * Allocate some number of mbuf clusters 351 * and place on cluster free list. 352 * 353 * Must be called with the mclfree lock held. 354 */ 355int 356m_clalloc(int ncl, int how) 357{ 358 caddr_t p; 359 int i; 360 int npg_sz; 361 362 npg_sz = round_page(ncl * MCLBYTES); 363 ncl = npg_sz / MCLBYTES; 364 365 /* 366 * If the map is now full (nothing will ever be freed to it). 367 * If we've hit the mcluster number limit, stop allocating from 368 * mb_map. 369 */ 370 if (mb_map_full || ((ncl + mbstat.m_clusters) > nmbclusters)) 371 return (0); 372 373 if (1 /* XXX: how == M_TRYWAIT */) 374 mtx_unlock(&mbuf_mtx); 375 p = (caddr_t)kmem_malloc(mb_map, npg_sz, 376 how == M_TRYWAIT ? M_WAITOK : M_NOWAIT); 377 if (1 /* XXX: how == M_TRYWAIT */) 378 mtx_lock(&mbuf_mtx); 379 380 /* 381 * Either the map is now full, or `how' is M_DONTWAIT and there 382 * are no pages left. 383 */ 384 if (p == NULL) 385 return (0); 386 387 for (i = 0; i < ncl; i++) { 388 ((union mcluster *)p)->mcl_next = mclfree.m_head; 389 mclfree.m_head = (union mcluster *)p; 390 p += MCLBYTES; 391 mbstat.m_clfree++; 392 } 393 mbstat.m_clusters += ncl; 394 return (1); 395} 396 397/* 398 * Once the mb_map submap has been exhausted and the allocation is called with 399 * M_TRYWAIT, we rely on the mclfree list. If nothing is free, we will 400 * block on a cv for a designated amount of time (mbuf_wait) or until we're 401 * signaled due to sudden mcluster availability. 402 * 403 * Must be called with the mclfree lock held. 404 */ 405caddr_t 406m_clalloc_wait(void) 407{ 408 caddr_t p = NULL; 409 int retval; 410 411 m_clalloc_wid++; 412 retval = cv_timedwait(&mclfree.m_starved, &mbuf_mtx, mbuf_wait); 413 m_clalloc_wid--; 414 415 /* 416 * Now that we (think) that we've got something, try again. 417 */ 418 if (retval == 0) 419 _MCLALLOC(p, M_DONTWAIT); 420 421 if (p != NULL) { 422 mbstat.m_wait++; 423 if (mclfree.m_head != NULL) 424 MBWAKEUP(m_clalloc_wid, &mclfree.m_starved); 425 } 426 427 return (p); 428} 429 430/* 431 * m_reclaim: drain protocols in hopes to free up some resources... 432 * 433 * XXX: No locks should be held going in here. The drain routines have 434 * to presently acquire some locks which raises the possibility of lock 435 * order violation if we're holding any mutex if that mutex is acquired in 436 * reverse order relative to one of the locks in the drain routines. 437 */ 438static void 439m_reclaim(void) 440{ 441 struct domain *dp; 442 struct protosw *pr; 443 444#ifdef WITNESS 445 KASSERT(witness_list(curproc) == 0, 446 ("m_reclaim called with locks held")); 447#endif 448 449 for (dp = domains; dp; dp = dp->dom_next) 450 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) 451 if (pr->pr_drain) 452 (*pr->pr_drain)(); 453 mbstat.m_drain++; 454} 455 456/* 457 * Space allocation routines. 458 * Some of these are also available as macros 459 * for critical paths. 460 */ 461struct mbuf * 462m_get(int how, int type) 463{ 464 struct mbuf *m; 465 466 MGET(m, how, type); 467 return (m); 468} 469 470struct mbuf * 471m_gethdr(int how, int type) 472{ 473 struct mbuf *m; 474 475 MGETHDR(m, how, type); 476 return (m); 477} 478 479struct mbuf * 480m_getclr(int how, int type) 481{ 482 struct mbuf *m; 483 484 MGET(m, how, type); 485 if (m != NULL) 486 bzero(mtod(m, caddr_t), MLEN); 487 return (m); 488} 489 490struct mbuf * 491m_free(struct mbuf *m) 492{ 493 struct mbuf *n; 494 495 MFREE(m, n); 496 return (n); 497} 498 499/* 500 * struct mbuf * 501 * m_getm(m, len, how, type) 502 * 503 * This will allocate len-worth of mbufs and/or mbuf clusters (whatever fits 504 * best) and return a pointer to the top of the allocated chain. If m is 505 * non-null, then we assume that it is a single mbuf or an mbuf chain to 506 * which we want len bytes worth of mbufs and/or clusters attached, and so 507 * if we succeed in allocating it, we will just return a pointer to m. 508 * 509 * If we happen to fail at any point during the allocation, we will free 510 * up everything we have already allocated and return NULL. 511 * 512 */ 513struct mbuf * 514m_getm(struct mbuf *m, int len, int how, int type) 515{ 516 struct mbuf *top, *tail, *mp, *mtail = NULL; 517 518 KASSERT(len >= 0, ("len is < 0 in m_getm")); 519 520 MGET(mp, how, type); 521 if (mp == NULL) 522 return (NULL); 523 else if (len > MINCLSIZE) { 524 MCLGET(mp, how); 525 if ((mp->m_flags & M_EXT) == 0) { 526 m_free(mp); 527 return (NULL); 528 } 529 } 530 mp->m_len = 0; 531 len -= M_TRAILINGSPACE(mp); 532 533 if (m != NULL) 534 for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next); 535 else 536 m = mp; 537 538 top = tail = mp; 539 while (len > 0) { 540 MGET(mp, how, type); 541 if (mp == NULL) 542 goto failed; 543 544 tail->m_next = mp; 545 tail = mp; 546 if (len > MINCLSIZE) { 547 MCLGET(mp, how); 548 if ((mp->m_flags & M_EXT) == 0) 549 goto failed; 550 } 551 552 mp->m_len = 0; 553 len -= M_TRAILINGSPACE(mp); 554 } 555 556 if (mtail != NULL) 557 mtail->m_next = top; 558 return (m); 559 560failed: 561 m_freem(top); 562 return (NULL); 563} 564 565void 566m_freem(struct mbuf *m) 567{ 568 struct mbuf *n; 569 570 if (m == NULL) 571 return; 572 do { 573 /* 574 * we do need to check non-first mbuf, since some of existing 575 * code does not call M_PREPEND properly. 576 * (example: call to bpf_mtap from drivers) 577 */ 578 if ((m->m_flags & M_PKTHDR) != 0 && m->m_pkthdr.aux) { 579 m_freem(m->m_pkthdr.aux); 580 m->m_pkthdr.aux = NULL; 581 } 582 MFREE(m, n); 583 m = n; 584 } while (m); 585} 586 587/* 588 * Lesser-used path for M_PREPEND: 589 * allocate new mbuf to prepend to chain, 590 * copy junk along. 591 */ 592struct mbuf * 593m_prepend(struct mbuf *m, int len, int how) 594{ 595 struct mbuf *mn; 596 597 MGET(mn, how, m->m_type); 598 if (mn == NULL) { 599 m_freem(m); 600 return (NULL); 601 } 602 if (m->m_flags & M_PKTHDR) { 603 M_COPY_PKTHDR(mn, m); 604 m->m_flags &= ~M_PKTHDR; 605 } 606 mn->m_next = m; 607 m = mn; 608 if (len < MHLEN) 609 MH_ALIGN(m, len); 610 m->m_len = len; 611 return (m); 612} 613 614/* 615 * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 616 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 617 * The wait parameter is a choice of M_TRYWAIT/M_DONTWAIT from caller. 618 * Note that the copy is read-only, because clusters are not copied, 619 * only their reference counts are incremented. 620 */ 621struct mbuf * 622m_copym(struct mbuf *m, int off0, int len, int wait) 623{ 624 struct mbuf *n, **np; 625 int off = off0; 626 struct mbuf *top; 627 int copyhdr = 0; 628 629 KASSERT(off >= 0, ("m_copym, negative off %d", off)); 630 KASSERT(len >= 0, ("m_copym, negative len %d", len)); 631 if (off == 0 && m->m_flags & M_PKTHDR) 632 copyhdr = 1; 633 while (off > 0) { 634 KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain")); 635 if (off < m->m_len) 636 break; 637 off -= m->m_len; 638 m = m->m_next; 639 } 640 np = ⊤ 641 top = 0; 642 while (len > 0) { 643 if (m == NULL) { 644 KASSERT(len == M_COPYALL, 645 ("m_copym, length > size of mbuf chain")); 646 break; 647 } 648 MGET(n, wait, m->m_type); 649 *np = n; 650 if (n == NULL) 651 goto nospace; 652 if (copyhdr) { 653 M_COPY_PKTHDR(n, m); 654 if (len == M_COPYALL) 655 n->m_pkthdr.len -= off0; 656 else 657 n->m_pkthdr.len = len; 658 copyhdr = 0; 659 } 660 n->m_len = min(len, m->m_len - off); 661 if (m->m_flags & M_EXT) { 662 n->m_data = m->m_data + off; 663 n->m_ext = m->m_ext; 664 n->m_flags |= M_EXT; 665 MEXT_ADD_REF(m); 666 } else 667 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), 668 (unsigned)n->m_len); 669 if (len != M_COPYALL) 670 len -= n->m_len; 671 off = 0; 672 m = m->m_next; 673 np = &n->m_next; 674 } 675 if (top == NULL) { 676 mtx_lock(&mbuf_mtx); 677 mbstat.m_mcfail++; 678 mtx_unlock(&mbuf_mtx); 679 } 680 return (top); 681nospace: 682 m_freem(top); 683 mtx_lock(&mbuf_mtx); 684 mbstat.m_mcfail++; 685 mtx_unlock(&mbuf_mtx); 686 return (NULL); 687} 688 689/* 690 * Copy an entire packet, including header (which must be present). 691 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'. 692 * Note that the copy is read-only, because clusters are not copied, 693 * only their reference counts are incremented. 694 * Preserve alignment of the first mbuf so if the creator has left 695 * some room at the beginning (e.g. for inserting protocol headers) 696 * the copies still have the room available. 697 */ 698struct mbuf * 699m_copypacket(struct mbuf *m, int how) 700{ 701 struct mbuf *top, *n, *o; 702 703 MGET(n, how, m->m_type); 704 top = n; 705 if (n == NULL) 706 goto nospace; 707 708 M_COPY_PKTHDR(n, m); 709 n->m_len = m->m_len; 710 if (m->m_flags & M_EXT) { 711 n->m_data = m->m_data; 712 n->m_ext = m->m_ext; 713 n->m_flags |= M_EXT; 714 MEXT_ADD_REF(m); 715 } else { 716 n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat ); 717 bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 718 } 719 720 m = m->m_next; 721 while (m) { 722 MGET(o, how, m->m_type); 723 if (o == NULL) 724 goto nospace; 725 726 n->m_next = o; 727 n = n->m_next; 728 729 n->m_len = m->m_len; 730 if (m->m_flags & M_EXT) { 731 n->m_data = m->m_data; 732 n->m_ext = m->m_ext; 733 n->m_flags |= M_EXT; 734 MEXT_ADD_REF(m); 735 } else { 736 bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 737 } 738 739 m = m->m_next; 740 } 741 return top; 742nospace: 743 m_freem(top); 744 mtx_lock(&mbuf_mtx); 745 mbstat.m_mcfail++; 746 mtx_unlock(&mbuf_mtx); 747 return (NULL); 748} 749 750/* 751 * Copy data from an mbuf chain starting "off" bytes from the beginning, 752 * continuing for "len" bytes, into the indicated buffer. 753 */ 754void 755m_copydata(struct mbuf *m, int off, int len, caddr_t cp) 756{ 757 unsigned count; 758 759 KASSERT(off >= 0, ("m_copydata, negative off %d", off)); 760 KASSERT(len >= 0, ("m_copydata, negative len %d", len)); 761 while (off > 0) { 762 KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain")); 763 if (off < m->m_len) 764 break; 765 off -= m->m_len; 766 m = m->m_next; 767 } 768 while (len > 0) { 769 KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain")); 770 count = min(m->m_len - off, len); 771 bcopy(mtod(m, caddr_t) + off, cp, count); 772 len -= count; 773 cp += count; 774 off = 0; 775 m = m->m_next; 776 } 777} 778 779/* 780 * Copy a packet header mbuf chain into a completely new chain, including 781 * copying any mbuf clusters. Use this instead of m_copypacket() when 782 * you need a writable copy of an mbuf chain. 783 */ 784struct mbuf * 785m_dup(struct mbuf *m, int how) 786{ 787 struct mbuf **p, *top = NULL; 788 int remain, moff, nsize; 789 790 /* Sanity check */ 791 if (m == NULL) 792 return (NULL); 793 KASSERT((m->m_flags & M_PKTHDR) != 0, ("%s: !PKTHDR", __FUNCTION__)); 794 795 /* While there's more data, get a new mbuf, tack it on, and fill it */ 796 remain = m->m_pkthdr.len; 797 moff = 0; 798 p = ⊤ 799 while (remain > 0 || top == NULL) { /* allow m->m_pkthdr.len == 0 */ 800 struct mbuf *n; 801 802 /* Get the next new mbuf */ 803 MGET(n, how, m->m_type); 804 if (n == NULL) 805 goto nospace; 806 if (top == NULL) { /* first one, must be PKTHDR */ 807 M_COPY_PKTHDR(n, m); 808 nsize = MHLEN; 809 } else /* not the first one */ 810 nsize = MLEN; 811 if (remain >= MINCLSIZE) { 812 MCLGET(n, how); 813 if ((n->m_flags & M_EXT) == 0) { 814 (void)m_free(n); 815 goto nospace; 816 } 817 nsize = MCLBYTES; 818 } 819 n->m_len = 0; 820 821 /* Link it into the new chain */ 822 *p = n; 823 p = &n->m_next; 824 825 /* Copy data from original mbuf(s) into new mbuf */ 826 while (n->m_len < nsize && m != NULL) { 827 int chunk = min(nsize - n->m_len, m->m_len - moff); 828 829 bcopy(m->m_data + moff, n->m_data + n->m_len, chunk); 830 moff += chunk; 831 n->m_len += chunk; 832 remain -= chunk; 833 if (moff == m->m_len) { 834 m = m->m_next; 835 moff = 0; 836 } 837 } 838 839 /* Check correct total mbuf length */ 840 KASSERT((remain > 0 && m != NULL) || (remain == 0 && m == NULL), 841 ("%s: bogus m_pkthdr.len", __FUNCTION__)); 842 } 843 return (top); 844 845nospace: 846 m_freem(top); 847 mtx_lock(&mbuf_mtx); 848 mbstat.m_mcfail++; 849 mtx_unlock(&mbuf_mtx); 850 return (NULL); 851} 852 853/* 854 * Concatenate mbuf chain n to m. 855 * Both chains must be of the same type (e.g. MT_DATA). 856 * Any m_pkthdr is not updated. 857 */ 858void 859m_cat(struct mbuf *m, struct mbuf *n) 860{ 861 while (m->m_next) 862 m = m->m_next; 863 while (n) { 864 if (m->m_flags & M_EXT || 865 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { 866 /* just join the two chains */ 867 m->m_next = n; 868 return; 869 } 870 /* splat the data from one into the other */ 871 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 872 (u_int)n->m_len); 873 m->m_len += n->m_len; 874 n = m_free(n); 875 } 876} 877 878void 879m_adj(struct mbuf *mp, int req_len) 880{ 881 int len = req_len; 882 struct mbuf *m; 883 int count; 884 885 if ((m = mp) == NULL) 886 return; 887 if (len >= 0) { 888 /* 889 * Trim from head. 890 */ 891 while (m != NULL && len > 0) { 892 if (m->m_len <= len) { 893 len -= m->m_len; 894 m->m_len = 0; 895 m = m->m_next; 896 } else { 897 m->m_len -= len; 898 m->m_data += len; 899 len = 0; 900 } 901 } 902 m = mp; 903 if (mp->m_flags & M_PKTHDR) 904 m->m_pkthdr.len -= (req_len - len); 905 } else { 906 /* 907 * Trim from tail. Scan the mbuf chain, 908 * calculating its length and finding the last mbuf. 909 * If the adjustment only affects this mbuf, then just 910 * adjust and return. Otherwise, rescan and truncate 911 * after the remaining size. 912 */ 913 len = -len; 914 count = 0; 915 for (;;) { 916 count += m->m_len; 917 if (m->m_next == (struct mbuf *)0) 918 break; 919 m = m->m_next; 920 } 921 if (m->m_len >= len) { 922 m->m_len -= len; 923 if (mp->m_flags & M_PKTHDR) 924 mp->m_pkthdr.len -= len; 925 return; 926 } 927 count -= len; 928 if (count < 0) 929 count = 0; 930 /* 931 * Correct length for chain is "count". 932 * Find the mbuf with last data, adjust its length, 933 * and toss data from remaining mbufs on chain. 934 */ 935 m = mp; 936 if (m->m_flags & M_PKTHDR) 937 m->m_pkthdr.len = count; 938 for (; m; m = m->m_next) { 939 if (m->m_len >= count) { 940 m->m_len = count; 941 break; 942 } 943 count -= m->m_len; 944 } 945 while (m->m_next) 946 (m = m->m_next) ->m_len = 0; 947 } 948} 949 950/* 951 * Rearange an mbuf chain so that len bytes are contiguous 952 * and in the data area of an mbuf (so that mtod and dtom 953 * will work for a structure of size len). Returns the resulting 954 * mbuf chain on success, frees it and returns null on failure. 955 * If there is room, it will add up to max_protohdr-len extra bytes to the 956 * contiguous region in an attempt to avoid being called next time. 957 */ 958struct mbuf * 959m_pullup(struct mbuf *n, int len) 960{ 961 struct mbuf *m; 962 int count; 963 int space; 964 965 /* 966 * If first mbuf has no cluster, and has room for len bytes 967 * without shifting current data, pullup into it, 968 * otherwise allocate a new mbuf to prepend to the chain. 969 */ 970 if ((n->m_flags & M_EXT) == 0 && 971 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 972 if (n->m_len >= len) 973 return (n); 974 m = n; 975 n = n->m_next; 976 len -= m->m_len; 977 } else { 978 if (len > MHLEN) 979 goto bad; 980 MGET(m, M_DONTWAIT, n->m_type); 981 if (m == NULL) 982 goto bad; 983 m->m_len = 0; 984 if (n->m_flags & M_PKTHDR) { 985 M_COPY_PKTHDR(m, n); 986 n->m_flags &= ~M_PKTHDR; 987 } 988 } 989 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 990 do { 991 count = min(min(max(len, max_protohdr), space), n->m_len); 992 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 993 (unsigned)count); 994 len -= count; 995 m->m_len += count; 996 n->m_len -= count; 997 space -= count; 998 if (n->m_len) 999 n->m_data += count; 1000 else 1001 n = m_free(n); 1002 } while (len > 0 && n); 1003 if (len > 0) { 1004 (void) m_free(m); 1005 goto bad; 1006 } 1007 m->m_next = n; 1008 return (m); 1009bad: 1010 m_freem(n); 1011 mtx_lock(&mbuf_mtx); 1012 mbstat.m_mpfail++; 1013 mtx_unlock(&mbuf_mtx); 1014 return (NULL); 1015} 1016 1017/* 1018 * Partition an mbuf chain in two pieces, returning the tail -- 1019 * all but the first len0 bytes. In case of failure, it returns NULL and 1020 * attempts to restore the chain to its original state. 1021 */ 1022struct mbuf * 1023m_split(struct mbuf *m0, int len0, int wait) 1024{ 1025 struct mbuf *m, *n; 1026 unsigned len = len0, remain; 1027 1028 for (m = m0; m && len > m->m_len; m = m->m_next) 1029 len -= m->m_len; 1030 if (m == NULL) 1031 return (NULL); 1032 remain = m->m_len - len; 1033 if (m0->m_flags & M_PKTHDR) { 1034 MGETHDR(n, wait, m0->m_type); 1035 if (n == NULL) 1036 return (NULL); 1037 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 1038 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 1039 m0->m_pkthdr.len = len0; 1040 if (m->m_flags & M_EXT) 1041 goto extpacket; 1042 if (remain > MHLEN) { 1043 /* m can't be the lead packet */ 1044 MH_ALIGN(n, 0); 1045 n->m_next = m_split(m, len, wait); 1046 if (n->m_next == NULL) { 1047 (void) m_free(n); 1048 return (NULL); 1049 } else 1050 return (n); 1051 } else 1052 MH_ALIGN(n, remain); 1053 } else if (remain == 0) { 1054 n = m->m_next; 1055 m->m_next = NULL; 1056 return (n); 1057 } else { 1058 MGET(n, wait, m->m_type); 1059 if (n == NULL) 1060 return (NULL); 1061 M_ALIGN(n, remain); 1062 } 1063extpacket: 1064 if (m->m_flags & M_EXT) { 1065 n->m_flags |= M_EXT; 1066 n->m_ext = m->m_ext; 1067 MEXT_ADD_REF(m); 1068 m->m_ext.ext_size = 0; /* For Accounting XXXXXX danger */ 1069 n->m_data = m->m_data + len; 1070 } else { 1071 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); 1072 } 1073 n->m_len = remain; 1074 m->m_len = len; 1075 n->m_next = m->m_next; 1076 m->m_next = NULL; 1077 return (n); 1078} 1079/* 1080 * Routine to copy from device local memory into mbufs. 1081 */ 1082struct mbuf * 1083m_devget(char *buf, int totlen, int off0, struct ifnet *ifp, 1084 void (*copy)(char *from, caddr_t to, u_int len)) 1085{ 1086 struct mbuf *m; 1087 struct mbuf *top = 0, **mp = ⊤ 1088 int off = off0, len; 1089 char *cp; 1090 char *epkt; 1091 1092 cp = buf; 1093 epkt = cp + totlen; 1094 if (off) { 1095 cp += off + 2 * sizeof(u_short); 1096 totlen -= 2 * sizeof(u_short); 1097 } 1098 MGETHDR(m, M_DONTWAIT, MT_DATA); 1099 if (m == NULL) 1100 return (NULL); 1101 m->m_pkthdr.rcvif = ifp; 1102 m->m_pkthdr.len = totlen; 1103 m->m_len = MHLEN; 1104 1105 while (totlen > 0) { 1106 if (top) { 1107 MGET(m, M_DONTWAIT, MT_DATA); 1108 if (m == NULL) { 1109 m_freem(top); 1110 return (NULL); 1111 } 1112 m->m_len = MLEN; 1113 } 1114 len = min(totlen, epkt - cp); 1115 if (len >= MINCLSIZE) { 1116 MCLGET(m, M_DONTWAIT); 1117 if (m->m_flags & M_EXT) 1118 m->m_len = len = min(len, MCLBYTES); 1119 else 1120 len = m->m_len; 1121 } else { 1122 /* 1123 * Place initial small packet/header at end of mbuf. 1124 */ 1125 if (len < m->m_len) { 1126 if (top == NULL && len + 1127 max_linkhdr <= m->m_len) 1128 m->m_data += max_linkhdr; 1129 m->m_len = len; 1130 } else 1131 len = m->m_len; 1132 } 1133 if (copy) 1134 copy(cp, mtod(m, caddr_t), (unsigned)len); 1135 else 1136 bcopy(cp, mtod(m, caddr_t), (unsigned)len); 1137 cp += len; 1138 *mp = m; 1139 mp = &m->m_next; 1140 totlen -= len; 1141 if (cp == epkt) 1142 cp = buf; 1143 } 1144 return (top); 1145} 1146 1147/* 1148 * Copy data from a buffer back into the indicated mbuf chain, 1149 * starting "off" bytes from the beginning, extending the mbuf 1150 * chain if necessary. 1151 */ 1152void 1153m_copyback(struct mbuf *m0, int off, int len, caddr_t cp) 1154{ 1155 int mlen; 1156 struct mbuf *m = m0, *n; 1157 int totlen = 0; 1158 1159 if (m0 == NULL) 1160 return; 1161 while (off > (mlen = m->m_len)) { 1162 off -= mlen; 1163 totlen += mlen; 1164 if (m->m_next == NULL) { 1165 n = m_getclr(M_DONTWAIT, m->m_type); 1166 if (n == NULL) 1167 goto out; 1168 n->m_len = min(MLEN, len + off); 1169 m->m_next = n; 1170 } 1171 m = m->m_next; 1172 } 1173 while (len > 0) { 1174 mlen = min (m->m_len - off, len); 1175 bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen); 1176 cp += mlen; 1177 len -= mlen; 1178 mlen += off; 1179 off = 0; 1180 totlen += mlen; 1181 if (len == 0) 1182 break; 1183 if (m->m_next == NULL) { 1184 n = m_get(M_DONTWAIT, m->m_type); 1185 if (n == NULL) 1186 break; 1187 n->m_len = min(MLEN, len); 1188 m->m_next = n; 1189 } 1190 m = m->m_next; 1191 } 1192out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) 1193 m->m_pkthdr.len = totlen; 1194} 1195 1196void 1197m_print(const struct mbuf *m) 1198{ 1199 int len; 1200 const struct mbuf *m2; 1201 1202 len = m->m_pkthdr.len; 1203 m2 = m; 1204 while (len) { 1205 printf("%p %*D\n", m2, m2->m_len, (u_char *)m2->m_data, "-"); 1206 len -= m2->m_len; 1207 m2 = m2->m_next; 1208 } 1209 return; 1210}
|