uipc_mbuf.c revision 153232
1139804Simp/*- 21541Srgrimes * Copyright (c) 1982, 1986, 1988, 1991, 1993 31541Srgrimes * The Regents of the University of California. All rights reserved. 41541Srgrimes * 51541Srgrimes * Redistribution and use in source and binary forms, with or without 61541Srgrimes * modification, are permitted provided that the following conditions 71541Srgrimes * are met: 81541Srgrimes * 1. Redistributions of source code must retain the above copyright 91541Srgrimes * notice, this list of conditions and the following disclaimer. 101541Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 111541Srgrimes * notice, this list of conditions and the following disclaimer in the 121541Srgrimes * documentation and/or other materials provided with the distribution. 131541Srgrimes * 4. Neither the name of the University nor the names of its contributors 141541Srgrimes * may be used to endorse or promote products derived from this software 151541Srgrimes * without specific prior written permission. 161541Srgrimes * 171541Srgrimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 181541Srgrimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 191541Srgrimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 201541Srgrimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 211541Srgrimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 221541Srgrimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 231541Srgrimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 241541Srgrimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 251541Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 261541Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 271541Srgrimes * SUCH DAMAGE. 281541Srgrimes * 291541Srgrimes * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94 301541Srgrimes */ 311541Srgrimes 32116182Sobrien#include <sys/cdefs.h> 33116182Sobrien__FBSDID("$FreeBSD: head/sys/kern/uipc_mbuf.c 153232 2005-12-08 13:13:06Z andre $"); 34116182Sobrien 35101007Srwatson#include "opt_mac.h" 3677572Sobrien#include "opt_param.h" 37113490Ssilby#include "opt_mbuf_stress_test.h" 38101007Srwatson 391541Srgrimes#include <sys/param.h> 401541Srgrimes#include <sys/systm.h> 4176166Smarkm#include <sys/kernel.h> 42125296Ssilby#include <sys/limits.h> 4376166Smarkm#include <sys/lock.h> 44101173Srwatson#include <sys/mac.h> 4532036Sbde#include <sys/malloc.h> 461541Srgrimes#include <sys/mbuf.h> 4723081Swollman#include <sys/sysctl.h> 481541Srgrimes#include <sys/domain.h> 491541Srgrimes#include <sys/protosw.h> 50125296Ssilby#include <sys/uio.h> 5176166Smarkm 529759Sbdeint max_linkhdr; 539759Sbdeint max_protohdr; 549759Sbdeint max_hdr; 559759Sbdeint max_datalen; 56116455Ssilby#ifdef MBUF_STRESS_TEST 57112777Ssilbyint m_defragpackets; 58112777Ssilbyint m_defragbytes; 59112777Ssilbyint m_defraguseless; 60112777Ssilbyint m_defragfailure; 61113490Ssilbyint m_defragrandomfailures; 62113490Ssilby#endif 631541Srgrimes 6466475Sbmilekic/* 6566475Sbmilekic * sysctl(8) exported objects 6666475Sbmilekic */ 6744078SdfrSYSCTL_DECL(_kern_ipc); 6823081SwollmanSYSCTL_INT(_kern_ipc, KIPC_MAX_LINKHDR, max_linkhdr, CTLFLAG_RW, 6923081Swollman &max_linkhdr, 0, ""); 7023081SwollmanSYSCTL_INT(_kern_ipc, KIPC_MAX_PROTOHDR, max_protohdr, CTLFLAG_RW, 7123081Swollman &max_protohdr, 0, ""); 7223081SwollmanSYSCTL_INT(_kern_ipc, KIPC_MAX_HDR, max_hdr, CTLFLAG_RW, &max_hdr, 0, ""); 7323081SwollmanSYSCTL_INT(_kern_ipc, KIPC_MAX_DATALEN, max_datalen, CTLFLAG_RW, 7423081Swollman &max_datalen, 0, ""); 75116455Ssilby#ifdef MBUF_STRESS_TEST 76112777SsilbySYSCTL_INT(_kern_ipc, OID_AUTO, m_defragpackets, CTLFLAG_RD, 77112777Ssilby &m_defragpackets, 0, ""); 78112777SsilbySYSCTL_INT(_kern_ipc, OID_AUTO, m_defragbytes, CTLFLAG_RD, 79112777Ssilby &m_defragbytes, 0, ""); 80112777SsilbySYSCTL_INT(_kern_ipc, OID_AUTO, m_defraguseless, CTLFLAG_RD, 81112777Ssilby &m_defraguseless, 0, ""); 82112777SsilbySYSCTL_INT(_kern_ipc, OID_AUTO, m_defragfailure, CTLFLAG_RD, 83112777Ssilby &m_defragfailure, 0, ""); 84113490SsilbySYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW, 85113490Ssilby &m_defragrandomfailures, 0, ""); 86113490Ssilby#endif 8775112Sbmilekic 881541Srgrimes/* 89129906Sbmilekic * Allocate a given length worth of mbufs and/or clusters (whatever fits 90129906Sbmilekic * best) and return a pointer to the top of the allocated chain. If an 91129906Sbmilekic * existing mbuf chain is provided, then we will append the new chain 92129906Sbmilekic * to the existing one but still return the top of the newly allocated 93129906Sbmilekic * chain. 94129906Sbmilekic */ 95129906Sbmilekicstruct mbuf * 96129906Sbmilekicm_getm(struct mbuf *m, int len, int how, short type) 97129906Sbmilekic{ 98129906Sbmilekic struct mbuf *mb, *top, *cur, *mtail; 99129906Sbmilekic int num, rem; 100129906Sbmilekic int i; 101129906Sbmilekic 102129906Sbmilekic KASSERT(len >= 0, ("m_getm(): len is < 0")); 103129906Sbmilekic 104129906Sbmilekic /* If m != NULL, we will append to the end of that chain. */ 105129906Sbmilekic if (m != NULL) 106129906Sbmilekic for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next); 107129906Sbmilekic else 108129906Sbmilekic mtail = NULL; 109129906Sbmilekic 110129906Sbmilekic /* 111129906Sbmilekic * Calculate how many mbufs+clusters ("packets") we need and how much 112129906Sbmilekic * leftover there is after that and allocate the first mbuf+cluster 113129906Sbmilekic * if required. 114129906Sbmilekic */ 115129906Sbmilekic num = len / MCLBYTES; 116129906Sbmilekic rem = len % MCLBYTES; 117129906Sbmilekic top = cur = NULL; 118129906Sbmilekic if (num > 0) { 119129906Sbmilekic if ((top = cur = m_getcl(how, type, 0)) == NULL) 120129906Sbmilekic goto failed; 121130271Smux top->m_len = 0; 122129906Sbmilekic } 123129906Sbmilekic num--; 124129906Sbmilekic 125129906Sbmilekic for (i = 0; i < num; i++) { 126129906Sbmilekic mb = m_getcl(how, type, 0); 127129906Sbmilekic if (mb == NULL) 128129906Sbmilekic goto failed; 129129906Sbmilekic mb->m_len = 0; 130129906Sbmilekic cur = (cur->m_next = mb); 131129906Sbmilekic } 132129906Sbmilekic if (rem > 0) { 133129906Sbmilekic mb = (rem > MINCLSIZE) ? 134129906Sbmilekic m_getcl(how, type, 0) : m_get(how, type); 135129906Sbmilekic if (mb == NULL) 136129906Sbmilekic goto failed; 137129906Sbmilekic mb->m_len = 0; 138129906Sbmilekic if (cur == NULL) 139129906Sbmilekic top = mb; 140129906Sbmilekic else 141129906Sbmilekic cur->m_next = mb; 142129906Sbmilekic } 143129906Sbmilekic 144129906Sbmilekic if (mtail != NULL) 145129906Sbmilekic mtail->m_next = top; 146129906Sbmilekic return top; 147129906Sbmilekicfailed: 148129906Sbmilekic if (top != NULL) 149129906Sbmilekic m_freem(top); 150129906Sbmilekic return NULL; 151129906Sbmilekic} 152129906Sbmilekic 153129906Sbmilekic/* 154129906Sbmilekic * Free an entire chain of mbufs and associated external buffers, if 155129906Sbmilekic * applicable. 156129906Sbmilekic */ 157129906Sbmilekicvoid 158129906Sbmilekicm_freem(struct mbuf *mb) 159129906Sbmilekic{ 160129906Sbmilekic 161129906Sbmilekic while (mb != NULL) 162129906Sbmilekic mb = m_free(mb); 163129906Sbmilekic} 164129906Sbmilekic 165129906Sbmilekic/*- 166129906Sbmilekic * Configure a provided mbuf to refer to the provided external storage 167129906Sbmilekic * buffer and setup a reference count for said buffer. If the setting 168129906Sbmilekic * up of the reference count fails, the M_EXT bit will not be set. If 169129906Sbmilekic * successfull, the M_EXT bit is set in the mbuf's flags. 170129906Sbmilekic * 171129906Sbmilekic * Arguments: 172129906Sbmilekic * mb The existing mbuf to which to attach the provided buffer. 173129906Sbmilekic * buf The address of the provided external storage buffer. 174129906Sbmilekic * size The size of the provided buffer. 175129906Sbmilekic * freef A pointer to a routine that is responsible for freeing the 176129906Sbmilekic * provided external storage buffer. 177129906Sbmilekic * args A pointer to an argument structure (of any type) to be passed 178129906Sbmilekic * to the provided freef routine (may be NULL). 179129906Sbmilekic * flags Any other flags to be passed to the provided mbuf. 180129906Sbmilekic * type The type that the external storage buffer should be 181129906Sbmilekic * labeled with. 182129906Sbmilekic * 183129906Sbmilekic * Returns: 184129906Sbmilekic * Nothing. 185129906Sbmilekic */ 186129906Sbmilekicvoid 187129906Sbmilekicm_extadd(struct mbuf *mb, caddr_t buf, u_int size, 188129906Sbmilekic void (*freef)(void *, void *), void *args, int flags, int type) 189129906Sbmilekic{ 190151976Sandre KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__)); 191129906Sbmilekic 192151976Sandre if (type != EXT_EXTREF) 193151976Sandre mb->m_ext.ref_cnt = (u_int *)uma_zalloc(zone_ext_refcnt, M_NOWAIT); 194129906Sbmilekic if (mb->m_ext.ref_cnt != NULL) { 195129906Sbmilekic *(mb->m_ext.ref_cnt) = 1; 196129906Sbmilekic mb->m_flags |= (M_EXT | flags); 197129906Sbmilekic mb->m_ext.ext_buf = buf; 198129906Sbmilekic mb->m_data = mb->m_ext.ext_buf; 199129906Sbmilekic mb->m_ext.ext_size = size; 200129906Sbmilekic mb->m_ext.ext_free = freef; 201129906Sbmilekic mb->m_ext.ext_args = args; 202129906Sbmilekic mb->m_ext.ext_type = type; 203129906Sbmilekic } 204129906Sbmilekic} 205129906Sbmilekic 206129906Sbmilekic/* 207129906Sbmilekic * Non-directly-exported function to clean up after mbufs with M_EXT 208151976Sandre * storage attached to them if the reference count hits 1. 209129906Sbmilekic */ 210129906Sbmilekicvoid 211129906Sbmilekicmb_free_ext(struct mbuf *m) 212129906Sbmilekic{ 213151976Sandre KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__)); 214151976Sandre KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__)); 215129906Sbmilekic 216151976Sandre /* Free attached storage if this mbuf is the only reference to it. */ 217151976Sandre if (*(m->m_ext.ref_cnt) == 1 || 218152035Sandre atomic_fetchadd_int(m->m_ext.ref_cnt, -1) == 0) { 219151976Sandre switch (m->m_ext.ext_type) { 220152101Sandre case EXT_PACKET: /* The packet zone is special. */ 221152035Sandre if (*(m->m_ext.ref_cnt) == 0) 222152035Sandre *(m->m_ext.ref_cnt) = 1; 223151976Sandre uma_zfree(zone_pack, m); 224151976Sandre return; /* Job done. */ 225152101Sandre case EXT_CLUSTER: 226152101Sandre uma_zfree(zone_clust, m->m_ext.ext_buf); 227130289Sbmilekic break; 228153232Sandre case EXT_JUMBO4: 229153232Sandre uma_zfree(zone_jumbo4, m->m_ext.ext_buf); 230153232Sandre break; 231151976Sandre case EXT_JUMBO9: 232151976Sandre uma_zfree(zone_jumbo9, m->m_ext.ext_buf); 233151976Sandre break; 234151976Sandre case EXT_JUMBO16: 235151976Sandre uma_zfree(zone_jumbo16, m->m_ext.ext_buf); 236151976Sandre break; 237151976Sandre case EXT_SFBUF: 238151976Sandre case EXT_NET_DRV: 239151976Sandre case EXT_MOD_TYPE: 240151976Sandre case EXT_DISPOSABLE: 241151976Sandre *(m->m_ext.ref_cnt) = 0; 242151976Sandre uma_zfree(zone_ext_refcnt, __DEVOLATILE(u_int *, 243151976Sandre m->m_ext.ref_cnt)); 244151976Sandre /* FALLTHROUGH */ 245151976Sandre case EXT_EXTREF: 246151976Sandre KASSERT(m->m_ext.ext_free != NULL, 247151976Sandre ("%s: ext_free not set", __func__)); 248141668Sbmilekic (*(m->m_ext.ext_free))(m->m_ext.ext_buf, 249141668Sbmilekic m->m_ext.ext_args); 250151976Sandre break; 251151976Sandre default: 252151976Sandre KASSERT(m->m_ext.ext_type == 0, 253151976Sandre ("%s: unknown ext_type", __func__)); 254141668Sbmilekic } 255141668Sbmilekic } 256151976Sandre /* 257151976Sandre * Free this mbuf back to the mbuf zone with all m_ext 258151976Sandre * information purged. 259151976Sandre */ 260151976Sandre m->m_ext.ext_buf = NULL; 261151976Sandre m->m_ext.ext_free = NULL; 262151976Sandre m->m_ext.ext_args = NULL; 263151976Sandre m->m_ext.ref_cnt = NULL; 264151976Sandre m->m_ext.ext_size = 0; 265151976Sandre m->m_ext.ext_type = 0; 266151976Sandre m->m_flags &= ~M_EXT; 267130357Sbmilekic uma_zfree(zone_mbuf, m); 268129906Sbmilekic} 269129906Sbmilekic 270129906Sbmilekic/* 271151976Sandre * Attach the the cluster from *m to *n, set up m_ext in *n 272151976Sandre * and bump the refcount of the cluster. 273151976Sandre */ 274151976Sandrestatic void 275151976Sandremb_dupcl(struct mbuf *n, struct mbuf *m) 276151976Sandre{ 277151976Sandre KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__)); 278151976Sandre KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__)); 279151976Sandre KASSERT((n->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__)); 280151976Sandre 281151976Sandre if (*(m->m_ext.ref_cnt) == 1) 282151976Sandre *(m->m_ext.ref_cnt) += 1; 283151976Sandre else 284151976Sandre atomic_add_int(m->m_ext.ref_cnt, 1); 285151976Sandre n->m_ext.ext_buf = m->m_ext.ext_buf; 286151976Sandre n->m_ext.ext_free = m->m_ext.ext_free; 287151976Sandre n->m_ext.ext_args = m->m_ext.ext_args; 288151976Sandre n->m_ext.ext_size = m->m_ext.ext_size; 289151976Sandre n->m_ext.ref_cnt = m->m_ext.ref_cnt; 290151976Sandre n->m_ext.ext_type = m->m_ext.ext_type; 291151976Sandre n->m_flags |= M_EXT; 292151976Sandre} 293151976Sandre 294151976Sandre/* 295149598Sandre * Clean up mbuf (chain) from any tags and packet headers. 296149647Sandre * If "all" is set then the first mbuf in the chain will be 297149647Sandre * cleaned too. 298149598Sandre */ 299149598Sandrevoid 300149647Sandrem_demote(struct mbuf *m0, int all) 301149598Sandre{ 302149598Sandre struct mbuf *m; 303149598Sandre 304149598Sandre for (m = all ? m0 : m0->m_next; m != NULL; m = m->m_next) { 305149598Sandre if (m->m_flags & M_PKTHDR) { 306149598Sandre m_tag_delete_chain(m, NULL); 307149598Sandre m->m_flags &= ~M_PKTHDR; 308149598Sandre bzero(&m->m_pkthdr, sizeof(struct pkthdr)); 309149598Sandre } 310149643Sandre if (m->m_type == MT_HEADER) 311149598Sandre m->m_type = MT_DATA; 312149643Sandre if (m != m0 && m->m_nextpkt != NULL) 313149598Sandre m->m_nextpkt = NULL; 314149598Sandre m->m_flags = m->m_flags & (M_EXT|M_EOR|M_RDONLY|M_FREELIST); 315149598Sandre } 316149598Sandre} 317149598Sandre 318149598Sandre/* 319149648Sandre * Sanity checks on mbuf (chain) for use in KASSERT() and general 320149648Sandre * debugging. 321149648Sandre * Returns 0 or panics when bad and 1 on all tests passed. 322149648Sandre * Sanitize, 0 to run M_SANITY_ACTION, 1 to garble things so they 323149648Sandre * blow up later. 324149599Sandre */ 325149599Sandreint 326149599Sandrem_sanity(struct mbuf *m0, int sanitize) 327149599Sandre{ 328149599Sandre struct mbuf *m; 329149599Sandre caddr_t a, b; 330149599Sandre int pktlen = 0; 331149599Sandre 332149599Sandre#define M_SANITY_ACTION(s) return (0) 333149599Sandre/* #define M_SANITY_ACTION(s) panic("mbuf %p: " s, m) */ 334149599Sandre 335149648Sandre for (m = m0; m != NULL; m = m->m_next) { 336149599Sandre /* 337149599Sandre * Basic pointer checks. If any of these fails then some 338149599Sandre * unrelated kernel memory before or after us is trashed. 339149599Sandre * No way to recover from that. 340149599Sandre */ 341149648Sandre a = ((m->m_flags & M_EXT) ? m->m_ext.ext_buf : 342149648Sandre ((m->m_flags & M_PKTHDR) ? (caddr_t)(&m->m_pktdat) : 343149599Sandre (caddr_t)(&m->m_dat)) ); 344149599Sandre b = (caddr_t)(a + (m->m_flags & M_EXT ? m->m_ext.ext_size : 345149648Sandre ((m->m_flags & M_PKTHDR) ? MHLEN : MLEN))); 346149599Sandre if ((caddr_t)m->m_data < a) 347149599Sandre M_SANITY_ACTION("m_data outside mbuf data range left"); 348149599Sandre if ((caddr_t)m->m_data > b) 349149599Sandre M_SANITY_ACTION("m_data outside mbuf data range right"); 350149599Sandre if ((caddr_t)m->m_data + m->m_len > b) 351149599Sandre M_SANITY_ACTION("m_data + m_len exeeds mbuf space"); 352149648Sandre if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.header) { 353149599Sandre if ((caddr_t)m->m_pkthdr.header < a || 354149599Sandre (caddr_t)m->m_pkthdr.header > b) 355149599Sandre M_SANITY_ACTION("m_pkthdr.header outside mbuf data range"); 356149599Sandre } 357149599Sandre 358149599Sandre /* m->m_nextpkt may only be set on first mbuf in chain. */ 359149648Sandre if (m != m0 && m->m_nextpkt != NULL) { 360149599Sandre if (sanitize) { 361149599Sandre m_freem(m->m_nextpkt); 362149599Sandre m->m_nextpkt = (struct mbuf *)0xDEADC0DE; 363149599Sandre } else 364149599Sandre M_SANITY_ACTION("m->m_nextpkt on in-chain mbuf"); 365149599Sandre } 366149599Sandre 367149599Sandre /* correct type correlations. */ 368149599Sandre if (m->m_type == MT_HEADER && !(m->m_flags & M_PKTHDR)) { 369149599Sandre if (sanitize) 370149599Sandre m->m_type = MT_DATA; 371149599Sandre else 372149599Sandre M_SANITY_ACTION("MT_HEADER set but not M_PKTHDR"); 373149599Sandre } 374149599Sandre 375149599Sandre /* packet length (not mbuf length!) calculation */ 376149599Sandre if (m0->m_flags & M_PKTHDR) 377149599Sandre pktlen += m->m_len; 378149599Sandre 379149599Sandre /* m_tags may only be attached to first mbuf in chain. */ 380149599Sandre if (m != m0 && m->m_flags & M_PKTHDR && 381149599Sandre !SLIST_EMPTY(&m->m_pkthdr.tags)) { 382149599Sandre if (sanitize) { 383149599Sandre m_tag_delete_chain(m, NULL); 384149599Sandre /* put in 0xDEADC0DE perhaps? */ 385149648Sandre } else 386149599Sandre M_SANITY_ACTION("m_tags on in-chain mbuf"); 387149599Sandre } 388149599Sandre 389149599Sandre /* M_PKTHDR may only be set on first mbuf in chain */ 390149599Sandre if (m != m0 && m->m_flags & M_PKTHDR) { 391149599Sandre if (sanitize) { 392149599Sandre bzero(&m->m_pkthdr, sizeof(m->m_pkthdr)); 393149599Sandre m->m_flags &= ~M_PKTHDR; 394149599Sandre /* put in 0xDEADCODE and leave hdr flag in */ 395149599Sandre } else 396149599Sandre M_SANITY_ACTION("M_PKTHDR on in-chain mbuf"); 397149599Sandre } 398149599Sandre } 399149648Sandre m = m0; 400149648Sandre if (pktlen && pktlen != m->m_pkthdr.len) { 401149599Sandre if (sanitize) 402149648Sandre m->m_pkthdr.len = 0; 403149599Sandre else 404149599Sandre M_SANITY_ACTION("m_pkthdr.len != mbuf chain length"); 405149599Sandre } 406149648Sandre return 1; 407149648Sandre 408149599Sandre#undef M_SANITY_ACTION 409149599Sandre} 410149599Sandre 411149599Sandre 412149599Sandre/* 413108466Ssam * "Move" mbuf pkthdr from "from" to "to". 414100960Srwatson * "from" must have M_PKTHDR set, and "to" must be empty. 415100960Srwatson */ 416100960Srwatsonvoid 417108466Ssamm_move_pkthdr(struct mbuf *to, struct mbuf *from) 418100960Srwatson{ 419100960Srwatson 420100960Srwatson#if 0 421108466Ssam /* see below for why these are not enabled */ 422113255Sdes M_ASSERTPKTHDR(to); 423113487Srwatson /* Note: with MAC, this may not be a good assertion. */ 424108466Ssam KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), 425108466Ssam ("m_move_pkthdr: to has tags")); 426100960Srwatson#endif 427101007Srwatson#ifdef MAC 428113487Srwatson /* 429113487Srwatson * XXXMAC: It could be this should also occur for non-MAC? 430113487Srwatson */ 431101007Srwatson if (to->m_flags & M_PKTHDR) 432113487Srwatson m_tag_delete_chain(to, NULL); 433101007Srwatson#endif 434143302Ssam to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT); 435143302Ssam if ((to->m_flags & M_EXT) == 0) 436143302Ssam to->m_data = to->m_pktdat; 437108466Ssam to->m_pkthdr = from->m_pkthdr; /* especially tags */ 438108466Ssam SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */ 439108466Ssam from->m_flags &= ~M_PKTHDR; 440108466Ssam} 441108466Ssam 442108466Ssam/* 443108466Ssam * Duplicate "from"'s mbuf pkthdr in "to". 444108466Ssam * "from" must have M_PKTHDR set, and "to" must be empty. 445108466Ssam * In particular, this does a deep copy of the packet tags. 446108466Ssam */ 447108466Ssamint 448108466Ssamm_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how) 449108466Ssam{ 450108466Ssam 451108466Ssam#if 0 452108466Ssam /* 453108466Ssam * The mbuf allocator only initializes the pkthdr 454108466Ssam * when the mbuf is allocated with MGETHDR. Many users 455108466Ssam * (e.g. m_copy*, m_prepend) use MGET and then 456108466Ssam * smash the pkthdr as needed causing these 457108466Ssam * assertions to trip. For now just disable them. 458108466Ssam */ 459113255Sdes M_ASSERTPKTHDR(to); 460113487Srwatson /* Note: with MAC, this may not be a good assertion. */ 461108466Ssam KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), ("m_dup_pkthdr: to has tags")); 462108466Ssam#endif 463132488Salfred MBUF_CHECKSLEEP(how); 464108466Ssam#ifdef MAC 465108466Ssam if (to->m_flags & M_PKTHDR) 466113487Srwatson m_tag_delete_chain(to, NULL); 467108466Ssam#endif 468112733Ssilby to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT); 469112733Ssilby if ((to->m_flags & M_EXT) == 0) 470112733Ssilby to->m_data = to->m_pktdat; 471100960Srwatson to->m_pkthdr = from->m_pkthdr; 472108466Ssam SLIST_INIT(&to->m_pkthdr.tags); 473113480Srwatson return (m_tag_copy_chain(to, from, MBTOM(how))); 474100960Srwatson} 475100960Srwatson 476100960Srwatson/* 4771541Srgrimes * Lesser-used path for M_PREPEND: 4781541Srgrimes * allocate new mbuf to prepend to chain, 4791541Srgrimes * copy junk along. 4801541Srgrimes */ 4811541Srgrimesstruct mbuf * 48272356Sbmilekicm_prepend(struct mbuf *m, int len, int how) 4831541Srgrimes{ 4841541Srgrimes struct mbuf *mn; 4851541Srgrimes 486117770Ssilby if (m->m_flags & M_PKTHDR) 487117770Ssilby MGETHDR(mn, how, m->m_type); 488117770Ssilby else 489117770Ssilby MGET(mn, how, m->m_type); 49072356Sbmilekic if (mn == NULL) { 4911541Srgrimes m_freem(m); 49272356Sbmilekic return (NULL); 4931541Srgrimes } 494113487Srwatson if (m->m_flags & M_PKTHDR) 495108466Ssam M_MOVE_PKTHDR(mn, m); 4961541Srgrimes mn->m_next = m; 4971541Srgrimes m = mn; 4981541Srgrimes if (len < MHLEN) 4991541Srgrimes MH_ALIGN(m, len); 5001541Srgrimes m->m_len = len; 5011541Srgrimes return (m); 5021541Srgrimes} 5031541Srgrimes 5041541Srgrimes/* 5051541Srgrimes * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 5061541Srgrimes * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 507111119Simp * The wait parameter is a choice of M_TRYWAIT/M_DONTWAIT from caller. 50854002Sarchie * Note that the copy is read-only, because clusters are not copied, 50954002Sarchie * only their reference counts are incremented. 5101541Srgrimes */ 5111541Srgrimesstruct mbuf * 51272356Sbmilekicm_copym(struct mbuf *m, int off0, int len, int wait) 5131541Srgrimes{ 51472356Sbmilekic struct mbuf *n, **np; 51572356Sbmilekic int off = off0; 5161541Srgrimes struct mbuf *top; 5171541Srgrimes int copyhdr = 0; 5181541Srgrimes 51952201Salfred KASSERT(off >= 0, ("m_copym, negative off %d", off)); 52052201Salfred KASSERT(len >= 0, ("m_copym, negative len %d", len)); 521132488Salfred MBUF_CHECKSLEEP(wait); 5221541Srgrimes if (off == 0 && m->m_flags & M_PKTHDR) 5231541Srgrimes copyhdr = 1; 5241541Srgrimes while (off > 0) { 52552201Salfred KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain")); 5261541Srgrimes if (off < m->m_len) 5271541Srgrimes break; 5281541Srgrimes off -= m->m_len; 5291541Srgrimes m = m->m_next; 5301541Srgrimes } 5311541Srgrimes np = ⊤ 5321541Srgrimes top = 0; 5331541Srgrimes while (len > 0) { 53472356Sbmilekic if (m == NULL) { 53552201Salfred KASSERT(len == M_COPYALL, 53652201Salfred ("m_copym, length > size of mbuf chain")); 5371541Srgrimes break; 5381541Srgrimes } 539117770Ssilby if (copyhdr) 540117770Ssilby MGETHDR(n, wait, m->m_type); 541117770Ssilby else 542117770Ssilby MGET(n, wait, m->m_type); 5431541Srgrimes *np = n; 54472356Sbmilekic if (n == NULL) 5451541Srgrimes goto nospace; 5461541Srgrimes if (copyhdr) { 547108466Ssam if (!m_dup_pkthdr(n, m, wait)) 548108466Ssam goto nospace; 5491541Srgrimes if (len == M_COPYALL) 5501541Srgrimes n->m_pkthdr.len -= off0; 5511541Srgrimes else 5521541Srgrimes n->m_pkthdr.len = len; 5531541Srgrimes copyhdr = 0; 5541541Srgrimes } 5551541Srgrimes n->m_len = min(len, m->m_len - off); 5561541Srgrimes if (m->m_flags & M_EXT) { 5571541Srgrimes n->m_data = m->m_data + off; 558151976Sandre mb_dupcl(n, m); 5591541Srgrimes } else 5601541Srgrimes bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), 561103569Sbmilekic (u_int)n->m_len); 5621541Srgrimes if (len != M_COPYALL) 5631541Srgrimes len -= n->m_len; 5641541Srgrimes off = 0; 5651541Srgrimes m = m->m_next; 5661541Srgrimes np = &n->m_next; 5671541Srgrimes } 56878592Sbmilekic if (top == NULL) 56978592Sbmilekic mbstat.m_mcfail++; /* XXX: No consistency. */ 57078592Sbmilekic 5711541Srgrimes return (top); 5721541Srgrimesnospace: 5731541Srgrimes m_freem(top); 57478592Sbmilekic mbstat.m_mcfail++; /* XXX: No consistency. */ 57572356Sbmilekic return (NULL); 5761541Srgrimes} 5771541Srgrimes 5781541Srgrimes/* 579149602Sandre * Returns mbuf chain with new head for the prepending case. 580149602Sandre * Copies from mbuf (chain) n from off for len to mbuf (chain) m 581149602Sandre * either prepending or appending the data. 582149602Sandre * The resulting mbuf (chain) m is fully writeable. 583149602Sandre * m is destination (is made writeable) 584149602Sandre * n is source, off is offset in source, len is len from offset 585149602Sandre * dir, 0 append, 1 prepend 586149602Sandre * how, wait or nowait 587149602Sandre */ 588149602Sandre 589149602Sandrestatic int 590149602Sandrem_bcopyxxx(void *s, void *t, u_int len) 591149602Sandre{ 592149602Sandre bcopy(s, t, (size_t)len); 593149602Sandre return 0; 594149602Sandre} 595149602Sandre 596149602Sandrestruct mbuf * 597149602Sandrem_copymdata(struct mbuf *m, struct mbuf *n, int off, int len, 598149602Sandre int prep, int how) 599149602Sandre{ 600151976Sandre struct mbuf *mm, *x, *z, *prev = NULL; 601149602Sandre caddr_t p; 602151976Sandre int i, nlen = 0; 603149602Sandre caddr_t buf[MLEN]; 604149602Sandre 605149602Sandre KASSERT(m != NULL && n != NULL, ("m_copymdata, no target or source")); 606149602Sandre KASSERT(off >= 0, ("m_copymdata, negative off %d", off)); 607149602Sandre KASSERT(len >= 0, ("m_copymdata, negative len %d", len)); 608149602Sandre KASSERT(prep == 0 || prep == 1, ("m_copymdata, unknown direction %d", prep)); 609149602Sandre 610151976Sandre mm = m; 611151976Sandre if (!prep) { 612151976Sandre while(mm->m_next) { 613151976Sandre prev = mm; 614151976Sandre mm = mm->m_next; 615149602Sandre } 616149602Sandre } 617149602Sandre for (z = n; z != NULL; z = z->m_next) 618149602Sandre nlen += z->m_len; 619149602Sandre if (len == M_COPYALL) 620149602Sandre len = nlen - off; 621149602Sandre if (off + len > nlen || len < 1) 622149602Sandre return NULL; 623149602Sandre 624151976Sandre if (!M_WRITABLE(mm)) { 625151976Sandre /* XXX: Use proper m_xxx function instead. */ 626151976Sandre x = m_getcl(how, MT_DATA, mm->m_flags); 627151976Sandre if (x == NULL) 628151976Sandre return NULL; 629151976Sandre bcopy(mm->m_ext.ext_buf, x->m_ext.ext_buf, x->m_ext.ext_size); 630151976Sandre p = x->m_ext.ext_buf + (mm->m_data - mm->m_ext.ext_buf); 631151976Sandre x->m_data = p; 632151976Sandre mm->m_next = NULL; 633151976Sandre if (mm != m) 634151976Sandre prev->m_next = x; 635151976Sandre m_free(mm); 636151976Sandre mm = x; 637151976Sandre } 638151976Sandre 639149602Sandre /* 640149602Sandre * Append/prepend the data. Allocating mbufs as necessary. 641149602Sandre */ 642149602Sandre /* Shortcut if enough free space in first/last mbuf. */ 643149602Sandre if (!prep && M_TRAILINGSPACE(mm) >= len) { 644149602Sandre m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t) + 645149602Sandre mm->m_len); 646149602Sandre mm->m_len += len; 647149602Sandre mm->m_pkthdr.len += len; 648149602Sandre return m; 649149602Sandre } 650149602Sandre if (prep && M_LEADINGSPACE(mm) >= len) { 651149602Sandre mm->m_data = mtod(mm, caddr_t) - len; 652149602Sandre m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t)); 653149602Sandre mm->m_len += len; 654149602Sandre mm->m_pkthdr.len += len; 655149602Sandre return mm; 656149602Sandre } 657149602Sandre 658149602Sandre /* Expand first/last mbuf to cluster if possible. */ 659149602Sandre if (!prep && !(mm->m_flags & M_EXT) && len > M_TRAILINGSPACE(mm)) { 660149602Sandre bcopy(mm->m_data, &buf, mm->m_len); 661149602Sandre m_clget(mm, how); 662149602Sandre if (!(mm->m_flags & M_EXT)) 663149602Sandre return NULL; 664149602Sandre bcopy(&buf, mm->m_ext.ext_buf, mm->m_len); 665149602Sandre mm->m_data = mm->m_ext.ext_buf; 666149602Sandre mm->m_pkthdr.header = NULL; 667149602Sandre } 668149602Sandre if (prep && !(mm->m_flags & M_EXT) && len > M_LEADINGSPACE(mm)) { 669149602Sandre bcopy(mm->m_data, &buf, mm->m_len); 670149602Sandre m_clget(mm, how); 671149602Sandre if (!(mm->m_flags & M_EXT)) 672149602Sandre return NULL; 673149602Sandre bcopy(&buf, (caddr_t *)mm->m_ext.ext_buf + 674149602Sandre mm->m_ext.ext_size - mm->m_len, mm->m_len); 675149602Sandre mm->m_data = (caddr_t)mm->m_ext.ext_buf + 676149602Sandre mm->m_ext.ext_size - mm->m_len; 677149602Sandre mm->m_pkthdr.header = NULL; 678149602Sandre } 679149602Sandre 680149602Sandre /* Append/prepend as many mbuf (clusters) as necessary to fit len. */ 681149602Sandre if (!prep && len > M_TRAILINGSPACE(mm)) { 682149602Sandre if (!m_getm(mm, len - M_TRAILINGSPACE(mm), how, MT_DATA)) 683149602Sandre return NULL; 684149602Sandre } 685149602Sandre if (prep && len > M_LEADINGSPACE(mm)) { 686149602Sandre if (!(z = m_getm(NULL, len - M_LEADINGSPACE(mm), how, MT_DATA))) 687149602Sandre return NULL; 688149602Sandre i = 0; 689149602Sandre for (x = z; x != NULL; x = x->m_next) { 690149602Sandre i += x->m_flags & M_EXT ? x->m_ext.ext_size : 691149602Sandre (x->m_flags & M_PKTHDR ? MHLEN : MLEN); 692149602Sandre if (!x->m_next) 693149602Sandre break; 694149602Sandre } 695149602Sandre z->m_data += i - len; 696149602Sandre m_move_pkthdr(mm, z); 697149602Sandre x->m_next = mm; 698149602Sandre mm = z; 699149602Sandre } 700149602Sandre 701149602Sandre /* Seek to start position in source mbuf. Optimization for long chains. */ 702149602Sandre while (off > 0) { 703149602Sandre if (off < n->m_len) 704149602Sandre break; 705149602Sandre off -= n->m_len; 706149602Sandre n = n->m_next; 707149602Sandre } 708149602Sandre 709149602Sandre /* Copy data into target mbuf. */ 710149602Sandre z = mm; 711149602Sandre while (len > 0) { 712149602Sandre KASSERT(z != NULL, ("m_copymdata, falling off target edge")); 713149602Sandre i = M_TRAILINGSPACE(z); 714149602Sandre m_apply(n, off, i, m_bcopyxxx, mtod(z, caddr_t) + z->m_len); 715149602Sandre z->m_len += i; 716149602Sandre /* fixup pkthdr.len if necessary */ 717149602Sandre if ((prep ? mm : m)->m_flags & M_PKTHDR) 718149602Sandre (prep ? mm : m)->m_pkthdr.len += i; 719149602Sandre off += i; 720149602Sandre len -= i; 721149602Sandre z = z->m_next; 722149602Sandre } 723149602Sandre return (prep ? mm : m); 724149602Sandre} 725149602Sandre 726149602Sandre/* 72715689Swollman * Copy an entire packet, including header (which must be present). 72815689Swollman * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'. 72954002Sarchie * Note that the copy is read-only, because clusters are not copied, 73054002Sarchie * only their reference counts are incremented. 73172750Sluigi * Preserve alignment of the first mbuf so if the creator has left 73272750Sluigi * some room at the beginning (e.g. for inserting protocol headers) 73372750Sluigi * the copies still have the room available. 73415689Swollman */ 73515689Swollmanstruct mbuf * 73672356Sbmilekicm_copypacket(struct mbuf *m, int how) 73715689Swollman{ 73815689Swollman struct mbuf *top, *n, *o; 73915689Swollman 740132488Salfred MBUF_CHECKSLEEP(how); 74115689Swollman MGET(n, how, m->m_type); 74215689Swollman top = n; 74372356Sbmilekic if (n == NULL) 74415689Swollman goto nospace; 74515689Swollman 746108466Ssam if (!m_dup_pkthdr(n, m, how)) 747108466Ssam goto nospace; 74815689Swollman n->m_len = m->m_len; 74915689Swollman if (m->m_flags & M_EXT) { 75015689Swollman n->m_data = m->m_data; 751151976Sandre mb_dupcl(n, m); 75215689Swollman } else { 75372750Sluigi n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat ); 75415689Swollman bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 75515689Swollman } 75615689Swollman 75715689Swollman m = m->m_next; 75815689Swollman while (m) { 75915689Swollman MGET(o, how, m->m_type); 76072356Sbmilekic if (o == NULL) 76115689Swollman goto nospace; 76215689Swollman 76315689Swollman n->m_next = o; 76415689Swollman n = n->m_next; 76515689Swollman 76615689Swollman n->m_len = m->m_len; 76715689Swollman if (m->m_flags & M_EXT) { 76815689Swollman n->m_data = m->m_data; 769151976Sandre mb_dupcl(n, m); 77015689Swollman } else { 77115689Swollman bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 77215689Swollman } 77315689Swollman 77415689Swollman m = m->m_next; 77515689Swollman } 77615689Swollman return top; 77715689Swollmannospace: 77815689Swollman m_freem(top); 77978592Sbmilekic mbstat.m_mcfail++; /* XXX: No consistency. */ 78072356Sbmilekic return (NULL); 78115689Swollman} 78215689Swollman 78315689Swollman/* 7841541Srgrimes * Copy data from an mbuf chain starting "off" bytes from the beginning, 7851541Srgrimes * continuing for "len" bytes, into the indicated buffer. 7861541Srgrimes */ 7871549Srgrimesvoid 78881907Sjulianm_copydata(const struct mbuf *m, int off, int len, caddr_t cp) 7891541Srgrimes{ 790103569Sbmilekic u_int count; 7911541Srgrimes 79252201Salfred KASSERT(off >= 0, ("m_copydata, negative off %d", off)); 79352201Salfred KASSERT(len >= 0, ("m_copydata, negative len %d", len)); 7941541Srgrimes while (off > 0) { 79552201Salfred KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain")); 7961541Srgrimes if (off < m->m_len) 7971541Srgrimes break; 7981541Srgrimes off -= m->m_len; 7991541Srgrimes m = m->m_next; 8001541Srgrimes } 8011541Srgrimes while (len > 0) { 80252201Salfred KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain")); 8031541Srgrimes count = min(m->m_len - off, len); 8041541Srgrimes bcopy(mtod(m, caddr_t) + off, cp, count); 8051541Srgrimes len -= count; 8061541Srgrimes cp += count; 8071541Srgrimes off = 0; 8081541Srgrimes m = m->m_next; 8091541Srgrimes } 8101541Srgrimes} 8111541Srgrimes 8121541Srgrimes/* 81354002Sarchie * Copy a packet header mbuf chain into a completely new chain, including 81454002Sarchie * copying any mbuf clusters. Use this instead of m_copypacket() when 81554002Sarchie * you need a writable copy of an mbuf chain. 81654002Sarchie */ 81754002Sarchiestruct mbuf * 81872356Sbmilekicm_dup(struct mbuf *m, int how) 81954002Sarchie{ 82054002Sarchie struct mbuf **p, *top = NULL; 82154002Sarchie int remain, moff, nsize; 82254002Sarchie 823132488Salfred MBUF_CHECKSLEEP(how); 82454002Sarchie /* Sanity check */ 82554002Sarchie if (m == NULL) 82672356Sbmilekic return (NULL); 827113255Sdes M_ASSERTPKTHDR(m); 82854002Sarchie 82954002Sarchie /* While there's more data, get a new mbuf, tack it on, and fill it */ 83054002Sarchie remain = m->m_pkthdr.len; 83154002Sarchie moff = 0; 83254002Sarchie p = ⊤ 83354002Sarchie while (remain > 0 || top == NULL) { /* allow m->m_pkthdr.len == 0 */ 83454002Sarchie struct mbuf *n; 83554002Sarchie 83654002Sarchie /* Get the next new mbuf */ 837129906Sbmilekic if (remain >= MINCLSIZE) { 838129906Sbmilekic n = m_getcl(how, m->m_type, 0); 839129906Sbmilekic nsize = MCLBYTES; 840129906Sbmilekic } else { 841129906Sbmilekic n = m_get(how, m->m_type); 842129906Sbmilekic nsize = MLEN; 843129906Sbmilekic } 84454002Sarchie if (n == NULL) 84554002Sarchie goto nospace; 846129906Sbmilekic 847129906Sbmilekic if (top == NULL) { /* First one, must be PKTHDR */ 848129906Sbmilekic if (!m_dup_pkthdr(n, m, how)) { 849129906Sbmilekic m_free(n); 850108466Ssam goto nospace; 851129906Sbmilekic } 85254002Sarchie nsize = MHLEN; 85354002Sarchie } 85454002Sarchie n->m_len = 0; 85554002Sarchie 85654002Sarchie /* Link it into the new chain */ 85754002Sarchie *p = n; 85854002Sarchie p = &n->m_next; 85954002Sarchie 86054002Sarchie /* Copy data from original mbuf(s) into new mbuf */ 86154002Sarchie while (n->m_len < nsize && m != NULL) { 86254002Sarchie int chunk = min(nsize - n->m_len, m->m_len - moff); 86354002Sarchie 86454002Sarchie bcopy(m->m_data + moff, n->m_data + n->m_len, chunk); 86554002Sarchie moff += chunk; 86654002Sarchie n->m_len += chunk; 86754002Sarchie remain -= chunk; 86854002Sarchie if (moff == m->m_len) { 86954002Sarchie m = m->m_next; 87054002Sarchie moff = 0; 87154002Sarchie } 87254002Sarchie } 87354002Sarchie 87454002Sarchie /* Check correct total mbuf length */ 87554002Sarchie KASSERT((remain > 0 && m != NULL) || (remain == 0 && m == NULL), 87687594Sobrien ("%s: bogus m_pkthdr.len", __func__)); 87754002Sarchie } 87854002Sarchie return (top); 87954002Sarchie 88054002Sarchienospace: 88154002Sarchie m_freem(top); 88278592Sbmilekic mbstat.m_mcfail++; /* XXX: No consistency. */ 88372356Sbmilekic return (NULL); 88454002Sarchie} 88554002Sarchie 88654002Sarchie/* 8871541Srgrimes * Concatenate mbuf chain n to m. 8881541Srgrimes * Both chains must be of the same type (e.g. MT_DATA). 8891541Srgrimes * Any m_pkthdr is not updated. 8901541Srgrimes */ 8911549Srgrimesvoid 89272356Sbmilekicm_cat(struct mbuf *m, struct mbuf *n) 8931541Srgrimes{ 8941541Srgrimes while (m->m_next) 8951541Srgrimes m = m->m_next; 8961541Srgrimes while (n) { 8971541Srgrimes if (m->m_flags & M_EXT || 8981541Srgrimes m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { 8991541Srgrimes /* just join the two chains */ 9001541Srgrimes m->m_next = n; 9011541Srgrimes return; 9021541Srgrimes } 9031541Srgrimes /* splat the data from one into the other */ 9041541Srgrimes bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 9051541Srgrimes (u_int)n->m_len); 9061541Srgrimes m->m_len += n->m_len; 9071541Srgrimes n = m_free(n); 9081541Srgrimes } 9091541Srgrimes} 9101541Srgrimes 9111549Srgrimesvoid 91272356Sbmilekicm_adj(struct mbuf *mp, int req_len) 9131541Srgrimes{ 91472356Sbmilekic int len = req_len; 91572356Sbmilekic struct mbuf *m; 91672356Sbmilekic int count; 9171541Srgrimes 9181541Srgrimes if ((m = mp) == NULL) 9191541Srgrimes return; 9201541Srgrimes if (len >= 0) { 9211541Srgrimes /* 9221541Srgrimes * Trim from head. 9231541Srgrimes */ 9241541Srgrimes while (m != NULL && len > 0) { 9251541Srgrimes if (m->m_len <= len) { 9261541Srgrimes len -= m->m_len; 9271541Srgrimes m->m_len = 0; 9281541Srgrimes m = m->m_next; 9291541Srgrimes } else { 9301541Srgrimes m->m_len -= len; 9311541Srgrimes m->m_data += len; 9321541Srgrimes len = 0; 9331541Srgrimes } 9341541Srgrimes } 9351541Srgrimes m = mp; 9361541Srgrimes if (mp->m_flags & M_PKTHDR) 9371541Srgrimes m->m_pkthdr.len -= (req_len - len); 9381541Srgrimes } else { 9391541Srgrimes /* 9401541Srgrimes * Trim from tail. Scan the mbuf chain, 9411541Srgrimes * calculating its length and finding the last mbuf. 9421541Srgrimes * If the adjustment only affects this mbuf, then just 9431541Srgrimes * adjust and return. Otherwise, rescan and truncate 9441541Srgrimes * after the remaining size. 9451541Srgrimes */ 9461541Srgrimes len = -len; 9471541Srgrimes count = 0; 9481541Srgrimes for (;;) { 9491541Srgrimes count += m->m_len; 9501541Srgrimes if (m->m_next == (struct mbuf *)0) 9511541Srgrimes break; 9521541Srgrimes m = m->m_next; 9531541Srgrimes } 9541541Srgrimes if (m->m_len >= len) { 9551541Srgrimes m->m_len -= len; 9561541Srgrimes if (mp->m_flags & M_PKTHDR) 9571541Srgrimes mp->m_pkthdr.len -= len; 9581541Srgrimes return; 9591541Srgrimes } 9601541Srgrimes count -= len; 9611541Srgrimes if (count < 0) 9621541Srgrimes count = 0; 9631541Srgrimes /* 9641541Srgrimes * Correct length for chain is "count". 9651541Srgrimes * Find the mbuf with last data, adjust its length, 9661541Srgrimes * and toss data from remaining mbufs on chain. 9671541Srgrimes */ 9681541Srgrimes m = mp; 9691541Srgrimes if (m->m_flags & M_PKTHDR) 9701541Srgrimes m->m_pkthdr.len = count; 9711541Srgrimes for (; m; m = m->m_next) { 9721541Srgrimes if (m->m_len >= count) { 9731541Srgrimes m->m_len = count; 974142350Ssam if (m->m_next != NULL) { 975142350Ssam m_freem(m->m_next); 976142350Ssam m->m_next = NULL; 977142350Ssam } 9781541Srgrimes break; 9791541Srgrimes } 9801541Srgrimes count -= m->m_len; 9811541Srgrimes } 9821541Srgrimes } 9831541Srgrimes} 9841541Srgrimes 9851541Srgrimes/* 9861541Srgrimes * Rearange an mbuf chain so that len bytes are contiguous 9871541Srgrimes * and in the data area of an mbuf (so that mtod and dtom 9881541Srgrimes * will work for a structure of size len). Returns the resulting 9891541Srgrimes * mbuf chain on success, frees it and returns null on failure. 9901541Srgrimes * If there is room, it will add up to max_protohdr-len extra bytes to the 9911541Srgrimes * contiguous region in an attempt to avoid being called next time. 9921541Srgrimes */ 9931541Srgrimesstruct mbuf * 99472356Sbmilekicm_pullup(struct mbuf *n, int len) 9951541Srgrimes{ 99672356Sbmilekic struct mbuf *m; 99772356Sbmilekic int count; 9981541Srgrimes int space; 9991541Srgrimes 10001541Srgrimes /* 10011541Srgrimes * If first mbuf has no cluster, and has room for len bytes 10021541Srgrimes * without shifting current data, pullup into it, 10031541Srgrimes * otherwise allocate a new mbuf to prepend to the chain. 10041541Srgrimes */ 10051541Srgrimes if ((n->m_flags & M_EXT) == 0 && 10061541Srgrimes n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 10071541Srgrimes if (n->m_len >= len) 10081541Srgrimes return (n); 10091541Srgrimes m = n; 10101541Srgrimes n = n->m_next; 10111541Srgrimes len -= m->m_len; 10121541Srgrimes } else { 10131541Srgrimes if (len > MHLEN) 10141541Srgrimes goto bad; 1015111119Simp MGET(m, M_DONTWAIT, n->m_type); 101672356Sbmilekic if (m == NULL) 10171541Srgrimes goto bad; 10181541Srgrimes m->m_len = 0; 1019108466Ssam if (n->m_flags & M_PKTHDR) 1020108466Ssam M_MOVE_PKTHDR(m, n); 10211541Srgrimes } 10221541Srgrimes space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 10231541Srgrimes do { 10241541Srgrimes count = min(min(max(len, max_protohdr), space), n->m_len); 10251541Srgrimes bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 1026103569Sbmilekic (u_int)count); 10271541Srgrimes len -= count; 10281541Srgrimes m->m_len += count; 10291541Srgrimes n->m_len -= count; 10301541Srgrimes space -= count; 10311541Srgrimes if (n->m_len) 10321541Srgrimes n->m_data += count; 10331541Srgrimes else 10341541Srgrimes n = m_free(n); 10351541Srgrimes } while (len > 0 && n); 10361541Srgrimes if (len > 0) { 10371541Srgrimes (void) m_free(m); 10381541Srgrimes goto bad; 10391541Srgrimes } 10401541Srgrimes m->m_next = n; 10411541Srgrimes return (m); 10421541Srgrimesbad: 10431541Srgrimes m_freem(n); 104478592Sbmilekic mbstat.m_mpfail++; /* XXX: No consistency. */ 104572356Sbmilekic return (NULL); 10461541Srgrimes} 10471541Srgrimes 10481541Srgrimes/* 1049143761Sjmg * Like m_pullup(), except a new mbuf is always allocated, and we allow 1050143761Sjmg * the amount of empty space before the data in the new mbuf to be specified 1051143761Sjmg * (in the event that the caller expects to prepend later). 1052143761Sjmg */ 1053143761Sjmgint MSFail; 1054143761Sjmg 1055143761Sjmgstruct mbuf * 1056143761Sjmgm_copyup(struct mbuf *n, int len, int dstoff) 1057143761Sjmg{ 1058143761Sjmg struct mbuf *m; 1059143761Sjmg int count, space; 1060143761Sjmg 1061143761Sjmg if (len > (MHLEN - dstoff)) 1062143761Sjmg goto bad; 1063143761Sjmg MGET(m, M_DONTWAIT, n->m_type); 1064143761Sjmg if (m == NULL) 1065143761Sjmg goto bad; 1066143761Sjmg m->m_len = 0; 1067143761Sjmg if (n->m_flags & M_PKTHDR) 1068143761Sjmg M_MOVE_PKTHDR(m, n); 1069143761Sjmg m->m_data += dstoff; 1070143761Sjmg space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 1071143761Sjmg do { 1072143761Sjmg count = min(min(max(len, max_protohdr), space), n->m_len); 1073143761Sjmg memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), 1074143761Sjmg (unsigned)count); 1075143761Sjmg len -= count; 1076143761Sjmg m->m_len += count; 1077143761Sjmg n->m_len -= count; 1078143761Sjmg space -= count; 1079143761Sjmg if (n->m_len) 1080143761Sjmg n->m_data += count; 1081143761Sjmg else 1082143761Sjmg n = m_free(n); 1083143761Sjmg } while (len > 0 && n); 1084143761Sjmg if (len > 0) { 1085143761Sjmg (void) m_free(m); 1086143761Sjmg goto bad; 1087143761Sjmg } 1088143761Sjmg m->m_next = n; 1089143761Sjmg return (m); 1090143761Sjmg bad: 1091143761Sjmg m_freem(n); 1092143761Sjmg MSFail++; 1093143761Sjmg return (NULL); 1094143761Sjmg} 1095143761Sjmg 1096143761Sjmg/* 10971541Srgrimes * Partition an mbuf chain in two pieces, returning the tail -- 10981541Srgrimes * all but the first len0 bytes. In case of failure, it returns NULL and 10991541Srgrimes * attempts to restore the chain to its original state. 110097681Sarchie * 110197681Sarchie * Note that the resulting mbufs might be read-only, because the new 110297681Sarchie * mbuf can end up sharing an mbuf cluster with the original mbuf if 110397681Sarchie * the "breaking point" happens to lie within a cluster mbuf. Use the 110497681Sarchie * M_WRITABLE() macro to check for this case. 11051541Srgrimes */ 11061541Srgrimesstruct mbuf * 110772356Sbmilekicm_split(struct mbuf *m0, int len0, int wait) 11081541Srgrimes{ 110972356Sbmilekic struct mbuf *m, *n; 1110103569Sbmilekic u_int len = len0, remain; 11111541Srgrimes 1112132488Salfred MBUF_CHECKSLEEP(wait); 11131541Srgrimes for (m = m0; m && len > m->m_len; m = m->m_next) 11141541Srgrimes len -= m->m_len; 111572356Sbmilekic if (m == NULL) 111672356Sbmilekic return (NULL); 11171541Srgrimes remain = m->m_len - len; 11181541Srgrimes if (m0->m_flags & M_PKTHDR) { 11191541Srgrimes MGETHDR(n, wait, m0->m_type); 112072356Sbmilekic if (n == NULL) 112172356Sbmilekic return (NULL); 11221541Srgrimes n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 11231541Srgrimes n->m_pkthdr.len = m0->m_pkthdr.len - len0; 11241541Srgrimes m0->m_pkthdr.len = len0; 11251541Srgrimes if (m->m_flags & M_EXT) 11261541Srgrimes goto extpacket; 11271541Srgrimes if (remain > MHLEN) { 11281541Srgrimes /* m can't be the lead packet */ 11291541Srgrimes MH_ALIGN(n, 0); 11301541Srgrimes n->m_next = m_split(m, len, wait); 113172356Sbmilekic if (n->m_next == NULL) { 11321541Srgrimes (void) m_free(n); 113372356Sbmilekic return (NULL); 113494471Shsu } else { 113594471Shsu n->m_len = 0; 11361541Srgrimes return (n); 113794471Shsu } 11381541Srgrimes } else 11391541Srgrimes MH_ALIGN(n, remain); 11401541Srgrimes } else if (remain == 0) { 11411541Srgrimes n = m->m_next; 114272356Sbmilekic m->m_next = NULL; 11431541Srgrimes return (n); 11441541Srgrimes } else { 11451541Srgrimes MGET(n, wait, m->m_type); 114672356Sbmilekic if (n == NULL) 114772356Sbmilekic return (NULL); 11481541Srgrimes M_ALIGN(n, remain); 11491541Srgrimes } 11501541Srgrimesextpacket: 11511541Srgrimes if (m->m_flags & M_EXT) { 11521541Srgrimes n->m_data = m->m_data + len; 1153151976Sandre mb_dupcl(n, m); 11541541Srgrimes } else { 11551541Srgrimes bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); 11561541Srgrimes } 11571541Srgrimes n->m_len = remain; 11581541Srgrimes m->m_len = len; 11591541Srgrimes n->m_next = m->m_next; 116072356Sbmilekic m->m_next = NULL; 11611541Srgrimes return (n); 11621541Srgrimes} 11631541Srgrimes/* 11641541Srgrimes * Routine to copy from device local memory into mbufs. 116578508Sbmilekic * Note that `off' argument is offset into first mbuf of target chain from 116678508Sbmilekic * which to begin copying the data to. 11671541Srgrimes */ 11681541Srgrimesstruct mbuf * 116978508Sbmilekicm_devget(char *buf, int totlen, int off, struct ifnet *ifp, 117072356Sbmilekic void (*copy)(char *from, caddr_t to, u_int len)) 11711541Srgrimes{ 117272356Sbmilekic struct mbuf *m; 1173129906Sbmilekic struct mbuf *top = NULL, **mp = ⊤ 117478508Sbmilekic int len; 11751541Srgrimes 117678508Sbmilekic if (off < 0 || off > MHLEN) 117778508Sbmilekic return (NULL); 117878508Sbmilekic 1179129906Sbmilekic while (totlen > 0) { 1180129906Sbmilekic if (top == NULL) { /* First one, must be PKTHDR */ 1181129906Sbmilekic if (totlen + off >= MINCLSIZE) { 1182129906Sbmilekic m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); 1183129906Sbmilekic len = MCLBYTES; 1184129906Sbmilekic } else { 1185129906Sbmilekic m = m_gethdr(M_DONTWAIT, MT_DATA); 1186129906Sbmilekic len = MHLEN; 11871541Srgrimes 1188129906Sbmilekic /* Place initial small packet/header at end of mbuf */ 1189129906Sbmilekic if (m && totlen + off + max_linkhdr <= MLEN) { 1190129906Sbmilekic m->m_data += max_linkhdr; 1191129906Sbmilekic len -= max_linkhdr; 1192129906Sbmilekic } 1193129906Sbmilekic } 1194129906Sbmilekic if (m == NULL) 1195129906Sbmilekic return NULL; 1196129906Sbmilekic m->m_pkthdr.rcvif = ifp; 1197129906Sbmilekic m->m_pkthdr.len = totlen; 1198129906Sbmilekic } else { 1199129906Sbmilekic if (totlen + off >= MINCLSIZE) { 1200129906Sbmilekic m = m_getcl(M_DONTWAIT, MT_DATA, 0); 1201129906Sbmilekic len = MCLBYTES; 1202129906Sbmilekic } else { 1203129906Sbmilekic m = m_get(M_DONTWAIT, MT_DATA); 1204129906Sbmilekic len = MLEN; 1205129906Sbmilekic } 120672356Sbmilekic if (m == NULL) { 12071541Srgrimes m_freem(top); 1208129906Sbmilekic return NULL; 12091541Srgrimes } 12101541Srgrimes } 121178508Sbmilekic if (off) { 121278508Sbmilekic m->m_data += off; 121378508Sbmilekic len -= off; 121478508Sbmilekic off = 0; 121578508Sbmilekic } 121678508Sbmilekic m->m_len = len = min(totlen, len); 12171541Srgrimes if (copy) 1218103569Sbmilekic copy(buf, mtod(m, caddr_t), (u_int)len); 12191541Srgrimes else 1220103569Sbmilekic bcopy(buf, mtod(m, caddr_t), (u_int)len); 122178508Sbmilekic buf += len; 12221541Srgrimes *mp = m; 12231541Srgrimes mp = &m->m_next; 12241541Srgrimes totlen -= len; 12251541Srgrimes } 12261541Srgrimes return (top); 12271541Srgrimes} 12283352Sphk 12293352Sphk/* 12303352Sphk * Copy data from a buffer back into the indicated mbuf chain, 12313352Sphk * starting "off" bytes from the beginning, extending the mbuf 12323352Sphk * chain if necessary. 12333352Sphk */ 12343352Sphkvoid 1235128402Sluigim_copyback(struct mbuf *m0, int off, int len, c_caddr_t cp) 12363352Sphk{ 123772356Sbmilekic int mlen; 123872356Sbmilekic struct mbuf *m = m0, *n; 12393352Sphk int totlen = 0; 12403352Sphk 124172356Sbmilekic if (m0 == NULL) 12423352Sphk return; 12433352Sphk while (off > (mlen = m->m_len)) { 12443352Sphk off -= mlen; 12453352Sphk totlen += mlen; 124672356Sbmilekic if (m->m_next == NULL) { 1247129906Sbmilekic n = m_get(M_DONTWAIT, m->m_type); 124872356Sbmilekic if (n == NULL) 12493352Sphk goto out; 1250129906Sbmilekic bzero(mtod(n, caddr_t), MLEN); 12513352Sphk n->m_len = min(MLEN, len + off); 12523352Sphk m->m_next = n; 12533352Sphk } 12543352Sphk m = m->m_next; 12553352Sphk } 12563352Sphk while (len > 0) { 12573352Sphk mlen = min (m->m_len - off, len); 1258103569Sbmilekic bcopy(cp, off + mtod(m, caddr_t), (u_int)mlen); 12593352Sphk cp += mlen; 12603352Sphk len -= mlen; 12613352Sphk mlen += off; 12623352Sphk off = 0; 12633352Sphk totlen += mlen; 12643352Sphk if (len == 0) 12653352Sphk break; 126672356Sbmilekic if (m->m_next == NULL) { 1267111119Simp n = m_get(M_DONTWAIT, m->m_type); 126872356Sbmilekic if (n == NULL) 12693352Sphk break; 12703352Sphk n->m_len = min(MLEN, len); 12713352Sphk m->m_next = n; 12723352Sphk } 12733352Sphk m = m->m_next; 12743352Sphk } 12753352Sphkout: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) 12763352Sphk m->m_pkthdr.len = totlen; 12773352Sphk} 127852756Sphk 1279123557Sbms/* 1280138541Ssam * Append the specified data to the indicated mbuf chain, 1281138541Ssam * Extend the mbuf chain if the new data does not fit in 1282138541Ssam * existing space. 1283138541Ssam * 1284138541Ssam * Return 1 if able to complete the job; otherwise 0. 1285138541Ssam */ 1286138541Ssamint 1287138541Ssamm_append(struct mbuf *m0, int len, c_caddr_t cp) 1288138541Ssam{ 1289138541Ssam struct mbuf *m, *n; 1290138541Ssam int remainder, space; 1291138541Ssam 1292138541Ssam for (m = m0; m->m_next != NULL; m = m->m_next) 1293138541Ssam ; 1294138541Ssam remainder = len; 1295138541Ssam space = M_TRAILINGSPACE(m); 1296138541Ssam if (space > 0) { 1297138541Ssam /* 1298138541Ssam * Copy into available space. 1299138541Ssam */ 1300138541Ssam if (space > remainder) 1301138541Ssam space = remainder; 1302138541Ssam bcopy(cp, mtod(m, caddr_t) + m->m_len, space); 1303138541Ssam m->m_len += space; 1304138541Ssam cp += space, remainder -= space; 1305138541Ssam } 1306138541Ssam while (remainder > 0) { 1307138541Ssam /* 1308138541Ssam * Allocate a new mbuf; could check space 1309138541Ssam * and allocate a cluster instead. 1310138541Ssam */ 1311138541Ssam n = m_get(M_DONTWAIT, m->m_type); 1312138541Ssam if (n == NULL) 1313138541Ssam break; 1314138541Ssam n->m_len = min(MLEN, remainder); 1315138894Ssam bcopy(cp, mtod(n, caddr_t), n->m_len); 1316138894Ssam cp += n->m_len, remainder -= n->m_len; 1317138541Ssam m->m_next = n; 1318138541Ssam m = n; 1319138541Ssam } 1320138541Ssam if (m0->m_flags & M_PKTHDR) 1321138541Ssam m0->m_pkthdr.len += len - remainder; 1322138541Ssam return (remainder == 0); 1323138541Ssam} 1324138541Ssam 1325138541Ssam/* 1326123557Sbms * Apply function f to the data in an mbuf chain starting "off" bytes from 1327123557Sbms * the beginning, continuing for "len" bytes. 1328123557Sbms */ 1329123557Sbmsint 1330123557Sbmsm_apply(struct mbuf *m, int off, int len, 1331123564Sbms int (*f)(void *, void *, u_int), void *arg) 1332123557Sbms{ 1333123564Sbms u_int count; 1334123557Sbms int rval; 1335123557Sbms 1336123557Sbms KASSERT(off >= 0, ("m_apply, negative off %d", off)); 1337123557Sbms KASSERT(len >= 0, ("m_apply, negative len %d", len)); 1338123557Sbms while (off > 0) { 1339123557Sbms KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain")); 1340123557Sbms if (off < m->m_len) 1341123557Sbms break; 1342123557Sbms off -= m->m_len; 1343123557Sbms m = m->m_next; 1344123557Sbms } 1345123557Sbms while (len > 0) { 1346123557Sbms KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain")); 1347123557Sbms count = min(m->m_len - off, len); 1348123557Sbms rval = (*f)(arg, mtod(m, caddr_t) + off, count); 1349123557Sbms if (rval) 1350123557Sbms return (rval); 1351123557Sbms len -= count; 1352123557Sbms off = 0; 1353123557Sbms m = m->m_next; 1354123557Sbms } 1355123557Sbms return (0); 1356123557Sbms} 1357123557Sbms 1358123557Sbms/* 1359123557Sbms * Return a pointer to mbuf/offset of location in mbuf chain. 1360123557Sbms */ 1361123557Sbmsstruct mbuf * 1362123557Sbmsm_getptr(struct mbuf *m, int loc, int *off) 1363123557Sbms{ 1364123557Sbms 1365123557Sbms while (loc >= 0) { 1366123564Sbms /* Normal end of search. */ 1367123557Sbms if (m->m_len > loc) { 1368123557Sbms *off = loc; 1369123557Sbms return (m); 1370123557Sbms } else { 1371123557Sbms loc -= m->m_len; 1372123557Sbms if (m->m_next == NULL) { 1373123557Sbms if (loc == 0) { 1374123564Sbms /* Point at the end of valid data. */ 1375123557Sbms *off = m->m_len; 1376123557Sbms return (m); 1377123564Sbms } 1378123564Sbms return (NULL); 1379123564Sbms } 1380123564Sbms m = m->m_next; 1381123557Sbms } 1382123557Sbms } 1383123557Sbms return (NULL); 1384123557Sbms} 1385123557Sbms 138652756Sphkvoid 1387135904Sjmgm_print(const struct mbuf *m, int maxlen) 138852756Sphk{ 138952756Sphk int len; 1390135904Sjmg int pdata; 139154906Seivind const struct mbuf *m2; 139252756Sphk 1393135904Sjmg if (m->m_flags & M_PKTHDR) 1394135904Sjmg len = m->m_pkthdr.len; 1395135904Sjmg else 1396135904Sjmg len = -1; 139752756Sphk m2 = m; 1398135904Sjmg while (m2 != NULL && (len == -1 || len)) { 1399135904Sjmg pdata = m2->m_len; 1400135904Sjmg if (maxlen != -1 && pdata > maxlen) 1401135904Sjmg pdata = maxlen; 1402135904Sjmg printf("mbuf: %p len: %d, next: %p, %b%s", m2, m2->m_len, 1403135904Sjmg m2->m_next, m2->m_flags, "\20\20freelist\17skipfw" 1404135904Sjmg "\11proto5\10proto4\7proto3\6proto2\5proto1\4rdonly" 1405135904Sjmg "\3eor\2pkthdr\1ext", pdata ? "" : "\n"); 1406135904Sjmg if (pdata) 1407135904Sjmg printf(", %*D\n", m2->m_len, (u_char *)m2->m_data, "-"); 1408135904Sjmg if (len != -1) 1409135904Sjmg len -= m2->m_len; 141052756Sphk m2 = m2->m_next; 141152756Sphk } 1412135904Sjmg if (len > 0) 1413135904Sjmg printf("%d bytes unaccounted for.\n", len); 141452756Sphk return; 141552756Sphk} 1416103540Sphk 1417103569Sbmilekicu_int 1418103540Sphkm_fixhdr(struct mbuf *m0) 1419103540Sphk{ 1420103569Sbmilekic u_int len; 1421103540Sphk 1422103544Sphk len = m_length(m0, NULL); 1423103544Sphk m0->m_pkthdr.len = len; 1424103544Sphk return (len); 1425103544Sphk} 1426103544Sphk 1427103569Sbmilekicu_int 1428103544Sphkm_length(struct mbuf *m0, struct mbuf **last) 1429103544Sphk{ 1430103544Sphk struct mbuf *m; 1431103569Sbmilekic u_int len; 1432103544Sphk 1433103544Sphk len = 0; 1434103544Sphk for (m = m0; m != NULL; m = m->m_next) { 1435103540Sphk len += m->m_len; 1436103544Sphk if (m->m_next == NULL) 1437103544Sphk break; 1438103540Sphk } 1439103544Sphk if (last != NULL) 1440103544Sphk *last = m; 1441103544Sphk return (len); 1442103540Sphk} 1443112777Ssilby 1444112777Ssilby/* 1445112777Ssilby * Defragment a mbuf chain, returning the shortest possible 1446112777Ssilby * chain of mbufs and clusters. If allocation fails and 1447112777Ssilby * this cannot be completed, NULL will be returned, but 1448112777Ssilby * the passed in chain will be unchanged. Upon success, 1449112777Ssilby * the original chain will be freed, and the new chain 1450112777Ssilby * will be returned. 1451112777Ssilby * 1452112777Ssilby * If a non-packet header is passed in, the original 1453112777Ssilby * mbuf (chain?) will be returned unharmed. 1454112777Ssilby */ 1455112777Ssilbystruct mbuf * 1456112777Ssilbym_defrag(struct mbuf *m0, int how) 1457112777Ssilby{ 1458125472Ssilby struct mbuf *m_new = NULL, *m_final = NULL; 1459125472Ssilby int progress = 0, length; 1460112777Ssilby 1461132488Salfred MBUF_CHECKSLEEP(how); 1462112777Ssilby if (!(m0->m_flags & M_PKTHDR)) 1463112777Ssilby return (m0); 1464112777Ssilby 1465117770Ssilby m_fixhdr(m0); /* Needed sanity check */ 1466117770Ssilby 1467113490Ssilby#ifdef MBUF_STRESS_TEST 1468113490Ssilby if (m_defragrandomfailures) { 1469113490Ssilby int temp = arc4random() & 0xff; 1470113490Ssilby if (temp == 0xba) 1471113490Ssilby goto nospace; 1472113490Ssilby } 1473113490Ssilby#endif 1474112777Ssilby 1475112777Ssilby if (m0->m_pkthdr.len > MHLEN) 1476112777Ssilby m_final = m_getcl(how, MT_DATA, M_PKTHDR); 1477112777Ssilby else 1478112777Ssilby m_final = m_gethdr(how, MT_DATA); 1479112777Ssilby 1480112777Ssilby if (m_final == NULL) 1481112777Ssilby goto nospace; 1482112777Ssilby 1483123740Speter if (m_dup_pkthdr(m_final, m0, how) == 0) 1484112777Ssilby goto nospace; 1485112777Ssilby 1486112777Ssilby m_new = m_final; 1487112777Ssilby 1488112777Ssilby while (progress < m0->m_pkthdr.len) { 1489112777Ssilby length = m0->m_pkthdr.len - progress; 1490112777Ssilby if (length > MCLBYTES) 1491112777Ssilby length = MCLBYTES; 1492112777Ssilby 1493112777Ssilby if (m_new == NULL) { 1494112777Ssilby if (length > MLEN) 1495112777Ssilby m_new = m_getcl(how, MT_DATA, 0); 1496112777Ssilby else 1497112777Ssilby m_new = m_get(how, MT_DATA); 1498112777Ssilby if (m_new == NULL) 1499112777Ssilby goto nospace; 1500112777Ssilby } 1501112777Ssilby 1502112777Ssilby m_copydata(m0, progress, length, mtod(m_new, caddr_t)); 1503112777Ssilby progress += length; 1504112777Ssilby m_new->m_len = length; 1505112777Ssilby if (m_new != m_final) 1506112777Ssilby m_cat(m_final, m_new); 1507112777Ssilby m_new = NULL; 1508112777Ssilby } 1509116455Ssilby#ifdef MBUF_STRESS_TEST 1510112777Ssilby if (m0->m_next == NULL) 1511112777Ssilby m_defraguseless++; 1512116455Ssilby#endif 1513112777Ssilby m_freem(m0); 1514112777Ssilby m0 = m_final; 1515116455Ssilby#ifdef MBUF_STRESS_TEST 1516112777Ssilby m_defragpackets++; 1517112777Ssilby m_defragbytes += m0->m_pkthdr.len; 1518116455Ssilby#endif 1519112777Ssilby return (m0); 1520112777Ssilbynospace: 1521116455Ssilby#ifdef MBUF_STRESS_TEST 1522112777Ssilby m_defragfailure++; 1523116455Ssilby#endif 1524112777Ssilby if (m_final) 1525112777Ssilby m_freem(m_final); 1526112777Ssilby return (NULL); 1527112777Ssilby} 1528119644Ssilby 1529119644Ssilby#ifdef MBUF_STRESS_TEST 1530119644Ssilby 1531119644Ssilby/* 1532119644Ssilby * Fragment an mbuf chain. There's no reason you'd ever want to do 1533119644Ssilby * this in normal usage, but it's great for stress testing various 1534119644Ssilby * mbuf consumers. 1535119644Ssilby * 1536119644Ssilby * If fragmentation is not possible, the original chain will be 1537119644Ssilby * returned. 1538119644Ssilby * 1539119644Ssilby * Possible length values: 1540119644Ssilby * 0 no fragmentation will occur 1541119644Ssilby * > 0 each fragment will be of the specified length 1542119644Ssilby * -1 each fragment will be the same random value in length 1543119644Ssilby * -2 each fragment's length will be entirely random 1544119644Ssilby * (Random values range from 1 to 256) 1545119644Ssilby */ 1546119644Ssilbystruct mbuf * 1547119644Ssilbym_fragment(struct mbuf *m0, int how, int length) 1548119644Ssilby{ 1549125472Ssilby struct mbuf *m_new = NULL, *m_final = NULL; 1550125472Ssilby int progress = 0; 1551119644Ssilby 1552119644Ssilby if (!(m0->m_flags & M_PKTHDR)) 1553119644Ssilby return (m0); 1554119644Ssilby 1555119644Ssilby if ((length == 0) || (length < -2)) 1556119644Ssilby return (m0); 1557119644Ssilby 1558119644Ssilby m_fixhdr(m0); /* Needed sanity check */ 1559119644Ssilby 1560119644Ssilby m_final = m_getcl(how, MT_DATA, M_PKTHDR); 1561119644Ssilby 1562119644Ssilby if (m_final == NULL) 1563119644Ssilby goto nospace; 1564119644Ssilby 1565123823Ssilby if (m_dup_pkthdr(m_final, m0, how) == 0) 1566119644Ssilby goto nospace; 1567119644Ssilby 1568119644Ssilby m_new = m_final; 1569119644Ssilby 1570119644Ssilby if (length == -1) 1571119644Ssilby length = 1 + (arc4random() & 255); 1572119644Ssilby 1573119644Ssilby while (progress < m0->m_pkthdr.len) { 1574119644Ssilby int fraglen; 1575119644Ssilby 1576119644Ssilby if (length > 0) 1577119644Ssilby fraglen = length; 1578119644Ssilby else 1579119644Ssilby fraglen = 1 + (arc4random() & 255); 1580119644Ssilby if (fraglen > m0->m_pkthdr.len - progress) 1581119644Ssilby fraglen = m0->m_pkthdr.len - progress; 1582119644Ssilby 1583119644Ssilby if (fraglen > MCLBYTES) 1584119644Ssilby fraglen = MCLBYTES; 1585119644Ssilby 1586119644Ssilby if (m_new == NULL) { 1587119644Ssilby m_new = m_getcl(how, MT_DATA, 0); 1588119644Ssilby if (m_new == NULL) 1589119644Ssilby goto nospace; 1590119644Ssilby } 1591119644Ssilby 1592119644Ssilby m_copydata(m0, progress, fraglen, mtod(m_new, caddr_t)); 1593119644Ssilby progress += fraglen; 1594119644Ssilby m_new->m_len = fraglen; 1595119644Ssilby if (m_new != m_final) 1596119644Ssilby m_cat(m_final, m_new); 1597119644Ssilby m_new = NULL; 1598119644Ssilby } 1599119644Ssilby m_freem(m0); 1600119644Ssilby m0 = m_final; 1601119644Ssilby return (m0); 1602119644Ssilbynospace: 1603119644Ssilby if (m_final) 1604119644Ssilby m_freem(m_final); 1605119644Ssilby /* Return the original chain on failure */ 1606119644Ssilby return (m0); 1607119644Ssilby} 1608119644Ssilby 1609119644Ssilby#endif 1610125296Ssilby 1611125296Ssilbystruct mbuf * 1612145883Semaxm_uiotombuf(struct uio *uio, int how, int len, int align) 1613125296Ssilby{ 1614125472Ssilby struct mbuf *m_new = NULL, *m_final = NULL; 1615125472Ssilby int progress = 0, error = 0, length, total; 1616125296Ssilby 1617125296Ssilby if (len > 0) 1618125296Ssilby total = min(uio->uio_resid, len); 1619125296Ssilby else 1620125296Ssilby total = uio->uio_resid; 1621145883Semax if (align >= MHLEN) 1622145883Semax goto nospace; 1623145883Semax if (total + align > MHLEN) 1624125296Ssilby m_final = m_getcl(how, MT_DATA, M_PKTHDR); 1625125296Ssilby else 1626125296Ssilby m_final = m_gethdr(how, MT_DATA); 1627125296Ssilby if (m_final == NULL) 1628125296Ssilby goto nospace; 1629145883Semax m_final->m_data += align; 1630125296Ssilby m_new = m_final; 1631125296Ssilby while (progress < total) { 1632125296Ssilby length = total - progress; 1633125296Ssilby if (length > MCLBYTES) 1634125296Ssilby length = MCLBYTES; 1635125296Ssilby if (m_new == NULL) { 1636125296Ssilby if (length > MLEN) 1637125296Ssilby m_new = m_getcl(how, MT_DATA, 0); 1638125296Ssilby else 1639125296Ssilby m_new = m_get(how, MT_DATA); 1640125296Ssilby if (m_new == NULL) 1641125296Ssilby goto nospace; 1642125296Ssilby } 1643125296Ssilby error = uiomove(mtod(m_new, void *), length, uio); 1644125296Ssilby if (error) 1645125296Ssilby goto nospace; 1646125296Ssilby progress += length; 1647125296Ssilby m_new->m_len = length; 1648125296Ssilby if (m_new != m_final) 1649125296Ssilby m_cat(m_final, m_new); 1650125296Ssilby m_new = NULL; 1651125296Ssilby } 1652125296Ssilby m_fixhdr(m_final); 1653125296Ssilby return (m_final); 1654125296Ssilbynospace: 1655125296Ssilby if (m_new) 1656125296Ssilby m_free(m_new); 1657125296Ssilby if (m_final) 1658125296Ssilby m_freem(m_final); 1659125296Ssilby return (NULL); 1660125296Ssilby} 1661148552Ssam 1662148552Ssam/* 1663148552Ssam * Set the m_data pointer of a newly-allocated mbuf 1664148552Ssam * to place an object of the specified size at the 1665148552Ssam * end of the mbuf, longword aligned. 1666148552Ssam */ 1667148552Ssamvoid 1668148552Ssamm_align(struct mbuf *m, int len) 1669148552Ssam{ 1670148552Ssam int adjust; 1671148552Ssam 1672148552Ssam if (m->m_flags & M_EXT) 1673148552Ssam adjust = m->m_ext.ext_size - len; 1674148552Ssam else if (m->m_flags & M_PKTHDR) 1675148552Ssam adjust = MHLEN - len; 1676148552Ssam else 1677148552Ssam adjust = MLEN - len; 1678148552Ssam m->m_data += adjust &~ (sizeof(long)-1); 1679148552Ssam} 1680