uipc_mbuf.c revision 149602
1139804Simp/*- 21541Srgrimes * Copyright (c) 1982, 1986, 1988, 1991, 1993 31541Srgrimes * The Regents of the University of California. All rights reserved. 41541Srgrimes * 51541Srgrimes * Redistribution and use in source and binary forms, with or without 61541Srgrimes * modification, are permitted provided that the following conditions 71541Srgrimes * are met: 81541Srgrimes * 1. Redistributions of source code must retain the above copyright 91541Srgrimes * notice, this list of conditions and the following disclaimer. 101541Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 111541Srgrimes * notice, this list of conditions and the following disclaimer in the 121541Srgrimes * documentation and/or other materials provided with the distribution. 131541Srgrimes * 4. Neither the name of the University nor the names of its contributors 141541Srgrimes * may be used to endorse or promote products derived from this software 151541Srgrimes * without specific prior written permission. 161541Srgrimes * 171541Srgrimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 181541Srgrimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 191541Srgrimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 201541Srgrimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 211541Srgrimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 221541Srgrimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 231541Srgrimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 241541Srgrimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 251541Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 261541Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 271541Srgrimes * SUCH DAMAGE. 281541Srgrimes * 291541Srgrimes * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94 301541Srgrimes */ 311541Srgrimes 32116182Sobrien#include <sys/cdefs.h> 33116182Sobrien__FBSDID("$FreeBSD: head/sys/kern/uipc_mbuf.c 149602 2005-08-29 20:15:33Z andre $"); 34116182Sobrien 35101007Srwatson#include "opt_mac.h" 3677572Sobrien#include "opt_param.h" 37113490Ssilby#include "opt_mbuf_stress_test.h" 38101007Srwatson 391541Srgrimes#include <sys/param.h> 401541Srgrimes#include <sys/systm.h> 4176166Smarkm#include <sys/kernel.h> 42125296Ssilby#include <sys/limits.h> 4376166Smarkm#include <sys/lock.h> 44101173Srwatson#include <sys/mac.h> 4532036Sbde#include <sys/malloc.h> 461541Srgrimes#include <sys/mbuf.h> 4723081Swollman#include <sys/sysctl.h> 481541Srgrimes#include <sys/domain.h> 491541Srgrimes#include <sys/protosw.h> 50125296Ssilby#include <sys/uio.h> 5176166Smarkm 529759Sbdeint max_linkhdr; 539759Sbdeint max_protohdr; 549759Sbdeint max_hdr; 559759Sbdeint max_datalen; 56116455Ssilby#ifdef MBUF_STRESS_TEST 57112777Ssilbyint m_defragpackets; 58112777Ssilbyint m_defragbytes; 59112777Ssilbyint m_defraguseless; 60112777Ssilbyint m_defragfailure; 61113490Ssilbyint m_defragrandomfailures; 62113490Ssilby#endif 631541Srgrimes 6466475Sbmilekic/* 6566475Sbmilekic * sysctl(8) exported objects 6666475Sbmilekic */ 6744078SdfrSYSCTL_DECL(_kern_ipc); 6823081SwollmanSYSCTL_INT(_kern_ipc, KIPC_MAX_LINKHDR, max_linkhdr, CTLFLAG_RW, 6923081Swollman &max_linkhdr, 0, ""); 7023081SwollmanSYSCTL_INT(_kern_ipc, KIPC_MAX_PROTOHDR, max_protohdr, CTLFLAG_RW, 7123081Swollman &max_protohdr, 0, ""); 7223081SwollmanSYSCTL_INT(_kern_ipc, KIPC_MAX_HDR, max_hdr, CTLFLAG_RW, &max_hdr, 0, ""); 7323081SwollmanSYSCTL_INT(_kern_ipc, KIPC_MAX_DATALEN, max_datalen, CTLFLAG_RW, 7423081Swollman &max_datalen, 0, ""); 75116455Ssilby#ifdef MBUF_STRESS_TEST 76112777SsilbySYSCTL_INT(_kern_ipc, OID_AUTO, m_defragpackets, CTLFLAG_RD, 77112777Ssilby &m_defragpackets, 0, ""); 78112777SsilbySYSCTL_INT(_kern_ipc, OID_AUTO, m_defragbytes, CTLFLAG_RD, 79112777Ssilby &m_defragbytes, 0, ""); 80112777SsilbySYSCTL_INT(_kern_ipc, OID_AUTO, m_defraguseless, CTLFLAG_RD, 81112777Ssilby &m_defraguseless, 0, ""); 82112777SsilbySYSCTL_INT(_kern_ipc, OID_AUTO, m_defragfailure, CTLFLAG_RD, 83112777Ssilby &m_defragfailure, 0, ""); 84113490SsilbySYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW, 85113490Ssilby &m_defragrandomfailures, 0, ""); 86113490Ssilby#endif 8775112Sbmilekic 881541Srgrimes/* 89129906Sbmilekic * Malloc-type for external ext_buf ref counts. 90129906Sbmilekic */ 91141616Sphkstatic MALLOC_DEFINE(M_MBUF, "mbextcnt", "mbuf external ref counts"); 92129906Sbmilekic 93129906Sbmilekic/* 94129906Sbmilekic * Allocate a given length worth of mbufs and/or clusters (whatever fits 95129906Sbmilekic * best) and return a pointer to the top of the allocated chain. If an 96129906Sbmilekic * existing mbuf chain is provided, then we will append the new chain 97129906Sbmilekic * to the existing one but still return the top of the newly allocated 98129906Sbmilekic * chain. 99129906Sbmilekic */ 100129906Sbmilekicstruct mbuf * 101129906Sbmilekicm_getm(struct mbuf *m, int len, int how, short type) 102129906Sbmilekic{ 103129906Sbmilekic struct mbuf *mb, *top, *cur, *mtail; 104129906Sbmilekic int num, rem; 105129906Sbmilekic int i; 106129906Sbmilekic 107129906Sbmilekic KASSERT(len >= 0, ("m_getm(): len is < 0")); 108129906Sbmilekic 109129906Sbmilekic /* If m != NULL, we will append to the end of that chain. */ 110129906Sbmilekic if (m != NULL) 111129906Sbmilekic for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next); 112129906Sbmilekic else 113129906Sbmilekic mtail = NULL; 114129906Sbmilekic 115129906Sbmilekic /* 116129906Sbmilekic * Calculate how many mbufs+clusters ("packets") we need and how much 117129906Sbmilekic * leftover there is after that and allocate the first mbuf+cluster 118129906Sbmilekic * if required. 119129906Sbmilekic */ 120129906Sbmilekic num = len / MCLBYTES; 121129906Sbmilekic rem = len % MCLBYTES; 122129906Sbmilekic top = cur = NULL; 123129906Sbmilekic if (num > 0) { 124129906Sbmilekic if ((top = cur = m_getcl(how, type, 0)) == NULL) 125129906Sbmilekic goto failed; 126130271Smux top->m_len = 0; 127129906Sbmilekic } 128129906Sbmilekic num--; 129129906Sbmilekic 130129906Sbmilekic for (i = 0; i < num; i++) { 131129906Sbmilekic mb = m_getcl(how, type, 0); 132129906Sbmilekic if (mb == NULL) 133129906Sbmilekic goto failed; 134129906Sbmilekic mb->m_len = 0; 135129906Sbmilekic cur = (cur->m_next = mb); 136129906Sbmilekic } 137129906Sbmilekic if (rem > 0) { 138129906Sbmilekic mb = (rem > MINCLSIZE) ? 139129906Sbmilekic m_getcl(how, type, 0) : m_get(how, type); 140129906Sbmilekic if (mb == NULL) 141129906Sbmilekic goto failed; 142129906Sbmilekic mb->m_len = 0; 143129906Sbmilekic if (cur == NULL) 144129906Sbmilekic top = mb; 145129906Sbmilekic else 146129906Sbmilekic cur->m_next = mb; 147129906Sbmilekic } 148129906Sbmilekic 149129906Sbmilekic if (mtail != NULL) 150129906Sbmilekic mtail->m_next = top; 151129906Sbmilekic return top; 152129906Sbmilekicfailed: 153129906Sbmilekic if (top != NULL) 154129906Sbmilekic m_freem(top); 155129906Sbmilekic return NULL; 156129906Sbmilekic} 157129906Sbmilekic 158129906Sbmilekic/* 159129906Sbmilekic * Free an entire chain of mbufs and associated external buffers, if 160129906Sbmilekic * applicable. 161129906Sbmilekic */ 162129906Sbmilekicvoid 163129906Sbmilekicm_freem(struct mbuf *mb) 164129906Sbmilekic{ 165129906Sbmilekic 166129906Sbmilekic while (mb != NULL) 167129906Sbmilekic mb = m_free(mb); 168129906Sbmilekic} 169129906Sbmilekic 170129906Sbmilekic/*- 171129906Sbmilekic * Configure a provided mbuf to refer to the provided external storage 172129906Sbmilekic * buffer and setup a reference count for said buffer. If the setting 173129906Sbmilekic * up of the reference count fails, the M_EXT bit will not be set. If 174129906Sbmilekic * successfull, the M_EXT bit is set in the mbuf's flags. 175129906Sbmilekic * 176129906Sbmilekic * Arguments: 177129906Sbmilekic * mb The existing mbuf to which to attach the provided buffer. 178129906Sbmilekic * buf The address of the provided external storage buffer. 179129906Sbmilekic * size The size of the provided buffer. 180129906Sbmilekic * freef A pointer to a routine that is responsible for freeing the 181129906Sbmilekic * provided external storage buffer. 182129906Sbmilekic * args A pointer to an argument structure (of any type) to be passed 183129906Sbmilekic * to the provided freef routine (may be NULL). 184129906Sbmilekic * flags Any other flags to be passed to the provided mbuf. 185129906Sbmilekic * type The type that the external storage buffer should be 186129906Sbmilekic * labeled with. 187129906Sbmilekic * 188129906Sbmilekic * Returns: 189129906Sbmilekic * Nothing. 190129906Sbmilekic */ 191129906Sbmilekicvoid 192129906Sbmilekicm_extadd(struct mbuf *mb, caddr_t buf, u_int size, 193129906Sbmilekic void (*freef)(void *, void *), void *args, int flags, int type) 194129906Sbmilekic{ 195129906Sbmilekic u_int *ref_cnt = NULL; 196129906Sbmilekic 197129906Sbmilekic /* XXX Shouldn't be adding EXT_CLUSTER with this API */ 198129906Sbmilekic if (type == EXT_CLUSTER) 199129906Sbmilekic ref_cnt = (u_int *)uma_find_refcnt(zone_clust, 200129906Sbmilekic mb->m_ext.ext_buf); 201129906Sbmilekic else if (type == EXT_EXTREF) 202143188Salc ref_cnt = __DEVOLATILE(u_int *, mb->m_ext.ref_cnt); 203129906Sbmilekic mb->m_ext.ref_cnt = (ref_cnt == NULL) ? 204129906Sbmilekic malloc(sizeof(u_int), M_MBUF, M_NOWAIT) : (u_int *)ref_cnt; 205129906Sbmilekic if (mb->m_ext.ref_cnt != NULL) { 206129906Sbmilekic *(mb->m_ext.ref_cnt) = 1; 207129906Sbmilekic mb->m_flags |= (M_EXT | flags); 208129906Sbmilekic mb->m_ext.ext_buf = buf; 209129906Sbmilekic mb->m_data = mb->m_ext.ext_buf; 210129906Sbmilekic mb->m_ext.ext_size = size; 211129906Sbmilekic mb->m_ext.ext_free = freef; 212129906Sbmilekic mb->m_ext.ext_args = args; 213129906Sbmilekic mb->m_ext.ext_type = type; 214129906Sbmilekic } 215129906Sbmilekic} 216129906Sbmilekic 217129906Sbmilekic/* 218129906Sbmilekic * Non-directly-exported function to clean up after mbufs with M_EXT 219129906Sbmilekic * storage attached to them if the reference count hits 0. 220129906Sbmilekic */ 221129906Sbmilekicvoid 222129906Sbmilekicmb_free_ext(struct mbuf *m) 223129906Sbmilekic{ 224130289Sbmilekic u_int cnt; 225141668Sbmilekic int dofree; 226129906Sbmilekic 227141668Sbmilekic /* Account for lazy ref count assign. */ 228141668Sbmilekic if (m->m_ext.ref_cnt == NULL) 229141668Sbmilekic dofree = 1; 230141668Sbmilekic else 231141668Sbmilekic dofree = 0; 232141668Sbmilekic 233130289Sbmilekic /* 234130289Sbmilekic * This is tricky. We need to make sure to decrement the 235130289Sbmilekic * refcount in a safe way but to also clean up if we're the 236130289Sbmilekic * last reference. This method seems to do it without race. 237130289Sbmilekic */ 238141668Sbmilekic while (dofree == 0) { 239143188Salc cnt = *(m->m_ext.ref_cnt); 240130289Sbmilekic if (atomic_cmpset_int(m->m_ext.ref_cnt, cnt, cnt - 1)) { 241141668Sbmilekic if (cnt == 1) 242141668Sbmilekic dofree = 1; 243130289Sbmilekic break; 244129906Sbmilekic } 245141668Sbmilekic } 246141668Sbmilekic 247141668Sbmilekic if (dofree) { 248141668Sbmilekic /* 249141668Sbmilekic * Do the free, should be safe. 250141668Sbmilekic */ 251141668Sbmilekic if (m->m_ext.ext_type == EXT_PACKET) { 252141668Sbmilekic uma_zfree(zone_pack, m); 253141668Sbmilekic return; 254141668Sbmilekic } else if (m->m_ext.ext_type == EXT_CLUSTER) { 255141668Sbmilekic uma_zfree(zone_clust, m->m_ext.ext_buf); 256141668Sbmilekic m->m_ext.ext_buf = NULL; 257141668Sbmilekic } else { 258141668Sbmilekic (*(m->m_ext.ext_free))(m->m_ext.ext_buf, 259141668Sbmilekic m->m_ext.ext_args); 260141668Sbmilekic if (m->m_ext.ext_type != EXT_EXTREF) { 261141668Sbmilekic if (m->m_ext.ref_cnt != NULL) 262143188Salc free(__DEVOLATILE(u_int *, 263143188Salc m->m_ext.ref_cnt), M_MBUF); 264141668Sbmilekic m->m_ext.ref_cnt = NULL; 265141668Sbmilekic } 266141668Sbmilekic m->m_ext.ext_buf = NULL; 267141668Sbmilekic } 268141668Sbmilekic } 269130357Sbmilekic uma_zfree(zone_mbuf, m); 270129906Sbmilekic} 271129906Sbmilekic 272129906Sbmilekic/* 273149598Sandre * Clean up mbuf (chain) from any tags and packet headers. 274149598Sandre * If "all" is set then the first mbuf in the chain will be 275149598Sandre * cleaned too. 276149598Sandre */ 277149598Sandrevoid 278149598Sandrem_demote(struct mbuf *m0, int all) 279149598Sandre{ 280149598Sandre struct mbuf *m; 281149598Sandre 282149598Sandre for (m = all ? m0 : m0->m_next; m != NULL; m = m->m_next) { 283149598Sandre if (m->m_flags & M_PKTHDR) { 284149598Sandre m_tag_delete_chain(m, NULL); 285149598Sandre m->m_flags &= ~M_PKTHDR; 286149598Sandre bzero(&m->m_pkthdr, sizeof(struct pkthdr)); 287149598Sandre } 288149598Sandre if (m->m_type & MT_HEADER) 289149598Sandre m->m_type = MT_DATA; 290149598Sandre if (m != m0 && m->m_nextpkt) 291149598Sandre m->m_nextpkt = NULL; 292149598Sandre m->m_flags = m->m_flags & (M_EXT|M_EOR|M_RDONLY|M_FREELIST); 293149598Sandre } 294149598Sandre} 295149598Sandre 296149598Sandre/* 297149599Sandre * Sanity checks on mbuf (chain). 298149599Sandre * Returns 0 bad, 1 good, panic worse. 299149599Sandre * sanitize, 0 run M_SANITY_ACTION, 1 garble things so they blow up later. 300149599Sandre */ 301149599Sandreint 302149599Sandrem_sanity(struct mbuf *m0, int sanitize) 303149599Sandre{ 304149599Sandre struct mbuf *m; 305149599Sandre caddr_t a, b; 306149599Sandre int pktlen = 0; 307149599Sandre 308149599Sandre#define M_SANITY_ACTION(s) return (0) 309149599Sandre/* #define M_SANITY_ACTION(s) panic("mbuf %p: " s, m) */ 310149599Sandre 311149599Sandre m = m0; 312149599Sandre while (m) { 313149599Sandre /* 314149599Sandre * Basic pointer checks. If any of these fails then some 315149599Sandre * unrelated kernel memory before or after us is trashed. 316149599Sandre * No way to recover from that. 317149599Sandre */ 318149599Sandre a = (m->m_flags & M_EXT ? m->m_ext.ext_buf : 319149599Sandre (m->m_flags & M_PKTHDR ? (caddr_t)(&m->m_pktdat) : 320149599Sandre (caddr_t)(&m->m_dat)) ); 321149599Sandre b = (caddr_t)(a + (m->m_flags & M_EXT ? m->m_ext.ext_size : 322149599Sandre (m->m_flags & M_PKTHDR ? MHLEN : MLEN))); 323149599Sandre if ((caddr_t)m->m_data < a) 324149599Sandre M_SANITY_ACTION("m_data outside mbuf data range left"); 325149599Sandre if ((caddr_t)m->m_data > b) 326149599Sandre M_SANITY_ACTION("m_data outside mbuf data range right"); 327149599Sandre if ((caddr_t)m->m_data + m->m_len > b) 328149599Sandre M_SANITY_ACTION("m_data + m_len exeeds mbuf space"); 329149599Sandre if (m->m_flags & M_PKTHDR && m->m_pkthdr.header) { 330149599Sandre if ((caddr_t)m->m_pkthdr.header < a || 331149599Sandre (caddr_t)m->m_pkthdr.header > b) 332149599Sandre M_SANITY_ACTION("m_pkthdr.header outside mbuf data range"); 333149599Sandre } 334149599Sandre 335149599Sandre /* m->m_nextpkt may only be set on first mbuf in chain. */ 336149599Sandre if (m != m0 && m->m_nextpkt) { 337149599Sandre if (sanitize) { 338149599Sandre m_freem(m->m_nextpkt); 339149599Sandre m->m_nextpkt = (struct mbuf *)0xDEADC0DE; 340149599Sandre } else 341149599Sandre M_SANITY_ACTION("m->m_nextpkt on in-chain mbuf"); 342149599Sandre } 343149599Sandre 344149599Sandre /* correct type correlations. */ 345149599Sandre if (m->m_type == MT_HEADER && !(m->m_flags & M_PKTHDR)) { 346149599Sandre if (sanitize) 347149599Sandre m->m_type = MT_DATA; 348149599Sandre else 349149599Sandre M_SANITY_ACTION("MT_HEADER set but not M_PKTHDR"); 350149599Sandre } 351149599Sandre 352149599Sandre /* packet length (not mbuf length!) calculation */ 353149599Sandre if (m0->m_flags & M_PKTHDR) 354149599Sandre pktlen += m->m_len; 355149599Sandre 356149599Sandre /* m_tags may only be attached to first mbuf in chain. */ 357149599Sandre if (m != m0 && m->m_flags & M_PKTHDR && 358149599Sandre !SLIST_EMPTY(&m->m_pkthdr.tags)) { 359149599Sandre if (sanitize) { 360149599Sandre m_tag_delete_chain(m, NULL); 361149599Sandre /* put in 0xDEADC0DE perhaps? */ 362149599Sandre } 363149599Sandre else 364149599Sandre M_SANITY_ACTION("m_tags on in-chain mbuf"); 365149599Sandre } 366149599Sandre 367149599Sandre /* M_PKTHDR may only be set on first mbuf in chain */ 368149599Sandre if (m != m0 && m->m_flags & M_PKTHDR) { 369149599Sandre if (sanitize) { 370149599Sandre bzero(&m->m_pkthdr, sizeof(m->m_pkthdr)); 371149599Sandre m->m_flags &= ~M_PKTHDR; 372149599Sandre /* put in 0xDEADCODE and leave hdr flag in */ 373149599Sandre } else 374149599Sandre M_SANITY_ACTION("M_PKTHDR on in-chain mbuf"); 375149599Sandre } 376149599Sandre 377149599Sandre m = m->m_next; 378149599Sandre } 379149599Sandre if (pktlen && pktlen != m0->m_pkthdr.len) { 380149599Sandre if (sanitize) 381149599Sandre m0->m_pkthdr.len = 0; 382149599Sandre else 383149599Sandre M_SANITY_ACTION("m_pkthdr.len != mbuf chain length"); 384149599Sandre } 385149599Sandre#undef M_SANITY_ACTION 386149599Sandre 387149599Sandre return 1; 388149599Sandre} 389149599Sandre 390149599Sandre 391149599Sandre/* 392108466Ssam * "Move" mbuf pkthdr from "from" to "to". 393100960Srwatson * "from" must have M_PKTHDR set, and "to" must be empty. 394100960Srwatson */ 395100960Srwatsonvoid 396108466Ssamm_move_pkthdr(struct mbuf *to, struct mbuf *from) 397100960Srwatson{ 398100960Srwatson 399100960Srwatson#if 0 400108466Ssam /* see below for why these are not enabled */ 401113255Sdes M_ASSERTPKTHDR(to); 402113487Srwatson /* Note: with MAC, this may not be a good assertion. */ 403108466Ssam KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), 404108466Ssam ("m_move_pkthdr: to has tags")); 405100960Srwatson#endif 406101007Srwatson#ifdef MAC 407113487Srwatson /* 408113487Srwatson * XXXMAC: It could be this should also occur for non-MAC? 409113487Srwatson */ 410101007Srwatson if (to->m_flags & M_PKTHDR) 411113487Srwatson m_tag_delete_chain(to, NULL); 412101007Srwatson#endif 413143302Ssam to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT); 414143302Ssam if ((to->m_flags & M_EXT) == 0) 415143302Ssam to->m_data = to->m_pktdat; 416108466Ssam to->m_pkthdr = from->m_pkthdr; /* especially tags */ 417108466Ssam SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */ 418108466Ssam from->m_flags &= ~M_PKTHDR; 419108466Ssam} 420108466Ssam 421108466Ssam/* 422108466Ssam * Duplicate "from"'s mbuf pkthdr in "to". 423108466Ssam * "from" must have M_PKTHDR set, and "to" must be empty. 424108466Ssam * In particular, this does a deep copy of the packet tags. 425108466Ssam */ 426108466Ssamint 427108466Ssamm_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how) 428108466Ssam{ 429108466Ssam 430108466Ssam#if 0 431108466Ssam /* 432108466Ssam * The mbuf allocator only initializes the pkthdr 433108466Ssam * when the mbuf is allocated with MGETHDR. Many users 434108466Ssam * (e.g. m_copy*, m_prepend) use MGET and then 435108466Ssam * smash the pkthdr as needed causing these 436108466Ssam * assertions to trip. For now just disable them. 437108466Ssam */ 438113255Sdes M_ASSERTPKTHDR(to); 439113487Srwatson /* Note: with MAC, this may not be a good assertion. */ 440108466Ssam KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), ("m_dup_pkthdr: to has tags")); 441108466Ssam#endif 442132488Salfred MBUF_CHECKSLEEP(how); 443108466Ssam#ifdef MAC 444108466Ssam if (to->m_flags & M_PKTHDR) 445113487Srwatson m_tag_delete_chain(to, NULL); 446108466Ssam#endif 447112733Ssilby to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT); 448112733Ssilby if ((to->m_flags & M_EXT) == 0) 449112733Ssilby to->m_data = to->m_pktdat; 450100960Srwatson to->m_pkthdr = from->m_pkthdr; 451108466Ssam SLIST_INIT(&to->m_pkthdr.tags); 452113480Srwatson return (m_tag_copy_chain(to, from, MBTOM(how))); 453100960Srwatson} 454100960Srwatson 455100960Srwatson/* 4561541Srgrimes * Lesser-used path for M_PREPEND: 4571541Srgrimes * allocate new mbuf to prepend to chain, 4581541Srgrimes * copy junk along. 4591541Srgrimes */ 4601541Srgrimesstruct mbuf * 46172356Sbmilekicm_prepend(struct mbuf *m, int len, int how) 4621541Srgrimes{ 4631541Srgrimes struct mbuf *mn; 4641541Srgrimes 465117770Ssilby if (m->m_flags & M_PKTHDR) 466117770Ssilby MGETHDR(mn, how, m->m_type); 467117770Ssilby else 468117770Ssilby MGET(mn, how, m->m_type); 46972356Sbmilekic if (mn == NULL) { 4701541Srgrimes m_freem(m); 47172356Sbmilekic return (NULL); 4721541Srgrimes } 473113487Srwatson if (m->m_flags & M_PKTHDR) 474108466Ssam M_MOVE_PKTHDR(mn, m); 4751541Srgrimes mn->m_next = m; 4761541Srgrimes m = mn; 4771541Srgrimes if (len < MHLEN) 4781541Srgrimes MH_ALIGN(m, len); 4791541Srgrimes m->m_len = len; 4801541Srgrimes return (m); 4811541Srgrimes} 4821541Srgrimes 4831541Srgrimes/* 4841541Srgrimes * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 4851541Srgrimes * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 486111119Simp * The wait parameter is a choice of M_TRYWAIT/M_DONTWAIT from caller. 48754002Sarchie * Note that the copy is read-only, because clusters are not copied, 48854002Sarchie * only their reference counts are incremented. 4891541Srgrimes */ 4901541Srgrimesstruct mbuf * 49172356Sbmilekicm_copym(struct mbuf *m, int off0, int len, int wait) 4921541Srgrimes{ 49372356Sbmilekic struct mbuf *n, **np; 49472356Sbmilekic int off = off0; 4951541Srgrimes struct mbuf *top; 4961541Srgrimes int copyhdr = 0; 4971541Srgrimes 49852201Salfred KASSERT(off >= 0, ("m_copym, negative off %d", off)); 49952201Salfred KASSERT(len >= 0, ("m_copym, negative len %d", len)); 500132488Salfred MBUF_CHECKSLEEP(wait); 5011541Srgrimes if (off == 0 && m->m_flags & M_PKTHDR) 5021541Srgrimes copyhdr = 1; 5031541Srgrimes while (off > 0) { 50452201Salfred KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain")); 5051541Srgrimes if (off < m->m_len) 5061541Srgrimes break; 5071541Srgrimes off -= m->m_len; 5081541Srgrimes m = m->m_next; 5091541Srgrimes } 5101541Srgrimes np = ⊤ 5111541Srgrimes top = 0; 5121541Srgrimes while (len > 0) { 51372356Sbmilekic if (m == NULL) { 51452201Salfred KASSERT(len == M_COPYALL, 51552201Salfred ("m_copym, length > size of mbuf chain")); 5161541Srgrimes break; 5171541Srgrimes } 518117770Ssilby if (copyhdr) 519117770Ssilby MGETHDR(n, wait, m->m_type); 520117770Ssilby else 521117770Ssilby MGET(n, wait, m->m_type); 5221541Srgrimes *np = n; 52372356Sbmilekic if (n == NULL) 5241541Srgrimes goto nospace; 5251541Srgrimes if (copyhdr) { 526108466Ssam if (!m_dup_pkthdr(n, m, wait)) 527108466Ssam goto nospace; 5281541Srgrimes if (len == M_COPYALL) 5291541Srgrimes n->m_pkthdr.len -= off0; 5301541Srgrimes else 5311541Srgrimes n->m_pkthdr.len = len; 5321541Srgrimes copyhdr = 0; 5331541Srgrimes } 5341541Srgrimes n->m_len = min(len, m->m_len - off); 5351541Srgrimes if (m->m_flags & M_EXT) { 5361541Srgrimes n->m_data = m->m_data + off; 5371541Srgrimes n->m_ext = m->m_ext; 5381541Srgrimes n->m_flags |= M_EXT; 53964837Sdwmalone MEXT_ADD_REF(m); 540141668Sbmilekic n->m_ext.ref_cnt = m->m_ext.ref_cnt; 5411541Srgrimes } else 5421541Srgrimes bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), 543103569Sbmilekic (u_int)n->m_len); 5441541Srgrimes if (len != M_COPYALL) 5451541Srgrimes len -= n->m_len; 5461541Srgrimes off = 0; 5471541Srgrimes m = m->m_next; 5481541Srgrimes np = &n->m_next; 5491541Srgrimes } 55078592Sbmilekic if (top == NULL) 55178592Sbmilekic mbstat.m_mcfail++; /* XXX: No consistency. */ 55278592Sbmilekic 5531541Srgrimes return (top); 5541541Srgrimesnospace: 5551541Srgrimes m_freem(top); 55678592Sbmilekic mbstat.m_mcfail++; /* XXX: No consistency. */ 55772356Sbmilekic return (NULL); 5581541Srgrimes} 5591541Srgrimes 5601541Srgrimes/* 561149602Sandre * Returns mbuf chain with new head for the prepending case. 562149602Sandre * Copies from mbuf (chain) n from off for len to mbuf (chain) m 563149602Sandre * either prepending or appending the data. 564149602Sandre * The resulting mbuf (chain) m is fully writeable. 565149602Sandre * m is destination (is made writeable) 566149602Sandre * n is source, off is offset in source, len is len from offset 567149602Sandre * dir, 0 append, 1 prepend 568149602Sandre * how, wait or nowait 569149602Sandre */ 570149602Sandre 571149602Sandrestatic int 572149602Sandrem_bcopyxxx(void *s, void *t, u_int len) 573149602Sandre{ 574149602Sandre bcopy(s, t, (size_t)len); 575149602Sandre return 0; 576149602Sandre} 577149602Sandre 578149602Sandrestruct mbuf * 579149602Sandrem_copymdata(struct mbuf *m, struct mbuf *n, int off, int len, 580149602Sandre int prep, int how) 581149602Sandre{ 582149602Sandre struct mbuf *mm, *x, *z; 583149602Sandre caddr_t p; 584149602Sandre int i, mlen, nlen = 0; 585149602Sandre caddr_t buf[MLEN]; 586149602Sandre 587149602Sandre KASSERT(m != NULL && n != NULL, ("m_copymdata, no target or source")); 588149602Sandre KASSERT(off >= 0, ("m_copymdata, negative off %d", off)); 589149602Sandre KASSERT(len >= 0, ("m_copymdata, negative len %d", len)); 590149602Sandre KASSERT(prep == 0 || prep == 1, ("m_copymdata, unknown direction %d", prep)); 591149602Sandre 592149602Sandre /* Make sure environment is sane. */ 593149602Sandre for (z = m; z != NULL; z = z->m_next) { 594149602Sandre mlen += z->m_len; 595149602Sandre if (!M_WRITABLE(z)) { 596149602Sandre /* Make clusters writeable. */ 597149602Sandre if (z->m_flags & M_RDONLY) 598149602Sandre return NULL; /* Can't handle ext ref. */ 599149602Sandre x = m_getcl(how, MT_DATA, 0); 600149602Sandre if (!x) 601149602Sandre return NULL; 602149602Sandre bcopy(z->m_ext.ext_buf, x->m_ext.ext_buf, x->m_ext.ext_size); 603149602Sandre p = x->m_ext.ext_buf + (z->m_data - z->m_ext.ext_buf); 604149602Sandre MEXT_REM_REF(z); /* XXX */ 605149602Sandre z->m_data = p; 606149602Sandre x->m_flags &= ~M_EXT; 607149602Sandre (void)m_free(x); 608149602Sandre } 609149602Sandre } 610149602Sandre mm = prep ? m : z; 611149602Sandre for (z = n; z != NULL; z = z->m_next) 612149602Sandre nlen += z->m_len; 613149602Sandre if (len == M_COPYALL) 614149602Sandre len = nlen - off; 615149602Sandre if (off + len > nlen || len < 1) 616149602Sandre return NULL; 617149602Sandre 618149602Sandre /* 619149602Sandre * Append/prepend the data. Allocating mbufs as necessary. 620149602Sandre */ 621149602Sandre /* Shortcut if enough free space in first/last mbuf. */ 622149602Sandre if (!prep && M_TRAILINGSPACE(mm) >= len) { 623149602Sandre m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t) + 624149602Sandre mm->m_len); 625149602Sandre mm->m_len += len; 626149602Sandre mm->m_pkthdr.len += len; 627149602Sandre return m; 628149602Sandre } 629149602Sandre if (prep && M_LEADINGSPACE(mm) >= len) { 630149602Sandre mm->m_data = mtod(mm, caddr_t) - len; 631149602Sandre m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t)); 632149602Sandre mm->m_len += len; 633149602Sandre mm->m_pkthdr.len += len; 634149602Sandre return mm; 635149602Sandre } 636149602Sandre 637149602Sandre /* Expand first/last mbuf to cluster if possible. */ 638149602Sandre if (!prep && !(mm->m_flags & M_EXT) && len > M_TRAILINGSPACE(mm)) { 639149602Sandre bcopy(mm->m_data, &buf, mm->m_len); 640149602Sandre m_clget(mm, how); 641149602Sandre if (!(mm->m_flags & M_EXT)) 642149602Sandre return NULL; 643149602Sandre bcopy(&buf, mm->m_ext.ext_buf, mm->m_len); 644149602Sandre mm->m_data = mm->m_ext.ext_buf; 645149602Sandre mm->m_pkthdr.header = NULL; 646149602Sandre } 647149602Sandre if (prep && !(mm->m_flags & M_EXT) && len > M_LEADINGSPACE(mm)) { 648149602Sandre bcopy(mm->m_data, &buf, mm->m_len); 649149602Sandre m_clget(mm, how); 650149602Sandre if (!(mm->m_flags & M_EXT)) 651149602Sandre return NULL; 652149602Sandre bcopy(&buf, (caddr_t *)mm->m_ext.ext_buf + 653149602Sandre mm->m_ext.ext_size - mm->m_len, mm->m_len); 654149602Sandre mm->m_data = (caddr_t)mm->m_ext.ext_buf + 655149602Sandre mm->m_ext.ext_size - mm->m_len; 656149602Sandre mm->m_pkthdr.header = NULL; 657149602Sandre } 658149602Sandre 659149602Sandre /* Append/prepend as many mbuf (clusters) as necessary to fit len. */ 660149602Sandre if (!prep && len > M_TRAILINGSPACE(mm)) { 661149602Sandre if (!m_getm(mm, len - M_TRAILINGSPACE(mm), how, MT_DATA)) 662149602Sandre return NULL; 663149602Sandre } 664149602Sandre if (prep && len > M_LEADINGSPACE(mm)) { 665149602Sandre if (!(z = m_getm(NULL, len - M_LEADINGSPACE(mm), how, MT_DATA))) 666149602Sandre return NULL; 667149602Sandre i = 0; 668149602Sandre for (x = z; x != NULL; x = x->m_next) { 669149602Sandre i += x->m_flags & M_EXT ? x->m_ext.ext_size : 670149602Sandre (x->m_flags & M_PKTHDR ? MHLEN : MLEN); 671149602Sandre if (!x->m_next) 672149602Sandre break; 673149602Sandre } 674149602Sandre z->m_data += i - len; 675149602Sandre m_move_pkthdr(mm, z); 676149602Sandre x->m_next = mm; 677149602Sandre mm = z; 678149602Sandre } 679149602Sandre 680149602Sandre /* Seek to start position in source mbuf. Optimization for long chains. */ 681149602Sandre while (off > 0) { 682149602Sandre if (off < n->m_len) 683149602Sandre break; 684149602Sandre off -= n->m_len; 685149602Sandre n = n->m_next; 686149602Sandre } 687149602Sandre 688149602Sandre /* Copy data into target mbuf. */ 689149602Sandre z = mm; 690149602Sandre while (len > 0) { 691149602Sandre KASSERT(z != NULL, ("m_copymdata, falling off target edge")); 692149602Sandre i = M_TRAILINGSPACE(z); 693149602Sandre m_apply(n, off, i, m_bcopyxxx, mtod(z, caddr_t) + z->m_len); 694149602Sandre z->m_len += i; 695149602Sandre /* fixup pkthdr.len if necessary */ 696149602Sandre if ((prep ? mm : m)->m_flags & M_PKTHDR) 697149602Sandre (prep ? mm : m)->m_pkthdr.len += i; 698149602Sandre off += i; 699149602Sandre len -= i; 700149602Sandre z = z->m_next; 701149602Sandre } 702149602Sandre return (prep ? mm : m); 703149602Sandre} 704149602Sandre 705149602Sandre/* 70615689Swollman * Copy an entire packet, including header (which must be present). 70715689Swollman * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'. 70854002Sarchie * Note that the copy is read-only, because clusters are not copied, 70954002Sarchie * only their reference counts are incremented. 71072750Sluigi * Preserve alignment of the first mbuf so if the creator has left 71172750Sluigi * some room at the beginning (e.g. for inserting protocol headers) 71272750Sluigi * the copies still have the room available. 71315689Swollman */ 71415689Swollmanstruct mbuf * 71572356Sbmilekicm_copypacket(struct mbuf *m, int how) 71615689Swollman{ 71715689Swollman struct mbuf *top, *n, *o; 71815689Swollman 719132488Salfred MBUF_CHECKSLEEP(how); 72015689Swollman MGET(n, how, m->m_type); 72115689Swollman top = n; 72272356Sbmilekic if (n == NULL) 72315689Swollman goto nospace; 72415689Swollman 725108466Ssam if (!m_dup_pkthdr(n, m, how)) 726108466Ssam goto nospace; 72715689Swollman n->m_len = m->m_len; 72815689Swollman if (m->m_flags & M_EXT) { 72915689Swollman n->m_data = m->m_data; 73015689Swollman n->m_ext = m->m_ext; 73115689Swollman n->m_flags |= M_EXT; 73264837Sdwmalone MEXT_ADD_REF(m); 733141668Sbmilekic n->m_ext.ref_cnt = m->m_ext.ref_cnt; 73415689Swollman } else { 73572750Sluigi n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat ); 73615689Swollman bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 73715689Swollman } 73815689Swollman 73915689Swollman m = m->m_next; 74015689Swollman while (m) { 74115689Swollman MGET(o, how, m->m_type); 74272356Sbmilekic if (o == NULL) 74315689Swollman goto nospace; 74415689Swollman 74515689Swollman n->m_next = o; 74615689Swollman n = n->m_next; 74715689Swollman 74815689Swollman n->m_len = m->m_len; 74915689Swollman if (m->m_flags & M_EXT) { 75015689Swollman n->m_data = m->m_data; 75115689Swollman n->m_ext = m->m_ext; 75215689Swollman n->m_flags |= M_EXT; 75364837Sdwmalone MEXT_ADD_REF(m); 754141668Sbmilekic n->m_ext.ref_cnt = m->m_ext.ref_cnt; 75515689Swollman } else { 75615689Swollman bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 75715689Swollman } 75815689Swollman 75915689Swollman m = m->m_next; 76015689Swollman } 76115689Swollman return top; 76215689Swollmannospace: 76315689Swollman m_freem(top); 76478592Sbmilekic mbstat.m_mcfail++; /* XXX: No consistency. */ 76572356Sbmilekic return (NULL); 76615689Swollman} 76715689Swollman 76815689Swollman/* 7691541Srgrimes * Copy data from an mbuf chain starting "off" bytes from the beginning, 7701541Srgrimes * continuing for "len" bytes, into the indicated buffer. 7711541Srgrimes */ 7721549Srgrimesvoid 77381907Sjulianm_copydata(const struct mbuf *m, int off, int len, caddr_t cp) 7741541Srgrimes{ 775103569Sbmilekic u_int count; 7761541Srgrimes 77752201Salfred KASSERT(off >= 0, ("m_copydata, negative off %d", off)); 77852201Salfred KASSERT(len >= 0, ("m_copydata, negative len %d", len)); 7791541Srgrimes while (off > 0) { 78052201Salfred KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain")); 7811541Srgrimes if (off < m->m_len) 7821541Srgrimes break; 7831541Srgrimes off -= m->m_len; 7841541Srgrimes m = m->m_next; 7851541Srgrimes } 7861541Srgrimes while (len > 0) { 78752201Salfred KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain")); 7881541Srgrimes count = min(m->m_len - off, len); 7891541Srgrimes bcopy(mtod(m, caddr_t) + off, cp, count); 7901541Srgrimes len -= count; 7911541Srgrimes cp += count; 7921541Srgrimes off = 0; 7931541Srgrimes m = m->m_next; 7941541Srgrimes } 7951541Srgrimes} 7961541Srgrimes 7971541Srgrimes/* 79854002Sarchie * Copy a packet header mbuf chain into a completely new chain, including 79954002Sarchie * copying any mbuf clusters. Use this instead of m_copypacket() when 80054002Sarchie * you need a writable copy of an mbuf chain. 80154002Sarchie */ 80254002Sarchiestruct mbuf * 80372356Sbmilekicm_dup(struct mbuf *m, int how) 80454002Sarchie{ 80554002Sarchie struct mbuf **p, *top = NULL; 80654002Sarchie int remain, moff, nsize; 80754002Sarchie 808132488Salfred MBUF_CHECKSLEEP(how); 80954002Sarchie /* Sanity check */ 81054002Sarchie if (m == NULL) 81172356Sbmilekic return (NULL); 812113255Sdes M_ASSERTPKTHDR(m); 81354002Sarchie 81454002Sarchie /* While there's more data, get a new mbuf, tack it on, and fill it */ 81554002Sarchie remain = m->m_pkthdr.len; 81654002Sarchie moff = 0; 81754002Sarchie p = ⊤ 81854002Sarchie while (remain > 0 || top == NULL) { /* allow m->m_pkthdr.len == 0 */ 81954002Sarchie struct mbuf *n; 82054002Sarchie 82154002Sarchie /* Get the next new mbuf */ 822129906Sbmilekic if (remain >= MINCLSIZE) { 823129906Sbmilekic n = m_getcl(how, m->m_type, 0); 824129906Sbmilekic nsize = MCLBYTES; 825129906Sbmilekic } else { 826129906Sbmilekic n = m_get(how, m->m_type); 827129906Sbmilekic nsize = MLEN; 828129906Sbmilekic } 82954002Sarchie if (n == NULL) 83054002Sarchie goto nospace; 831129906Sbmilekic 832129906Sbmilekic if (top == NULL) { /* First one, must be PKTHDR */ 833129906Sbmilekic if (!m_dup_pkthdr(n, m, how)) { 834129906Sbmilekic m_free(n); 835108466Ssam goto nospace; 836129906Sbmilekic } 83754002Sarchie nsize = MHLEN; 83854002Sarchie } 83954002Sarchie n->m_len = 0; 84054002Sarchie 84154002Sarchie /* Link it into the new chain */ 84254002Sarchie *p = n; 84354002Sarchie p = &n->m_next; 84454002Sarchie 84554002Sarchie /* Copy data from original mbuf(s) into new mbuf */ 84654002Sarchie while (n->m_len < nsize && m != NULL) { 84754002Sarchie int chunk = min(nsize - n->m_len, m->m_len - moff); 84854002Sarchie 84954002Sarchie bcopy(m->m_data + moff, n->m_data + n->m_len, chunk); 85054002Sarchie moff += chunk; 85154002Sarchie n->m_len += chunk; 85254002Sarchie remain -= chunk; 85354002Sarchie if (moff == m->m_len) { 85454002Sarchie m = m->m_next; 85554002Sarchie moff = 0; 85654002Sarchie } 85754002Sarchie } 85854002Sarchie 85954002Sarchie /* Check correct total mbuf length */ 86054002Sarchie KASSERT((remain > 0 && m != NULL) || (remain == 0 && m == NULL), 86187594Sobrien ("%s: bogus m_pkthdr.len", __func__)); 86254002Sarchie } 86354002Sarchie return (top); 86454002Sarchie 86554002Sarchienospace: 86654002Sarchie m_freem(top); 86778592Sbmilekic mbstat.m_mcfail++; /* XXX: No consistency. */ 86872356Sbmilekic return (NULL); 86954002Sarchie} 87054002Sarchie 87154002Sarchie/* 8721541Srgrimes * Concatenate mbuf chain n to m. 8731541Srgrimes * Both chains must be of the same type (e.g. MT_DATA). 8741541Srgrimes * Any m_pkthdr is not updated. 8751541Srgrimes */ 8761549Srgrimesvoid 87772356Sbmilekicm_cat(struct mbuf *m, struct mbuf *n) 8781541Srgrimes{ 8791541Srgrimes while (m->m_next) 8801541Srgrimes m = m->m_next; 8811541Srgrimes while (n) { 8821541Srgrimes if (m->m_flags & M_EXT || 8831541Srgrimes m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { 8841541Srgrimes /* just join the two chains */ 8851541Srgrimes m->m_next = n; 8861541Srgrimes return; 8871541Srgrimes } 8881541Srgrimes /* splat the data from one into the other */ 8891541Srgrimes bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 8901541Srgrimes (u_int)n->m_len); 8911541Srgrimes m->m_len += n->m_len; 8921541Srgrimes n = m_free(n); 8931541Srgrimes } 8941541Srgrimes} 8951541Srgrimes 8961549Srgrimesvoid 89772356Sbmilekicm_adj(struct mbuf *mp, int req_len) 8981541Srgrimes{ 89972356Sbmilekic int len = req_len; 90072356Sbmilekic struct mbuf *m; 90172356Sbmilekic int count; 9021541Srgrimes 9031541Srgrimes if ((m = mp) == NULL) 9041541Srgrimes return; 9051541Srgrimes if (len >= 0) { 9061541Srgrimes /* 9071541Srgrimes * Trim from head. 9081541Srgrimes */ 9091541Srgrimes while (m != NULL && len > 0) { 9101541Srgrimes if (m->m_len <= len) { 9111541Srgrimes len -= m->m_len; 9121541Srgrimes m->m_len = 0; 9131541Srgrimes m = m->m_next; 9141541Srgrimes } else { 9151541Srgrimes m->m_len -= len; 9161541Srgrimes m->m_data += len; 9171541Srgrimes len = 0; 9181541Srgrimes } 9191541Srgrimes } 9201541Srgrimes m = mp; 9211541Srgrimes if (mp->m_flags & M_PKTHDR) 9221541Srgrimes m->m_pkthdr.len -= (req_len - len); 9231541Srgrimes } else { 9241541Srgrimes /* 9251541Srgrimes * Trim from tail. Scan the mbuf chain, 9261541Srgrimes * calculating its length and finding the last mbuf. 9271541Srgrimes * If the adjustment only affects this mbuf, then just 9281541Srgrimes * adjust and return. Otherwise, rescan and truncate 9291541Srgrimes * after the remaining size. 9301541Srgrimes */ 9311541Srgrimes len = -len; 9321541Srgrimes count = 0; 9331541Srgrimes for (;;) { 9341541Srgrimes count += m->m_len; 9351541Srgrimes if (m->m_next == (struct mbuf *)0) 9361541Srgrimes break; 9371541Srgrimes m = m->m_next; 9381541Srgrimes } 9391541Srgrimes if (m->m_len >= len) { 9401541Srgrimes m->m_len -= len; 9411541Srgrimes if (mp->m_flags & M_PKTHDR) 9421541Srgrimes mp->m_pkthdr.len -= len; 9431541Srgrimes return; 9441541Srgrimes } 9451541Srgrimes count -= len; 9461541Srgrimes if (count < 0) 9471541Srgrimes count = 0; 9481541Srgrimes /* 9491541Srgrimes * Correct length for chain is "count". 9501541Srgrimes * Find the mbuf with last data, adjust its length, 9511541Srgrimes * and toss data from remaining mbufs on chain. 9521541Srgrimes */ 9531541Srgrimes m = mp; 9541541Srgrimes if (m->m_flags & M_PKTHDR) 9551541Srgrimes m->m_pkthdr.len = count; 9561541Srgrimes for (; m; m = m->m_next) { 9571541Srgrimes if (m->m_len >= count) { 9581541Srgrimes m->m_len = count; 959142350Ssam if (m->m_next != NULL) { 960142350Ssam m_freem(m->m_next); 961142350Ssam m->m_next = NULL; 962142350Ssam } 9631541Srgrimes break; 9641541Srgrimes } 9651541Srgrimes count -= m->m_len; 9661541Srgrimes } 9671541Srgrimes } 9681541Srgrimes} 9691541Srgrimes 9701541Srgrimes/* 9711541Srgrimes * Rearange an mbuf chain so that len bytes are contiguous 9721541Srgrimes * and in the data area of an mbuf (so that mtod and dtom 9731541Srgrimes * will work for a structure of size len). Returns the resulting 9741541Srgrimes * mbuf chain on success, frees it and returns null on failure. 9751541Srgrimes * If there is room, it will add up to max_protohdr-len extra bytes to the 9761541Srgrimes * contiguous region in an attempt to avoid being called next time. 9771541Srgrimes */ 9781541Srgrimesstruct mbuf * 97972356Sbmilekicm_pullup(struct mbuf *n, int len) 9801541Srgrimes{ 98172356Sbmilekic struct mbuf *m; 98272356Sbmilekic int count; 9831541Srgrimes int space; 9841541Srgrimes 9851541Srgrimes /* 9861541Srgrimes * If first mbuf has no cluster, and has room for len bytes 9871541Srgrimes * without shifting current data, pullup into it, 9881541Srgrimes * otherwise allocate a new mbuf to prepend to the chain. 9891541Srgrimes */ 9901541Srgrimes if ((n->m_flags & M_EXT) == 0 && 9911541Srgrimes n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 9921541Srgrimes if (n->m_len >= len) 9931541Srgrimes return (n); 9941541Srgrimes m = n; 9951541Srgrimes n = n->m_next; 9961541Srgrimes len -= m->m_len; 9971541Srgrimes } else { 9981541Srgrimes if (len > MHLEN) 9991541Srgrimes goto bad; 1000111119Simp MGET(m, M_DONTWAIT, n->m_type); 100172356Sbmilekic if (m == NULL) 10021541Srgrimes goto bad; 10031541Srgrimes m->m_len = 0; 1004108466Ssam if (n->m_flags & M_PKTHDR) 1005108466Ssam M_MOVE_PKTHDR(m, n); 10061541Srgrimes } 10071541Srgrimes space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 10081541Srgrimes do { 10091541Srgrimes count = min(min(max(len, max_protohdr), space), n->m_len); 10101541Srgrimes bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 1011103569Sbmilekic (u_int)count); 10121541Srgrimes len -= count; 10131541Srgrimes m->m_len += count; 10141541Srgrimes n->m_len -= count; 10151541Srgrimes space -= count; 10161541Srgrimes if (n->m_len) 10171541Srgrimes n->m_data += count; 10181541Srgrimes else 10191541Srgrimes n = m_free(n); 10201541Srgrimes } while (len > 0 && n); 10211541Srgrimes if (len > 0) { 10221541Srgrimes (void) m_free(m); 10231541Srgrimes goto bad; 10241541Srgrimes } 10251541Srgrimes m->m_next = n; 10261541Srgrimes return (m); 10271541Srgrimesbad: 10281541Srgrimes m_freem(n); 102978592Sbmilekic mbstat.m_mpfail++; /* XXX: No consistency. */ 103072356Sbmilekic return (NULL); 10311541Srgrimes} 10321541Srgrimes 10331541Srgrimes/* 1034143761Sjmg * Like m_pullup(), except a new mbuf is always allocated, and we allow 1035143761Sjmg * the amount of empty space before the data in the new mbuf to be specified 1036143761Sjmg * (in the event that the caller expects to prepend later). 1037143761Sjmg */ 1038143761Sjmgint MSFail; 1039143761Sjmg 1040143761Sjmgstruct mbuf * 1041143761Sjmgm_copyup(struct mbuf *n, int len, int dstoff) 1042143761Sjmg{ 1043143761Sjmg struct mbuf *m; 1044143761Sjmg int count, space; 1045143761Sjmg 1046143761Sjmg if (len > (MHLEN - dstoff)) 1047143761Sjmg goto bad; 1048143761Sjmg MGET(m, M_DONTWAIT, n->m_type); 1049143761Sjmg if (m == NULL) 1050143761Sjmg goto bad; 1051143761Sjmg m->m_len = 0; 1052143761Sjmg if (n->m_flags & M_PKTHDR) 1053143761Sjmg M_MOVE_PKTHDR(m, n); 1054143761Sjmg m->m_data += dstoff; 1055143761Sjmg space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 1056143761Sjmg do { 1057143761Sjmg count = min(min(max(len, max_protohdr), space), n->m_len); 1058143761Sjmg memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), 1059143761Sjmg (unsigned)count); 1060143761Sjmg len -= count; 1061143761Sjmg m->m_len += count; 1062143761Sjmg n->m_len -= count; 1063143761Sjmg space -= count; 1064143761Sjmg if (n->m_len) 1065143761Sjmg n->m_data += count; 1066143761Sjmg else 1067143761Sjmg n = m_free(n); 1068143761Sjmg } while (len > 0 && n); 1069143761Sjmg if (len > 0) { 1070143761Sjmg (void) m_free(m); 1071143761Sjmg goto bad; 1072143761Sjmg } 1073143761Sjmg m->m_next = n; 1074143761Sjmg return (m); 1075143761Sjmg bad: 1076143761Sjmg m_freem(n); 1077143761Sjmg MSFail++; 1078143761Sjmg return (NULL); 1079143761Sjmg} 1080143761Sjmg 1081143761Sjmg/* 10821541Srgrimes * Partition an mbuf chain in two pieces, returning the tail -- 10831541Srgrimes * all but the first len0 bytes. In case of failure, it returns NULL and 10841541Srgrimes * attempts to restore the chain to its original state. 108597681Sarchie * 108697681Sarchie * Note that the resulting mbufs might be read-only, because the new 108797681Sarchie * mbuf can end up sharing an mbuf cluster with the original mbuf if 108897681Sarchie * the "breaking point" happens to lie within a cluster mbuf. Use the 108997681Sarchie * M_WRITABLE() macro to check for this case. 10901541Srgrimes */ 10911541Srgrimesstruct mbuf * 109272356Sbmilekicm_split(struct mbuf *m0, int len0, int wait) 10931541Srgrimes{ 109472356Sbmilekic struct mbuf *m, *n; 1095103569Sbmilekic u_int len = len0, remain; 10961541Srgrimes 1097132488Salfred MBUF_CHECKSLEEP(wait); 10981541Srgrimes for (m = m0; m && len > m->m_len; m = m->m_next) 10991541Srgrimes len -= m->m_len; 110072356Sbmilekic if (m == NULL) 110172356Sbmilekic return (NULL); 11021541Srgrimes remain = m->m_len - len; 11031541Srgrimes if (m0->m_flags & M_PKTHDR) { 11041541Srgrimes MGETHDR(n, wait, m0->m_type); 110572356Sbmilekic if (n == NULL) 110672356Sbmilekic return (NULL); 11071541Srgrimes n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 11081541Srgrimes n->m_pkthdr.len = m0->m_pkthdr.len - len0; 11091541Srgrimes m0->m_pkthdr.len = len0; 11101541Srgrimes if (m->m_flags & M_EXT) 11111541Srgrimes goto extpacket; 11121541Srgrimes if (remain > MHLEN) { 11131541Srgrimes /* m can't be the lead packet */ 11141541Srgrimes MH_ALIGN(n, 0); 11151541Srgrimes n->m_next = m_split(m, len, wait); 111672356Sbmilekic if (n->m_next == NULL) { 11171541Srgrimes (void) m_free(n); 111872356Sbmilekic return (NULL); 111994471Shsu } else { 112094471Shsu n->m_len = 0; 11211541Srgrimes return (n); 112294471Shsu } 11231541Srgrimes } else 11241541Srgrimes MH_ALIGN(n, remain); 11251541Srgrimes } else if (remain == 0) { 11261541Srgrimes n = m->m_next; 112772356Sbmilekic m->m_next = NULL; 11281541Srgrimes return (n); 11291541Srgrimes } else { 11301541Srgrimes MGET(n, wait, m->m_type); 113172356Sbmilekic if (n == NULL) 113272356Sbmilekic return (NULL); 11331541Srgrimes M_ALIGN(n, remain); 11341541Srgrimes } 11351541Srgrimesextpacket: 11361541Srgrimes if (m->m_flags & M_EXT) { 11371541Srgrimes n->m_flags |= M_EXT; 11381541Srgrimes n->m_ext = m->m_ext; 113964837Sdwmalone MEXT_ADD_REF(m); 1140141668Sbmilekic n->m_ext.ref_cnt = m->m_ext.ref_cnt; 11411541Srgrimes n->m_data = m->m_data + len; 11421541Srgrimes } else { 11431541Srgrimes bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); 11441541Srgrimes } 11451541Srgrimes n->m_len = remain; 11461541Srgrimes m->m_len = len; 11471541Srgrimes n->m_next = m->m_next; 114872356Sbmilekic m->m_next = NULL; 11491541Srgrimes return (n); 11501541Srgrimes} 11511541Srgrimes/* 11521541Srgrimes * Routine to copy from device local memory into mbufs. 115378508Sbmilekic * Note that `off' argument is offset into first mbuf of target chain from 115478508Sbmilekic * which to begin copying the data to. 11551541Srgrimes */ 11561541Srgrimesstruct mbuf * 115778508Sbmilekicm_devget(char *buf, int totlen, int off, struct ifnet *ifp, 115872356Sbmilekic void (*copy)(char *from, caddr_t to, u_int len)) 11591541Srgrimes{ 116072356Sbmilekic struct mbuf *m; 1161129906Sbmilekic struct mbuf *top = NULL, **mp = ⊤ 116278508Sbmilekic int len; 11631541Srgrimes 116478508Sbmilekic if (off < 0 || off > MHLEN) 116578508Sbmilekic return (NULL); 116678508Sbmilekic 1167129906Sbmilekic while (totlen > 0) { 1168129906Sbmilekic if (top == NULL) { /* First one, must be PKTHDR */ 1169129906Sbmilekic if (totlen + off >= MINCLSIZE) { 1170129906Sbmilekic m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); 1171129906Sbmilekic len = MCLBYTES; 1172129906Sbmilekic } else { 1173129906Sbmilekic m = m_gethdr(M_DONTWAIT, MT_DATA); 1174129906Sbmilekic len = MHLEN; 11751541Srgrimes 1176129906Sbmilekic /* Place initial small packet/header at end of mbuf */ 1177129906Sbmilekic if (m && totlen + off + max_linkhdr <= MLEN) { 1178129906Sbmilekic m->m_data += max_linkhdr; 1179129906Sbmilekic len -= max_linkhdr; 1180129906Sbmilekic } 1181129906Sbmilekic } 1182129906Sbmilekic if (m == NULL) 1183129906Sbmilekic return NULL; 1184129906Sbmilekic m->m_pkthdr.rcvif = ifp; 1185129906Sbmilekic m->m_pkthdr.len = totlen; 1186129906Sbmilekic } else { 1187129906Sbmilekic if (totlen + off >= MINCLSIZE) { 1188129906Sbmilekic m = m_getcl(M_DONTWAIT, MT_DATA, 0); 1189129906Sbmilekic len = MCLBYTES; 1190129906Sbmilekic } else { 1191129906Sbmilekic m = m_get(M_DONTWAIT, MT_DATA); 1192129906Sbmilekic len = MLEN; 1193129906Sbmilekic } 119472356Sbmilekic if (m == NULL) { 11951541Srgrimes m_freem(top); 1196129906Sbmilekic return NULL; 11971541Srgrimes } 11981541Srgrimes } 119978508Sbmilekic if (off) { 120078508Sbmilekic m->m_data += off; 120178508Sbmilekic len -= off; 120278508Sbmilekic off = 0; 120378508Sbmilekic } 120478508Sbmilekic m->m_len = len = min(totlen, len); 12051541Srgrimes if (copy) 1206103569Sbmilekic copy(buf, mtod(m, caddr_t), (u_int)len); 12071541Srgrimes else 1208103569Sbmilekic bcopy(buf, mtod(m, caddr_t), (u_int)len); 120978508Sbmilekic buf += len; 12101541Srgrimes *mp = m; 12111541Srgrimes mp = &m->m_next; 12121541Srgrimes totlen -= len; 12131541Srgrimes } 12141541Srgrimes return (top); 12151541Srgrimes} 12163352Sphk 12173352Sphk/* 12183352Sphk * Copy data from a buffer back into the indicated mbuf chain, 12193352Sphk * starting "off" bytes from the beginning, extending the mbuf 12203352Sphk * chain if necessary. 12213352Sphk */ 12223352Sphkvoid 1223128402Sluigim_copyback(struct mbuf *m0, int off, int len, c_caddr_t cp) 12243352Sphk{ 122572356Sbmilekic int mlen; 122672356Sbmilekic struct mbuf *m = m0, *n; 12273352Sphk int totlen = 0; 12283352Sphk 122972356Sbmilekic if (m0 == NULL) 12303352Sphk return; 12313352Sphk while (off > (mlen = m->m_len)) { 12323352Sphk off -= mlen; 12333352Sphk totlen += mlen; 123472356Sbmilekic if (m->m_next == NULL) { 1235129906Sbmilekic n = m_get(M_DONTWAIT, m->m_type); 123672356Sbmilekic if (n == NULL) 12373352Sphk goto out; 1238129906Sbmilekic bzero(mtod(n, caddr_t), MLEN); 12393352Sphk n->m_len = min(MLEN, len + off); 12403352Sphk m->m_next = n; 12413352Sphk } 12423352Sphk m = m->m_next; 12433352Sphk } 12443352Sphk while (len > 0) { 12453352Sphk mlen = min (m->m_len - off, len); 1246103569Sbmilekic bcopy(cp, off + mtod(m, caddr_t), (u_int)mlen); 12473352Sphk cp += mlen; 12483352Sphk len -= mlen; 12493352Sphk mlen += off; 12503352Sphk off = 0; 12513352Sphk totlen += mlen; 12523352Sphk if (len == 0) 12533352Sphk break; 125472356Sbmilekic if (m->m_next == NULL) { 1255111119Simp n = m_get(M_DONTWAIT, m->m_type); 125672356Sbmilekic if (n == NULL) 12573352Sphk break; 12583352Sphk n->m_len = min(MLEN, len); 12593352Sphk m->m_next = n; 12603352Sphk } 12613352Sphk m = m->m_next; 12623352Sphk } 12633352Sphkout: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) 12643352Sphk m->m_pkthdr.len = totlen; 12653352Sphk} 126652756Sphk 1267123557Sbms/* 1268138541Ssam * Append the specified data to the indicated mbuf chain, 1269138541Ssam * Extend the mbuf chain if the new data does not fit in 1270138541Ssam * existing space. 1271138541Ssam * 1272138541Ssam * Return 1 if able to complete the job; otherwise 0. 1273138541Ssam */ 1274138541Ssamint 1275138541Ssamm_append(struct mbuf *m0, int len, c_caddr_t cp) 1276138541Ssam{ 1277138541Ssam struct mbuf *m, *n; 1278138541Ssam int remainder, space; 1279138541Ssam 1280138541Ssam for (m = m0; m->m_next != NULL; m = m->m_next) 1281138541Ssam ; 1282138541Ssam remainder = len; 1283138541Ssam space = M_TRAILINGSPACE(m); 1284138541Ssam if (space > 0) { 1285138541Ssam /* 1286138541Ssam * Copy into available space. 1287138541Ssam */ 1288138541Ssam if (space > remainder) 1289138541Ssam space = remainder; 1290138541Ssam bcopy(cp, mtod(m, caddr_t) + m->m_len, space); 1291138541Ssam m->m_len += space; 1292138541Ssam cp += space, remainder -= space; 1293138541Ssam } 1294138541Ssam while (remainder > 0) { 1295138541Ssam /* 1296138541Ssam * Allocate a new mbuf; could check space 1297138541Ssam * and allocate a cluster instead. 1298138541Ssam */ 1299138541Ssam n = m_get(M_DONTWAIT, m->m_type); 1300138541Ssam if (n == NULL) 1301138541Ssam break; 1302138541Ssam n->m_len = min(MLEN, remainder); 1303138894Ssam bcopy(cp, mtod(n, caddr_t), n->m_len); 1304138894Ssam cp += n->m_len, remainder -= n->m_len; 1305138541Ssam m->m_next = n; 1306138541Ssam m = n; 1307138541Ssam } 1308138541Ssam if (m0->m_flags & M_PKTHDR) 1309138541Ssam m0->m_pkthdr.len += len - remainder; 1310138541Ssam return (remainder == 0); 1311138541Ssam} 1312138541Ssam 1313138541Ssam/* 1314123557Sbms * Apply function f to the data in an mbuf chain starting "off" bytes from 1315123557Sbms * the beginning, continuing for "len" bytes. 1316123557Sbms */ 1317123557Sbmsint 1318123557Sbmsm_apply(struct mbuf *m, int off, int len, 1319123564Sbms int (*f)(void *, void *, u_int), void *arg) 1320123557Sbms{ 1321123564Sbms u_int count; 1322123557Sbms int rval; 1323123557Sbms 1324123557Sbms KASSERT(off >= 0, ("m_apply, negative off %d", off)); 1325123557Sbms KASSERT(len >= 0, ("m_apply, negative len %d", len)); 1326123557Sbms while (off > 0) { 1327123557Sbms KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain")); 1328123557Sbms if (off < m->m_len) 1329123557Sbms break; 1330123557Sbms off -= m->m_len; 1331123557Sbms m = m->m_next; 1332123557Sbms } 1333123557Sbms while (len > 0) { 1334123557Sbms KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain")); 1335123557Sbms count = min(m->m_len - off, len); 1336123557Sbms rval = (*f)(arg, mtod(m, caddr_t) + off, count); 1337123557Sbms if (rval) 1338123557Sbms return (rval); 1339123557Sbms len -= count; 1340123557Sbms off = 0; 1341123557Sbms m = m->m_next; 1342123557Sbms } 1343123557Sbms return (0); 1344123557Sbms} 1345123557Sbms 1346123557Sbms/* 1347123557Sbms * Return a pointer to mbuf/offset of location in mbuf chain. 1348123557Sbms */ 1349123557Sbmsstruct mbuf * 1350123557Sbmsm_getptr(struct mbuf *m, int loc, int *off) 1351123557Sbms{ 1352123557Sbms 1353123557Sbms while (loc >= 0) { 1354123564Sbms /* Normal end of search. */ 1355123557Sbms if (m->m_len > loc) { 1356123557Sbms *off = loc; 1357123557Sbms return (m); 1358123557Sbms } else { 1359123557Sbms loc -= m->m_len; 1360123557Sbms if (m->m_next == NULL) { 1361123557Sbms if (loc == 0) { 1362123564Sbms /* Point at the end of valid data. */ 1363123557Sbms *off = m->m_len; 1364123557Sbms return (m); 1365123564Sbms } 1366123564Sbms return (NULL); 1367123564Sbms } 1368123564Sbms m = m->m_next; 1369123557Sbms } 1370123557Sbms } 1371123557Sbms return (NULL); 1372123557Sbms} 1373123557Sbms 137452756Sphkvoid 1375135904Sjmgm_print(const struct mbuf *m, int maxlen) 137652756Sphk{ 137752756Sphk int len; 1378135904Sjmg int pdata; 137954906Seivind const struct mbuf *m2; 138052756Sphk 1381135904Sjmg if (m->m_flags & M_PKTHDR) 1382135904Sjmg len = m->m_pkthdr.len; 1383135904Sjmg else 1384135904Sjmg len = -1; 138552756Sphk m2 = m; 1386135904Sjmg while (m2 != NULL && (len == -1 || len)) { 1387135904Sjmg pdata = m2->m_len; 1388135904Sjmg if (maxlen != -1 && pdata > maxlen) 1389135904Sjmg pdata = maxlen; 1390135904Sjmg printf("mbuf: %p len: %d, next: %p, %b%s", m2, m2->m_len, 1391135904Sjmg m2->m_next, m2->m_flags, "\20\20freelist\17skipfw" 1392135904Sjmg "\11proto5\10proto4\7proto3\6proto2\5proto1\4rdonly" 1393135904Sjmg "\3eor\2pkthdr\1ext", pdata ? "" : "\n"); 1394135904Sjmg if (pdata) 1395135904Sjmg printf(", %*D\n", m2->m_len, (u_char *)m2->m_data, "-"); 1396135904Sjmg if (len != -1) 1397135904Sjmg len -= m2->m_len; 139852756Sphk m2 = m2->m_next; 139952756Sphk } 1400135904Sjmg if (len > 0) 1401135904Sjmg printf("%d bytes unaccounted for.\n", len); 140252756Sphk return; 140352756Sphk} 1404103540Sphk 1405103569Sbmilekicu_int 1406103540Sphkm_fixhdr(struct mbuf *m0) 1407103540Sphk{ 1408103569Sbmilekic u_int len; 1409103540Sphk 1410103544Sphk len = m_length(m0, NULL); 1411103544Sphk m0->m_pkthdr.len = len; 1412103544Sphk return (len); 1413103544Sphk} 1414103544Sphk 1415103569Sbmilekicu_int 1416103544Sphkm_length(struct mbuf *m0, struct mbuf **last) 1417103544Sphk{ 1418103544Sphk struct mbuf *m; 1419103569Sbmilekic u_int len; 1420103544Sphk 1421103544Sphk len = 0; 1422103544Sphk for (m = m0; m != NULL; m = m->m_next) { 1423103540Sphk len += m->m_len; 1424103544Sphk if (m->m_next == NULL) 1425103544Sphk break; 1426103540Sphk } 1427103544Sphk if (last != NULL) 1428103544Sphk *last = m; 1429103544Sphk return (len); 1430103540Sphk} 1431112777Ssilby 1432112777Ssilby/* 1433112777Ssilby * Defragment a mbuf chain, returning the shortest possible 1434112777Ssilby * chain of mbufs and clusters. If allocation fails and 1435112777Ssilby * this cannot be completed, NULL will be returned, but 1436112777Ssilby * the passed in chain will be unchanged. Upon success, 1437112777Ssilby * the original chain will be freed, and the new chain 1438112777Ssilby * will be returned. 1439112777Ssilby * 1440112777Ssilby * If a non-packet header is passed in, the original 1441112777Ssilby * mbuf (chain?) will be returned unharmed. 1442112777Ssilby */ 1443112777Ssilbystruct mbuf * 1444112777Ssilbym_defrag(struct mbuf *m0, int how) 1445112777Ssilby{ 1446125472Ssilby struct mbuf *m_new = NULL, *m_final = NULL; 1447125472Ssilby int progress = 0, length; 1448112777Ssilby 1449132488Salfred MBUF_CHECKSLEEP(how); 1450112777Ssilby if (!(m0->m_flags & M_PKTHDR)) 1451112777Ssilby return (m0); 1452112777Ssilby 1453117770Ssilby m_fixhdr(m0); /* Needed sanity check */ 1454117770Ssilby 1455113490Ssilby#ifdef MBUF_STRESS_TEST 1456113490Ssilby if (m_defragrandomfailures) { 1457113490Ssilby int temp = arc4random() & 0xff; 1458113490Ssilby if (temp == 0xba) 1459113490Ssilby goto nospace; 1460113490Ssilby } 1461113490Ssilby#endif 1462112777Ssilby 1463112777Ssilby if (m0->m_pkthdr.len > MHLEN) 1464112777Ssilby m_final = m_getcl(how, MT_DATA, M_PKTHDR); 1465112777Ssilby else 1466112777Ssilby m_final = m_gethdr(how, MT_DATA); 1467112777Ssilby 1468112777Ssilby if (m_final == NULL) 1469112777Ssilby goto nospace; 1470112777Ssilby 1471123740Speter if (m_dup_pkthdr(m_final, m0, how) == 0) 1472112777Ssilby goto nospace; 1473112777Ssilby 1474112777Ssilby m_new = m_final; 1475112777Ssilby 1476112777Ssilby while (progress < m0->m_pkthdr.len) { 1477112777Ssilby length = m0->m_pkthdr.len - progress; 1478112777Ssilby if (length > MCLBYTES) 1479112777Ssilby length = MCLBYTES; 1480112777Ssilby 1481112777Ssilby if (m_new == NULL) { 1482112777Ssilby if (length > MLEN) 1483112777Ssilby m_new = m_getcl(how, MT_DATA, 0); 1484112777Ssilby else 1485112777Ssilby m_new = m_get(how, MT_DATA); 1486112777Ssilby if (m_new == NULL) 1487112777Ssilby goto nospace; 1488112777Ssilby } 1489112777Ssilby 1490112777Ssilby m_copydata(m0, progress, length, mtod(m_new, caddr_t)); 1491112777Ssilby progress += length; 1492112777Ssilby m_new->m_len = length; 1493112777Ssilby if (m_new != m_final) 1494112777Ssilby m_cat(m_final, m_new); 1495112777Ssilby m_new = NULL; 1496112777Ssilby } 1497116455Ssilby#ifdef MBUF_STRESS_TEST 1498112777Ssilby if (m0->m_next == NULL) 1499112777Ssilby m_defraguseless++; 1500116455Ssilby#endif 1501112777Ssilby m_freem(m0); 1502112777Ssilby m0 = m_final; 1503116455Ssilby#ifdef MBUF_STRESS_TEST 1504112777Ssilby m_defragpackets++; 1505112777Ssilby m_defragbytes += m0->m_pkthdr.len; 1506116455Ssilby#endif 1507112777Ssilby return (m0); 1508112777Ssilbynospace: 1509116455Ssilby#ifdef MBUF_STRESS_TEST 1510112777Ssilby m_defragfailure++; 1511116455Ssilby#endif 1512112777Ssilby if (m_final) 1513112777Ssilby m_freem(m_final); 1514112777Ssilby return (NULL); 1515112777Ssilby} 1516119644Ssilby 1517119644Ssilby#ifdef MBUF_STRESS_TEST 1518119644Ssilby 1519119644Ssilby/* 1520119644Ssilby * Fragment an mbuf chain. There's no reason you'd ever want to do 1521119644Ssilby * this in normal usage, but it's great for stress testing various 1522119644Ssilby * mbuf consumers. 1523119644Ssilby * 1524119644Ssilby * If fragmentation is not possible, the original chain will be 1525119644Ssilby * returned. 1526119644Ssilby * 1527119644Ssilby * Possible length values: 1528119644Ssilby * 0 no fragmentation will occur 1529119644Ssilby * > 0 each fragment will be of the specified length 1530119644Ssilby * -1 each fragment will be the same random value in length 1531119644Ssilby * -2 each fragment's length will be entirely random 1532119644Ssilby * (Random values range from 1 to 256) 1533119644Ssilby */ 1534119644Ssilbystruct mbuf * 1535119644Ssilbym_fragment(struct mbuf *m0, int how, int length) 1536119644Ssilby{ 1537125472Ssilby struct mbuf *m_new = NULL, *m_final = NULL; 1538125472Ssilby int progress = 0; 1539119644Ssilby 1540119644Ssilby if (!(m0->m_flags & M_PKTHDR)) 1541119644Ssilby return (m0); 1542119644Ssilby 1543119644Ssilby if ((length == 0) || (length < -2)) 1544119644Ssilby return (m0); 1545119644Ssilby 1546119644Ssilby m_fixhdr(m0); /* Needed sanity check */ 1547119644Ssilby 1548119644Ssilby m_final = m_getcl(how, MT_DATA, M_PKTHDR); 1549119644Ssilby 1550119644Ssilby if (m_final == NULL) 1551119644Ssilby goto nospace; 1552119644Ssilby 1553123823Ssilby if (m_dup_pkthdr(m_final, m0, how) == 0) 1554119644Ssilby goto nospace; 1555119644Ssilby 1556119644Ssilby m_new = m_final; 1557119644Ssilby 1558119644Ssilby if (length == -1) 1559119644Ssilby length = 1 + (arc4random() & 255); 1560119644Ssilby 1561119644Ssilby while (progress < m0->m_pkthdr.len) { 1562119644Ssilby int fraglen; 1563119644Ssilby 1564119644Ssilby if (length > 0) 1565119644Ssilby fraglen = length; 1566119644Ssilby else 1567119644Ssilby fraglen = 1 + (arc4random() & 255); 1568119644Ssilby if (fraglen > m0->m_pkthdr.len - progress) 1569119644Ssilby fraglen = m0->m_pkthdr.len - progress; 1570119644Ssilby 1571119644Ssilby if (fraglen > MCLBYTES) 1572119644Ssilby fraglen = MCLBYTES; 1573119644Ssilby 1574119644Ssilby if (m_new == NULL) { 1575119644Ssilby m_new = m_getcl(how, MT_DATA, 0); 1576119644Ssilby if (m_new == NULL) 1577119644Ssilby goto nospace; 1578119644Ssilby } 1579119644Ssilby 1580119644Ssilby m_copydata(m0, progress, fraglen, mtod(m_new, caddr_t)); 1581119644Ssilby progress += fraglen; 1582119644Ssilby m_new->m_len = fraglen; 1583119644Ssilby if (m_new != m_final) 1584119644Ssilby m_cat(m_final, m_new); 1585119644Ssilby m_new = NULL; 1586119644Ssilby } 1587119644Ssilby m_freem(m0); 1588119644Ssilby m0 = m_final; 1589119644Ssilby return (m0); 1590119644Ssilbynospace: 1591119644Ssilby if (m_final) 1592119644Ssilby m_freem(m_final); 1593119644Ssilby /* Return the original chain on failure */ 1594119644Ssilby return (m0); 1595119644Ssilby} 1596119644Ssilby 1597119644Ssilby#endif 1598125296Ssilby 1599125296Ssilbystruct mbuf * 1600145883Semaxm_uiotombuf(struct uio *uio, int how, int len, int align) 1601125296Ssilby{ 1602125472Ssilby struct mbuf *m_new = NULL, *m_final = NULL; 1603125472Ssilby int progress = 0, error = 0, length, total; 1604125296Ssilby 1605125296Ssilby if (len > 0) 1606125296Ssilby total = min(uio->uio_resid, len); 1607125296Ssilby else 1608125296Ssilby total = uio->uio_resid; 1609145883Semax if (align >= MHLEN) 1610145883Semax goto nospace; 1611145883Semax if (total + align > MHLEN) 1612125296Ssilby m_final = m_getcl(how, MT_DATA, M_PKTHDR); 1613125296Ssilby else 1614125296Ssilby m_final = m_gethdr(how, MT_DATA); 1615125296Ssilby if (m_final == NULL) 1616125296Ssilby goto nospace; 1617145883Semax m_final->m_data += align; 1618125296Ssilby m_new = m_final; 1619125296Ssilby while (progress < total) { 1620125296Ssilby length = total - progress; 1621125296Ssilby if (length > MCLBYTES) 1622125296Ssilby length = MCLBYTES; 1623125296Ssilby if (m_new == NULL) { 1624125296Ssilby if (length > MLEN) 1625125296Ssilby m_new = m_getcl(how, MT_DATA, 0); 1626125296Ssilby else 1627125296Ssilby m_new = m_get(how, MT_DATA); 1628125296Ssilby if (m_new == NULL) 1629125296Ssilby goto nospace; 1630125296Ssilby } 1631125296Ssilby error = uiomove(mtod(m_new, void *), length, uio); 1632125296Ssilby if (error) 1633125296Ssilby goto nospace; 1634125296Ssilby progress += length; 1635125296Ssilby m_new->m_len = length; 1636125296Ssilby if (m_new != m_final) 1637125296Ssilby m_cat(m_final, m_new); 1638125296Ssilby m_new = NULL; 1639125296Ssilby } 1640125296Ssilby m_fixhdr(m_final); 1641125296Ssilby return (m_final); 1642125296Ssilbynospace: 1643125296Ssilby if (m_new) 1644125296Ssilby m_free(m_new); 1645125296Ssilby if (m_final) 1646125296Ssilby m_freem(m_final); 1647125296Ssilby return (NULL); 1648125296Ssilby} 1649148552Ssam 1650148552Ssam/* 1651148552Ssam * Set the m_data pointer of a newly-allocated mbuf 1652148552Ssam * to place an object of the specified size at the 1653148552Ssam * end of the mbuf, longword aligned. 1654148552Ssam */ 1655148552Ssamvoid 1656148552Ssamm_align(struct mbuf *m, int len) 1657148552Ssam{ 1658148552Ssam int adjust; 1659148552Ssam 1660148552Ssam if (m->m_flags & M_EXT) 1661148552Ssam adjust = m->m_ext.ext_size - len; 1662148552Ssam else if (m->m_flags & M_PKTHDR) 1663148552Ssam adjust = MHLEN - len; 1664148552Ssam else 1665148552Ssam adjust = MLEN - len; 1666148552Ssam m->m_data += adjust &~ (sizeof(long)-1); 1667148552Ssam} 1668