uipc_mbuf.c revision 230587
1139804Simp/*- 21541Srgrimes * Copyright (c) 1982, 1986, 1988, 1991, 1993 31541Srgrimes * The Regents of the University of California. All rights reserved. 41541Srgrimes * 51541Srgrimes * Redistribution and use in source and binary forms, with or without 61541Srgrimes * modification, are permitted provided that the following conditions 71541Srgrimes * are met: 81541Srgrimes * 1. Redistributions of source code must retain the above copyright 91541Srgrimes * notice, this list of conditions and the following disclaimer. 101541Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 111541Srgrimes * notice, this list of conditions and the following disclaimer in the 121541Srgrimes * documentation and/or other materials provided with the distribution. 131541Srgrimes * 4. Neither the name of the University nor the names of its contributors 141541Srgrimes * may be used to endorse or promote products derived from this software 151541Srgrimes * without specific prior written permission. 161541Srgrimes * 171541Srgrimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 181541Srgrimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 191541Srgrimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 201541Srgrimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 211541Srgrimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 221541Srgrimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 231541Srgrimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 241541Srgrimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 251541Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 261541Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 271541Srgrimes * SUCH DAMAGE. 281541Srgrimes * 291541Srgrimes * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94 301541Srgrimes */ 311541Srgrimes 32116182Sobrien#include <sys/cdefs.h> 33116182Sobrien__FBSDID("$FreeBSD: head/sys/kern/uipc_mbuf.c 230587 2012-01-26 16:35:09Z ken $"); 34116182Sobrien 3577572Sobrien#include "opt_param.h" 36113490Ssilby#include "opt_mbuf_stress_test.h" 37178674Sjulian#include "opt_mbuf_profiling.h" 38101007Srwatson 391541Srgrimes#include <sys/param.h> 401541Srgrimes#include <sys/systm.h> 4176166Smarkm#include <sys/kernel.h> 42125296Ssilby#include <sys/limits.h> 4376166Smarkm#include <sys/lock.h> 4432036Sbde#include <sys/malloc.h> 451541Srgrimes#include <sys/mbuf.h> 4623081Swollman#include <sys/sysctl.h> 471541Srgrimes#include <sys/domain.h> 481541Srgrimes#include <sys/protosw.h> 49125296Ssilby#include <sys/uio.h> 5076166Smarkm 519759Sbdeint max_linkhdr; 529759Sbdeint max_protohdr; 539759Sbdeint max_hdr; 549759Sbdeint max_datalen; 55116455Ssilby#ifdef MBUF_STRESS_TEST 56112777Ssilbyint m_defragpackets; 57112777Ssilbyint m_defragbytes; 58112777Ssilbyint m_defraguseless; 59112777Ssilbyint m_defragfailure; 60113490Ssilbyint m_defragrandomfailures; 61113490Ssilby#endif 621541Srgrimes 6366475Sbmilekic/* 6466475Sbmilekic * sysctl(8) exported objects 6566475Sbmilekic */ 66155820SandreSYSCTL_INT(_kern_ipc, KIPC_MAX_LINKHDR, max_linkhdr, CTLFLAG_RD, 67155820Sandre &max_linkhdr, 0, "Size of largest link layer header"); 68155820SandreSYSCTL_INT(_kern_ipc, KIPC_MAX_PROTOHDR, max_protohdr, CTLFLAG_RD, 69155820Sandre &max_protohdr, 0, "Size of largest protocol layer header"); 70155820SandreSYSCTL_INT(_kern_ipc, KIPC_MAX_HDR, max_hdr, CTLFLAG_RD, 71155820Sandre &max_hdr, 0, "Size of largest link plus protocol header"); 72155820SandreSYSCTL_INT(_kern_ipc, KIPC_MAX_DATALEN, max_datalen, CTLFLAG_RD, 73155820Sandre &max_datalen, 0, "Minimum space left in mbuf after max_hdr"); 74116455Ssilby#ifdef MBUF_STRESS_TEST 75112777SsilbySYSCTL_INT(_kern_ipc, OID_AUTO, m_defragpackets, CTLFLAG_RD, 76112777Ssilby &m_defragpackets, 0, ""); 77112777SsilbySYSCTL_INT(_kern_ipc, OID_AUTO, m_defragbytes, CTLFLAG_RD, 78112777Ssilby &m_defragbytes, 0, ""); 79112777SsilbySYSCTL_INT(_kern_ipc, OID_AUTO, m_defraguseless, CTLFLAG_RD, 80112777Ssilby &m_defraguseless, 0, ""); 81112777SsilbySYSCTL_INT(_kern_ipc, OID_AUTO, m_defragfailure, CTLFLAG_RD, 82112777Ssilby &m_defragfailure, 0, ""); 83113490SsilbySYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW, 84113490Ssilby &m_defragrandomfailures, 0, ""); 85113490Ssilby#endif 8675112Sbmilekic 871541Srgrimes/* 88129906Sbmilekic * Allocate a given length worth of mbufs and/or clusters (whatever fits 89129906Sbmilekic * best) and return a pointer to the top of the allocated chain. If an 90129906Sbmilekic * existing mbuf chain is provided, then we will append the new chain 91129906Sbmilekic * to the existing one but still return the top of the newly allocated 92129906Sbmilekic * chain. 93129906Sbmilekic */ 94129906Sbmilekicstruct mbuf * 95163915Sandrem_getm2(struct mbuf *m, int len, int how, short type, int flags) 96129906Sbmilekic{ 97163915Sandre struct mbuf *mb, *nm = NULL, *mtail = NULL; 98129906Sbmilekic 99163915Sandre KASSERT(len >= 0, ("%s: len is < 0", __func__)); 100129906Sbmilekic 101163915Sandre /* Validate flags. */ 102163915Sandre flags &= (M_PKTHDR | M_EOR); 103129906Sbmilekic 104163915Sandre /* Packet header mbuf must be first in chain. */ 105163915Sandre if ((flags & M_PKTHDR) && m != NULL) 106163915Sandre flags &= ~M_PKTHDR; 107129906Sbmilekic 108163915Sandre /* Loop and append maximum sized mbufs to the chain tail. */ 109163915Sandre while (len > 0) { 110163915Sandre if (len > MCLBYTES) 111163915Sandre mb = m_getjcl(how, type, (flags & M_PKTHDR), 112163915Sandre MJUMPAGESIZE); 113163915Sandre else if (len >= MINCLSIZE) 114163915Sandre mb = m_getcl(how, type, (flags & M_PKTHDR)); 115163915Sandre else if (flags & M_PKTHDR) 116163915Sandre mb = m_gethdr(how, type); 117129906Sbmilekic else 118163915Sandre mb = m_get(how, type); 119163915Sandre 120163915Sandre /* Fail the whole operation if one mbuf can't be allocated. */ 121163915Sandre if (mb == NULL) { 122163915Sandre if (nm != NULL) 123163915Sandre m_freem(nm); 124163915Sandre return (NULL); 125163915Sandre } 126163915Sandre 127163915Sandre /* Book keeping. */ 128163915Sandre len -= (mb->m_flags & M_EXT) ? mb->m_ext.ext_size : 129163915Sandre ((mb->m_flags & M_PKTHDR) ? MHLEN : MLEN); 130163915Sandre if (mtail != NULL) 131163915Sandre mtail->m_next = mb; 132163915Sandre else 133163915Sandre nm = mb; 134163915Sandre mtail = mb; 135163915Sandre flags &= ~M_PKTHDR; /* Only valid on the first mbuf. */ 136129906Sbmilekic } 137163915Sandre if (flags & M_EOR) 138163915Sandre mtail->m_flags |= M_EOR; /* Only valid on the last mbuf. */ 139129906Sbmilekic 140163915Sandre /* If mbuf was supplied, append new chain to the end of it. */ 141163915Sandre if (m != NULL) { 142163915Sandre for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next) 143163915Sandre ; 144163915Sandre mtail->m_next = nm; 145163915Sandre mtail->m_flags &= ~M_EOR; 146163915Sandre } else 147163915Sandre m = nm; 148163915Sandre 149163915Sandre return (m); 150129906Sbmilekic} 151129906Sbmilekic 152129906Sbmilekic/* 153129906Sbmilekic * Free an entire chain of mbufs and associated external buffers, if 154129906Sbmilekic * applicable. 155129906Sbmilekic */ 156129906Sbmilekicvoid 157129906Sbmilekicm_freem(struct mbuf *mb) 158129906Sbmilekic{ 159129906Sbmilekic 160129906Sbmilekic while (mb != NULL) 161129906Sbmilekic mb = m_free(mb); 162129906Sbmilekic} 163129906Sbmilekic 164210226Strasz/*- 165129906Sbmilekic * Configure a provided mbuf to refer to the provided external storage 166129906Sbmilekic * buffer and setup a reference count for said buffer. If the setting 167129906Sbmilekic * up of the reference count fails, the M_EXT bit will not be set. If 168129906Sbmilekic * successfull, the M_EXT bit is set in the mbuf's flags. 169129906Sbmilekic * 170129906Sbmilekic * Arguments: 171129906Sbmilekic * mb The existing mbuf to which to attach the provided buffer. 172129906Sbmilekic * buf The address of the provided external storage buffer. 173129906Sbmilekic * size The size of the provided buffer. 174129906Sbmilekic * freef A pointer to a routine that is responsible for freeing the 175129906Sbmilekic * provided external storage buffer. 176129906Sbmilekic * args A pointer to an argument structure (of any type) to be passed 177129906Sbmilekic * to the provided freef routine (may be NULL). 178129906Sbmilekic * flags Any other flags to be passed to the provided mbuf. 179129906Sbmilekic * type The type that the external storage buffer should be 180129906Sbmilekic * labeled with. 181129906Sbmilekic * 182129906Sbmilekic * Returns: 183129906Sbmilekic * Nothing. 184129906Sbmilekic */ 185129906Sbmilekicvoid 186129906Sbmilekicm_extadd(struct mbuf *mb, caddr_t buf, u_int size, 187175872Sphk void (*freef)(void *, void *), void *arg1, void *arg2, int flags, int type) 188129906Sbmilekic{ 189151976Sandre KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__)); 190129906Sbmilekic 191151976Sandre if (type != EXT_EXTREF) 192151976Sandre mb->m_ext.ref_cnt = (u_int *)uma_zalloc(zone_ext_refcnt, M_NOWAIT); 193129906Sbmilekic if (mb->m_ext.ref_cnt != NULL) { 194129906Sbmilekic *(mb->m_ext.ref_cnt) = 1; 195129906Sbmilekic mb->m_flags |= (M_EXT | flags); 196129906Sbmilekic mb->m_ext.ext_buf = buf; 197129906Sbmilekic mb->m_data = mb->m_ext.ext_buf; 198129906Sbmilekic mb->m_ext.ext_size = size; 199129906Sbmilekic mb->m_ext.ext_free = freef; 200175872Sphk mb->m_ext.ext_arg1 = arg1; 201175872Sphk mb->m_ext.ext_arg2 = arg2; 202129906Sbmilekic mb->m_ext.ext_type = type; 203129906Sbmilekic } 204129906Sbmilekic} 205129906Sbmilekic 206129906Sbmilekic/* 207129906Sbmilekic * Non-directly-exported function to clean up after mbufs with M_EXT 208151976Sandre * storage attached to them if the reference count hits 1. 209129906Sbmilekic */ 210129906Sbmilekicvoid 211129906Sbmilekicmb_free_ext(struct mbuf *m) 212129906Sbmilekic{ 213172463Skmacy int skipmbuf; 214172463Skmacy 215151976Sandre KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__)); 216151976Sandre KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__)); 217129906Sbmilekic 218172463Skmacy 219172463Skmacy /* 220172463Skmacy * check if the header is embedded in the cluster 221172463Skmacy */ 222172463Skmacy skipmbuf = (m->m_flags & M_NOFREE); 223172463Skmacy 224151976Sandre /* Free attached storage if this mbuf is the only reference to it. */ 225151976Sandre if (*(m->m_ext.ref_cnt) == 1 || 226162515Srrs atomic_fetchadd_int(m->m_ext.ref_cnt, -1) == 1) { 227151976Sandre switch (m->m_ext.ext_type) { 228152101Sandre case EXT_PACKET: /* The packet zone is special. */ 229152035Sandre if (*(m->m_ext.ref_cnt) == 0) 230152035Sandre *(m->m_ext.ref_cnt) = 1; 231151976Sandre uma_zfree(zone_pack, m); 232151976Sandre return; /* Job done. */ 233152101Sandre case EXT_CLUSTER: 234152101Sandre uma_zfree(zone_clust, m->m_ext.ext_buf); 235130289Sbmilekic break; 236155780Sandre case EXT_JUMBOP: 237155780Sandre uma_zfree(zone_jumbop, m->m_ext.ext_buf); 238153232Sandre break; 239151976Sandre case EXT_JUMBO9: 240151976Sandre uma_zfree(zone_jumbo9, m->m_ext.ext_buf); 241151976Sandre break; 242151976Sandre case EXT_JUMBO16: 243151976Sandre uma_zfree(zone_jumbo16, m->m_ext.ext_buf); 244151976Sandre break; 245151976Sandre case EXT_SFBUF: 246151976Sandre case EXT_NET_DRV: 247151976Sandre case EXT_MOD_TYPE: 248151976Sandre case EXT_DISPOSABLE: 249151976Sandre *(m->m_ext.ref_cnt) = 0; 250151976Sandre uma_zfree(zone_ext_refcnt, __DEVOLATILE(u_int *, 251151976Sandre m->m_ext.ref_cnt)); 252151976Sandre /* FALLTHROUGH */ 253151976Sandre case EXT_EXTREF: 254151976Sandre KASSERT(m->m_ext.ext_free != NULL, 255151976Sandre ("%s: ext_free not set", __func__)); 256175872Sphk (*(m->m_ext.ext_free))(m->m_ext.ext_arg1, 257175872Sphk m->m_ext.ext_arg2); 258151976Sandre break; 259151976Sandre default: 260151976Sandre KASSERT(m->m_ext.ext_type == 0, 261151976Sandre ("%s: unknown ext_type", __func__)); 262141668Sbmilekic } 263141668Sbmilekic } 264172463Skmacy if (skipmbuf) 265172463Skmacy return; 266172463Skmacy 267151976Sandre /* 268151976Sandre * Free this mbuf back to the mbuf zone with all m_ext 269151976Sandre * information purged. 270151976Sandre */ 271151976Sandre m->m_ext.ext_buf = NULL; 272151976Sandre m->m_ext.ext_free = NULL; 273175872Sphk m->m_ext.ext_arg1 = NULL; 274175872Sphk m->m_ext.ext_arg2 = NULL; 275151976Sandre m->m_ext.ref_cnt = NULL; 276151976Sandre m->m_ext.ext_size = 0; 277151976Sandre m->m_ext.ext_type = 0; 278151976Sandre m->m_flags &= ~M_EXT; 279130357Sbmilekic uma_zfree(zone_mbuf, m); 280129906Sbmilekic} 281129906Sbmilekic 282129906Sbmilekic/* 283218909Sbrucec * Attach the cluster from *m to *n, set up m_ext in *n 284151976Sandre * and bump the refcount of the cluster. 285151976Sandre */ 286151976Sandrestatic void 287151976Sandremb_dupcl(struct mbuf *n, struct mbuf *m) 288151976Sandre{ 289151976Sandre KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__)); 290151976Sandre KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__)); 291151976Sandre KASSERT((n->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__)); 292151976Sandre 293151976Sandre if (*(m->m_ext.ref_cnt) == 1) 294151976Sandre *(m->m_ext.ref_cnt) += 1; 295151976Sandre else 296151976Sandre atomic_add_int(m->m_ext.ref_cnt, 1); 297151976Sandre n->m_ext.ext_buf = m->m_ext.ext_buf; 298151976Sandre n->m_ext.ext_free = m->m_ext.ext_free; 299175872Sphk n->m_ext.ext_arg1 = m->m_ext.ext_arg1; 300175872Sphk n->m_ext.ext_arg2 = m->m_ext.ext_arg2; 301151976Sandre n->m_ext.ext_size = m->m_ext.ext_size; 302151976Sandre n->m_ext.ref_cnt = m->m_ext.ref_cnt; 303151976Sandre n->m_ext.ext_type = m->m_ext.ext_type; 304151976Sandre n->m_flags |= M_EXT; 305209964Scperciva n->m_flags |= m->m_flags & M_RDONLY; 306151976Sandre} 307151976Sandre 308151976Sandre/* 309149598Sandre * Clean up mbuf (chain) from any tags and packet headers. 310149647Sandre * If "all" is set then the first mbuf in the chain will be 311149647Sandre * cleaned too. 312149598Sandre */ 313149598Sandrevoid 314149647Sandrem_demote(struct mbuf *m0, int all) 315149598Sandre{ 316149598Sandre struct mbuf *m; 317149598Sandre 318149598Sandre for (m = all ? m0 : m0->m_next; m != NULL; m = m->m_next) { 319149598Sandre if (m->m_flags & M_PKTHDR) { 320149598Sandre m_tag_delete_chain(m, NULL); 321149598Sandre m->m_flags &= ~M_PKTHDR; 322149598Sandre bzero(&m->m_pkthdr, sizeof(struct pkthdr)); 323149598Sandre } 324194643Sandre if (m != m0 && m->m_nextpkt != NULL) { 325194643Sandre KASSERT(m->m_nextpkt == NULL, 326194643Sandre ("%s: m_nextpkt not NULL", __func__)); 327194643Sandre m_freem(m->m_nextpkt); 328149598Sandre m->m_nextpkt = NULL; 329194643Sandre } 330194643Sandre m->m_flags = m->m_flags & (M_EXT|M_RDONLY|M_FREELIST|M_NOFREE); 331149598Sandre } 332149598Sandre} 333149598Sandre 334149598Sandre/* 335149648Sandre * Sanity checks on mbuf (chain) for use in KASSERT() and general 336149648Sandre * debugging. 337149648Sandre * Returns 0 or panics when bad and 1 on all tests passed. 338149648Sandre * Sanitize, 0 to run M_SANITY_ACTION, 1 to garble things so they 339149648Sandre * blow up later. 340149599Sandre */ 341149599Sandreint 342149599Sandrem_sanity(struct mbuf *m0, int sanitize) 343149599Sandre{ 344149599Sandre struct mbuf *m; 345149599Sandre caddr_t a, b; 346149599Sandre int pktlen = 0; 347149599Sandre 348168734Skmacy#ifdef INVARIANTS 349168734Skmacy#define M_SANITY_ACTION(s) panic("mbuf %p: " s, m) 350168734Skmacy#else 351168734Skmacy#define M_SANITY_ACTION(s) printf("mbuf %p: " s, m) 352168734Skmacy#endif 353149599Sandre 354149648Sandre for (m = m0; m != NULL; m = m->m_next) { 355149599Sandre /* 356149599Sandre * Basic pointer checks. If any of these fails then some 357149599Sandre * unrelated kernel memory before or after us is trashed. 358149599Sandre * No way to recover from that. 359149599Sandre */ 360149648Sandre a = ((m->m_flags & M_EXT) ? m->m_ext.ext_buf : 361149648Sandre ((m->m_flags & M_PKTHDR) ? (caddr_t)(&m->m_pktdat) : 362149599Sandre (caddr_t)(&m->m_dat)) ); 363149599Sandre b = (caddr_t)(a + (m->m_flags & M_EXT ? m->m_ext.ext_size : 364149648Sandre ((m->m_flags & M_PKTHDR) ? MHLEN : MLEN))); 365149599Sandre if ((caddr_t)m->m_data < a) 366149599Sandre M_SANITY_ACTION("m_data outside mbuf data range left"); 367149599Sandre if ((caddr_t)m->m_data > b) 368149599Sandre M_SANITY_ACTION("m_data outside mbuf data range right"); 369149599Sandre if ((caddr_t)m->m_data + m->m_len > b) 370149599Sandre M_SANITY_ACTION("m_data + m_len exeeds mbuf space"); 371149648Sandre if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.header) { 372149599Sandre if ((caddr_t)m->m_pkthdr.header < a || 373149599Sandre (caddr_t)m->m_pkthdr.header > b) 374149599Sandre M_SANITY_ACTION("m_pkthdr.header outside mbuf data range"); 375149599Sandre } 376149599Sandre 377149599Sandre /* m->m_nextpkt may only be set on first mbuf in chain. */ 378149648Sandre if (m != m0 && m->m_nextpkt != NULL) { 379149599Sandre if (sanitize) { 380149599Sandre m_freem(m->m_nextpkt); 381149599Sandre m->m_nextpkt = (struct mbuf *)0xDEADC0DE; 382149599Sandre } else 383149599Sandre M_SANITY_ACTION("m->m_nextpkt on in-chain mbuf"); 384149599Sandre } 385149599Sandre 386149599Sandre /* packet length (not mbuf length!) calculation */ 387149599Sandre if (m0->m_flags & M_PKTHDR) 388149599Sandre pktlen += m->m_len; 389149599Sandre 390149599Sandre /* m_tags may only be attached to first mbuf in chain. */ 391149599Sandre if (m != m0 && m->m_flags & M_PKTHDR && 392149599Sandre !SLIST_EMPTY(&m->m_pkthdr.tags)) { 393149599Sandre if (sanitize) { 394149599Sandre m_tag_delete_chain(m, NULL); 395149599Sandre /* put in 0xDEADC0DE perhaps? */ 396149648Sandre } else 397149599Sandre M_SANITY_ACTION("m_tags on in-chain mbuf"); 398149599Sandre } 399149599Sandre 400149599Sandre /* M_PKTHDR may only be set on first mbuf in chain */ 401149599Sandre if (m != m0 && m->m_flags & M_PKTHDR) { 402149599Sandre if (sanitize) { 403149599Sandre bzero(&m->m_pkthdr, sizeof(m->m_pkthdr)); 404149599Sandre m->m_flags &= ~M_PKTHDR; 405149599Sandre /* put in 0xDEADCODE and leave hdr flag in */ 406149599Sandre } else 407149599Sandre M_SANITY_ACTION("M_PKTHDR on in-chain mbuf"); 408149599Sandre } 409149599Sandre } 410149648Sandre m = m0; 411149648Sandre if (pktlen && pktlen != m->m_pkthdr.len) { 412149599Sandre if (sanitize) 413149648Sandre m->m_pkthdr.len = 0; 414149599Sandre else 415149599Sandre M_SANITY_ACTION("m_pkthdr.len != mbuf chain length"); 416149599Sandre } 417149648Sandre return 1; 418149648Sandre 419149599Sandre#undef M_SANITY_ACTION 420149599Sandre} 421149599Sandre 422149599Sandre 423149599Sandre/* 424108466Ssam * "Move" mbuf pkthdr from "from" to "to". 425100960Srwatson * "from" must have M_PKTHDR set, and "to" must be empty. 426100960Srwatson */ 427100960Srwatsonvoid 428108466Ssamm_move_pkthdr(struct mbuf *to, struct mbuf *from) 429100960Srwatson{ 430100960Srwatson 431100960Srwatson#if 0 432108466Ssam /* see below for why these are not enabled */ 433113255Sdes M_ASSERTPKTHDR(to); 434113487Srwatson /* Note: with MAC, this may not be a good assertion. */ 435108466Ssam KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), 436108466Ssam ("m_move_pkthdr: to has tags")); 437100960Srwatson#endif 438101007Srwatson#ifdef MAC 439113487Srwatson /* 440113487Srwatson * XXXMAC: It could be this should also occur for non-MAC? 441113487Srwatson */ 442101007Srwatson if (to->m_flags & M_PKTHDR) 443113487Srwatson m_tag_delete_chain(to, NULL); 444101007Srwatson#endif 445143302Ssam to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT); 446143302Ssam if ((to->m_flags & M_EXT) == 0) 447143302Ssam to->m_data = to->m_pktdat; 448108466Ssam to->m_pkthdr = from->m_pkthdr; /* especially tags */ 449108466Ssam SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */ 450108466Ssam from->m_flags &= ~M_PKTHDR; 451108466Ssam} 452108466Ssam 453108466Ssam/* 454108466Ssam * Duplicate "from"'s mbuf pkthdr in "to". 455108466Ssam * "from" must have M_PKTHDR set, and "to" must be empty. 456108466Ssam * In particular, this does a deep copy of the packet tags. 457108466Ssam */ 458108466Ssamint 459108466Ssamm_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how) 460108466Ssam{ 461108466Ssam 462108466Ssam#if 0 463108466Ssam /* 464108466Ssam * The mbuf allocator only initializes the pkthdr 465108466Ssam * when the mbuf is allocated with MGETHDR. Many users 466108466Ssam * (e.g. m_copy*, m_prepend) use MGET and then 467108466Ssam * smash the pkthdr as needed causing these 468108466Ssam * assertions to trip. For now just disable them. 469108466Ssam */ 470113255Sdes M_ASSERTPKTHDR(to); 471113487Srwatson /* Note: with MAC, this may not be a good assertion. */ 472108466Ssam KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), ("m_dup_pkthdr: to has tags")); 473108466Ssam#endif 474132488Salfred MBUF_CHECKSLEEP(how); 475108466Ssam#ifdef MAC 476108466Ssam if (to->m_flags & M_PKTHDR) 477113487Srwatson m_tag_delete_chain(to, NULL); 478108466Ssam#endif 479112733Ssilby to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT); 480112733Ssilby if ((to->m_flags & M_EXT) == 0) 481112733Ssilby to->m_data = to->m_pktdat; 482100960Srwatson to->m_pkthdr = from->m_pkthdr; 483108466Ssam SLIST_INIT(&to->m_pkthdr.tags); 484113480Srwatson return (m_tag_copy_chain(to, from, MBTOM(how))); 485100960Srwatson} 486100960Srwatson 487100960Srwatson/* 4881541Srgrimes * Lesser-used path for M_PREPEND: 4891541Srgrimes * allocate new mbuf to prepend to chain, 4901541Srgrimes * copy junk along. 4911541Srgrimes */ 4921541Srgrimesstruct mbuf * 49372356Sbmilekicm_prepend(struct mbuf *m, int len, int how) 4941541Srgrimes{ 4951541Srgrimes struct mbuf *mn; 4961541Srgrimes 497117770Ssilby if (m->m_flags & M_PKTHDR) 498117770Ssilby MGETHDR(mn, how, m->m_type); 499117770Ssilby else 500117770Ssilby MGET(mn, how, m->m_type); 50172356Sbmilekic if (mn == NULL) { 5021541Srgrimes m_freem(m); 50372356Sbmilekic return (NULL); 5041541Srgrimes } 505113487Srwatson if (m->m_flags & M_PKTHDR) 506108466Ssam M_MOVE_PKTHDR(mn, m); 5071541Srgrimes mn->m_next = m; 5081541Srgrimes m = mn; 509165447Srrs if(m->m_flags & M_PKTHDR) { 510165447Srrs if (len < MHLEN) 511165447Srrs MH_ALIGN(m, len); 512165447Srrs } else { 513165447Srrs if (len < MLEN) 514165447Srrs M_ALIGN(m, len); 515165447Srrs } 5161541Srgrimes m->m_len = len; 5171541Srgrimes return (m); 5181541Srgrimes} 5191541Srgrimes 5201541Srgrimes/* 5211541Srgrimes * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 5221541Srgrimes * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 523177599Sru * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. 52454002Sarchie * Note that the copy is read-only, because clusters are not copied, 52554002Sarchie * only their reference counts are incremented. 5261541Srgrimes */ 5271541Srgrimesstruct mbuf * 52872356Sbmilekicm_copym(struct mbuf *m, int off0, int len, int wait) 5291541Srgrimes{ 53072356Sbmilekic struct mbuf *n, **np; 53172356Sbmilekic int off = off0; 5321541Srgrimes struct mbuf *top; 5331541Srgrimes int copyhdr = 0; 5341541Srgrimes 53552201Salfred KASSERT(off >= 0, ("m_copym, negative off %d", off)); 53652201Salfred KASSERT(len >= 0, ("m_copym, negative len %d", len)); 537132488Salfred MBUF_CHECKSLEEP(wait); 5381541Srgrimes if (off == 0 && m->m_flags & M_PKTHDR) 5391541Srgrimes copyhdr = 1; 5401541Srgrimes while (off > 0) { 54152201Salfred KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain")); 5421541Srgrimes if (off < m->m_len) 5431541Srgrimes break; 5441541Srgrimes off -= m->m_len; 5451541Srgrimes m = m->m_next; 5461541Srgrimes } 5471541Srgrimes np = ⊤ 5481541Srgrimes top = 0; 5491541Srgrimes while (len > 0) { 55072356Sbmilekic if (m == NULL) { 55152201Salfred KASSERT(len == M_COPYALL, 55252201Salfred ("m_copym, length > size of mbuf chain")); 5531541Srgrimes break; 5541541Srgrimes } 555117770Ssilby if (copyhdr) 556117770Ssilby MGETHDR(n, wait, m->m_type); 557117770Ssilby else 558117770Ssilby MGET(n, wait, m->m_type); 5591541Srgrimes *np = n; 56072356Sbmilekic if (n == NULL) 5611541Srgrimes goto nospace; 5621541Srgrimes if (copyhdr) { 563108466Ssam if (!m_dup_pkthdr(n, m, wait)) 564108466Ssam goto nospace; 5651541Srgrimes if (len == M_COPYALL) 5661541Srgrimes n->m_pkthdr.len -= off0; 5671541Srgrimes else 5681541Srgrimes n->m_pkthdr.len = len; 5691541Srgrimes copyhdr = 0; 5701541Srgrimes } 5711541Srgrimes n->m_len = min(len, m->m_len - off); 5721541Srgrimes if (m->m_flags & M_EXT) { 5731541Srgrimes n->m_data = m->m_data + off; 574151976Sandre mb_dupcl(n, m); 5751541Srgrimes } else 5761541Srgrimes bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), 577103569Sbmilekic (u_int)n->m_len); 5781541Srgrimes if (len != M_COPYALL) 5791541Srgrimes len -= n->m_len; 5801541Srgrimes off = 0; 5811541Srgrimes m = m->m_next; 5821541Srgrimes np = &n->m_next; 5831541Srgrimes } 58478592Sbmilekic if (top == NULL) 58578592Sbmilekic mbstat.m_mcfail++; /* XXX: No consistency. */ 58678592Sbmilekic 5871541Srgrimes return (top); 5881541Srgrimesnospace: 5891541Srgrimes m_freem(top); 59078592Sbmilekic mbstat.m_mcfail++; /* XXX: No consistency. */ 59172356Sbmilekic return (NULL); 5921541Srgrimes} 5931541Srgrimes 5941541Srgrimes/* 595149602Sandre * Returns mbuf chain with new head for the prepending case. 596149602Sandre * Copies from mbuf (chain) n from off for len to mbuf (chain) m 597149602Sandre * either prepending or appending the data. 598149602Sandre * The resulting mbuf (chain) m is fully writeable. 599149602Sandre * m is destination (is made writeable) 600149602Sandre * n is source, off is offset in source, len is len from offset 601149602Sandre * dir, 0 append, 1 prepend 602149602Sandre * how, wait or nowait 603149602Sandre */ 604149602Sandre 605149602Sandrestatic int 606149602Sandrem_bcopyxxx(void *s, void *t, u_int len) 607149602Sandre{ 608149602Sandre bcopy(s, t, (size_t)len); 609149602Sandre return 0; 610149602Sandre} 611149602Sandre 612149602Sandrestruct mbuf * 613149602Sandrem_copymdata(struct mbuf *m, struct mbuf *n, int off, int len, 614149602Sandre int prep, int how) 615149602Sandre{ 616151976Sandre struct mbuf *mm, *x, *z, *prev = NULL; 617149602Sandre caddr_t p; 618151976Sandre int i, nlen = 0; 619149602Sandre caddr_t buf[MLEN]; 620149602Sandre 621149602Sandre KASSERT(m != NULL && n != NULL, ("m_copymdata, no target or source")); 622149602Sandre KASSERT(off >= 0, ("m_copymdata, negative off %d", off)); 623149602Sandre KASSERT(len >= 0, ("m_copymdata, negative len %d", len)); 624149602Sandre KASSERT(prep == 0 || prep == 1, ("m_copymdata, unknown direction %d", prep)); 625149602Sandre 626151976Sandre mm = m; 627151976Sandre if (!prep) { 628151976Sandre while(mm->m_next) { 629151976Sandre prev = mm; 630151976Sandre mm = mm->m_next; 631149602Sandre } 632149602Sandre } 633149602Sandre for (z = n; z != NULL; z = z->m_next) 634149602Sandre nlen += z->m_len; 635149602Sandre if (len == M_COPYALL) 636149602Sandre len = nlen - off; 637149602Sandre if (off + len > nlen || len < 1) 638149602Sandre return NULL; 639149602Sandre 640151976Sandre if (!M_WRITABLE(mm)) { 641151976Sandre /* XXX: Use proper m_xxx function instead. */ 642151976Sandre x = m_getcl(how, MT_DATA, mm->m_flags); 643151976Sandre if (x == NULL) 644151976Sandre return NULL; 645151976Sandre bcopy(mm->m_ext.ext_buf, x->m_ext.ext_buf, x->m_ext.ext_size); 646151976Sandre p = x->m_ext.ext_buf + (mm->m_data - mm->m_ext.ext_buf); 647151976Sandre x->m_data = p; 648151976Sandre mm->m_next = NULL; 649151976Sandre if (mm != m) 650151976Sandre prev->m_next = x; 651151976Sandre m_free(mm); 652151976Sandre mm = x; 653151976Sandre } 654151976Sandre 655149602Sandre /* 656149602Sandre * Append/prepend the data. Allocating mbufs as necessary. 657149602Sandre */ 658149602Sandre /* Shortcut if enough free space in first/last mbuf. */ 659149602Sandre if (!prep && M_TRAILINGSPACE(mm) >= len) { 660149602Sandre m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t) + 661149602Sandre mm->m_len); 662149602Sandre mm->m_len += len; 663149602Sandre mm->m_pkthdr.len += len; 664149602Sandre return m; 665149602Sandre } 666149602Sandre if (prep && M_LEADINGSPACE(mm) >= len) { 667149602Sandre mm->m_data = mtod(mm, caddr_t) - len; 668149602Sandre m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t)); 669149602Sandre mm->m_len += len; 670149602Sandre mm->m_pkthdr.len += len; 671149602Sandre return mm; 672149602Sandre } 673149602Sandre 674149602Sandre /* Expand first/last mbuf to cluster if possible. */ 675149602Sandre if (!prep && !(mm->m_flags & M_EXT) && len > M_TRAILINGSPACE(mm)) { 676149602Sandre bcopy(mm->m_data, &buf, mm->m_len); 677149602Sandre m_clget(mm, how); 678149602Sandre if (!(mm->m_flags & M_EXT)) 679149602Sandre return NULL; 680149602Sandre bcopy(&buf, mm->m_ext.ext_buf, mm->m_len); 681149602Sandre mm->m_data = mm->m_ext.ext_buf; 682149602Sandre mm->m_pkthdr.header = NULL; 683149602Sandre } 684149602Sandre if (prep && !(mm->m_flags & M_EXT) && len > M_LEADINGSPACE(mm)) { 685149602Sandre bcopy(mm->m_data, &buf, mm->m_len); 686149602Sandre m_clget(mm, how); 687149602Sandre if (!(mm->m_flags & M_EXT)) 688149602Sandre return NULL; 689149602Sandre bcopy(&buf, (caddr_t *)mm->m_ext.ext_buf + 690149602Sandre mm->m_ext.ext_size - mm->m_len, mm->m_len); 691149602Sandre mm->m_data = (caddr_t)mm->m_ext.ext_buf + 692149602Sandre mm->m_ext.ext_size - mm->m_len; 693149602Sandre mm->m_pkthdr.header = NULL; 694149602Sandre } 695149602Sandre 696149602Sandre /* Append/prepend as many mbuf (clusters) as necessary to fit len. */ 697149602Sandre if (!prep && len > M_TRAILINGSPACE(mm)) { 698149602Sandre if (!m_getm(mm, len - M_TRAILINGSPACE(mm), how, MT_DATA)) 699149602Sandre return NULL; 700149602Sandre } 701149602Sandre if (prep && len > M_LEADINGSPACE(mm)) { 702149602Sandre if (!(z = m_getm(NULL, len - M_LEADINGSPACE(mm), how, MT_DATA))) 703149602Sandre return NULL; 704149602Sandre i = 0; 705149602Sandre for (x = z; x != NULL; x = x->m_next) { 706149602Sandre i += x->m_flags & M_EXT ? x->m_ext.ext_size : 707149602Sandre (x->m_flags & M_PKTHDR ? MHLEN : MLEN); 708149602Sandre if (!x->m_next) 709149602Sandre break; 710149602Sandre } 711149602Sandre z->m_data += i - len; 712149602Sandre m_move_pkthdr(mm, z); 713149602Sandre x->m_next = mm; 714149602Sandre mm = z; 715149602Sandre } 716149602Sandre 717149602Sandre /* Seek to start position in source mbuf. Optimization for long chains. */ 718149602Sandre while (off > 0) { 719149602Sandre if (off < n->m_len) 720149602Sandre break; 721149602Sandre off -= n->m_len; 722149602Sandre n = n->m_next; 723149602Sandre } 724149602Sandre 725149602Sandre /* Copy data into target mbuf. */ 726149602Sandre z = mm; 727149602Sandre while (len > 0) { 728149602Sandre KASSERT(z != NULL, ("m_copymdata, falling off target edge")); 729149602Sandre i = M_TRAILINGSPACE(z); 730149602Sandre m_apply(n, off, i, m_bcopyxxx, mtod(z, caddr_t) + z->m_len); 731149602Sandre z->m_len += i; 732149602Sandre /* fixup pkthdr.len if necessary */ 733149602Sandre if ((prep ? mm : m)->m_flags & M_PKTHDR) 734149602Sandre (prep ? mm : m)->m_pkthdr.len += i; 735149602Sandre off += i; 736149602Sandre len -= i; 737149602Sandre z = z->m_next; 738149602Sandre } 739149602Sandre return (prep ? mm : m); 740149602Sandre} 741149602Sandre 742149602Sandre/* 74315689Swollman * Copy an entire packet, including header (which must be present). 74415689Swollman * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'. 74554002Sarchie * Note that the copy is read-only, because clusters are not copied, 74654002Sarchie * only their reference counts are incremented. 74772750Sluigi * Preserve alignment of the first mbuf so if the creator has left 74872750Sluigi * some room at the beginning (e.g. for inserting protocol headers) 74972750Sluigi * the copies still have the room available. 75015689Swollman */ 75115689Swollmanstruct mbuf * 75272356Sbmilekicm_copypacket(struct mbuf *m, int how) 75315689Swollman{ 75415689Swollman struct mbuf *top, *n, *o; 75515689Swollman 756132488Salfred MBUF_CHECKSLEEP(how); 75715689Swollman MGET(n, how, m->m_type); 75815689Swollman top = n; 75972356Sbmilekic if (n == NULL) 76015689Swollman goto nospace; 76115689Swollman 762108466Ssam if (!m_dup_pkthdr(n, m, how)) 763108466Ssam goto nospace; 76415689Swollman n->m_len = m->m_len; 76515689Swollman if (m->m_flags & M_EXT) { 76615689Swollman n->m_data = m->m_data; 767151976Sandre mb_dupcl(n, m); 76815689Swollman } else { 76972750Sluigi n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat ); 77015689Swollman bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 77115689Swollman } 77215689Swollman 77315689Swollman m = m->m_next; 77415689Swollman while (m) { 77515689Swollman MGET(o, how, m->m_type); 77672356Sbmilekic if (o == NULL) 77715689Swollman goto nospace; 77815689Swollman 77915689Swollman n->m_next = o; 78015689Swollman n = n->m_next; 78115689Swollman 78215689Swollman n->m_len = m->m_len; 78315689Swollman if (m->m_flags & M_EXT) { 78415689Swollman n->m_data = m->m_data; 785151976Sandre mb_dupcl(n, m); 78615689Swollman } else { 78715689Swollman bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 78815689Swollman } 78915689Swollman 79015689Swollman m = m->m_next; 79115689Swollman } 79215689Swollman return top; 79315689Swollmannospace: 79415689Swollman m_freem(top); 79578592Sbmilekic mbstat.m_mcfail++; /* XXX: No consistency. */ 79672356Sbmilekic return (NULL); 79715689Swollman} 79815689Swollman 79915689Swollman/* 8001541Srgrimes * Copy data from an mbuf chain starting "off" bytes from the beginning, 8011541Srgrimes * continuing for "len" bytes, into the indicated buffer. 8021541Srgrimes */ 8031549Srgrimesvoid 80481907Sjulianm_copydata(const struct mbuf *m, int off, int len, caddr_t cp) 8051541Srgrimes{ 806103569Sbmilekic u_int count; 8071541Srgrimes 80852201Salfred KASSERT(off >= 0, ("m_copydata, negative off %d", off)); 80952201Salfred KASSERT(len >= 0, ("m_copydata, negative len %d", len)); 8101541Srgrimes while (off > 0) { 81152201Salfred KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain")); 8121541Srgrimes if (off < m->m_len) 8131541Srgrimes break; 8141541Srgrimes off -= m->m_len; 8151541Srgrimes m = m->m_next; 8161541Srgrimes } 8171541Srgrimes while (len > 0) { 81852201Salfred KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain")); 8191541Srgrimes count = min(m->m_len - off, len); 8201541Srgrimes bcopy(mtod(m, caddr_t) + off, cp, count); 8211541Srgrimes len -= count; 8221541Srgrimes cp += count; 8231541Srgrimes off = 0; 8241541Srgrimes m = m->m_next; 8251541Srgrimes } 8261541Srgrimes} 8271541Srgrimes 8281541Srgrimes/* 82954002Sarchie * Copy a packet header mbuf chain into a completely new chain, including 83054002Sarchie * copying any mbuf clusters. Use this instead of m_copypacket() when 83154002Sarchie * you need a writable copy of an mbuf chain. 83254002Sarchie */ 83354002Sarchiestruct mbuf * 83472356Sbmilekicm_dup(struct mbuf *m, int how) 83554002Sarchie{ 83654002Sarchie struct mbuf **p, *top = NULL; 83754002Sarchie int remain, moff, nsize; 83854002Sarchie 839132488Salfred MBUF_CHECKSLEEP(how); 84054002Sarchie /* Sanity check */ 84154002Sarchie if (m == NULL) 84272356Sbmilekic return (NULL); 843113255Sdes M_ASSERTPKTHDR(m); 84454002Sarchie 84554002Sarchie /* While there's more data, get a new mbuf, tack it on, and fill it */ 84654002Sarchie remain = m->m_pkthdr.len; 84754002Sarchie moff = 0; 84854002Sarchie p = ⊤ 84954002Sarchie while (remain > 0 || top == NULL) { /* allow m->m_pkthdr.len == 0 */ 85054002Sarchie struct mbuf *n; 85154002Sarchie 85254002Sarchie /* Get the next new mbuf */ 853129906Sbmilekic if (remain >= MINCLSIZE) { 854129906Sbmilekic n = m_getcl(how, m->m_type, 0); 855129906Sbmilekic nsize = MCLBYTES; 856129906Sbmilekic } else { 857129906Sbmilekic n = m_get(how, m->m_type); 858129906Sbmilekic nsize = MLEN; 859129906Sbmilekic } 86054002Sarchie if (n == NULL) 86154002Sarchie goto nospace; 862129906Sbmilekic 863129906Sbmilekic if (top == NULL) { /* First one, must be PKTHDR */ 864129906Sbmilekic if (!m_dup_pkthdr(n, m, how)) { 865129906Sbmilekic m_free(n); 866108466Ssam goto nospace; 867129906Sbmilekic } 868153428Semaste if ((n->m_flags & M_EXT) == 0) 869153428Semaste nsize = MHLEN; 87054002Sarchie } 87154002Sarchie n->m_len = 0; 87254002Sarchie 87354002Sarchie /* Link it into the new chain */ 87454002Sarchie *p = n; 87554002Sarchie p = &n->m_next; 87654002Sarchie 87754002Sarchie /* Copy data from original mbuf(s) into new mbuf */ 87854002Sarchie while (n->m_len < nsize && m != NULL) { 87954002Sarchie int chunk = min(nsize - n->m_len, m->m_len - moff); 88054002Sarchie 88154002Sarchie bcopy(m->m_data + moff, n->m_data + n->m_len, chunk); 88254002Sarchie moff += chunk; 88354002Sarchie n->m_len += chunk; 88454002Sarchie remain -= chunk; 88554002Sarchie if (moff == m->m_len) { 88654002Sarchie m = m->m_next; 88754002Sarchie moff = 0; 88854002Sarchie } 88954002Sarchie } 89054002Sarchie 89154002Sarchie /* Check correct total mbuf length */ 89254002Sarchie KASSERT((remain > 0 && m != NULL) || (remain == 0 && m == NULL), 89387594Sobrien ("%s: bogus m_pkthdr.len", __func__)); 89454002Sarchie } 89554002Sarchie return (top); 89654002Sarchie 89754002Sarchienospace: 89854002Sarchie m_freem(top); 89978592Sbmilekic mbstat.m_mcfail++; /* XXX: No consistency. */ 90072356Sbmilekic return (NULL); 90154002Sarchie} 90254002Sarchie 90354002Sarchie/* 9041541Srgrimes * Concatenate mbuf chain n to m. 9051541Srgrimes * Both chains must be of the same type (e.g. MT_DATA). 9061541Srgrimes * Any m_pkthdr is not updated. 9071541Srgrimes */ 9081549Srgrimesvoid 90972356Sbmilekicm_cat(struct mbuf *m, struct mbuf *n) 9101541Srgrimes{ 9111541Srgrimes while (m->m_next) 9121541Srgrimes m = m->m_next; 9131541Srgrimes while (n) { 9141541Srgrimes if (m->m_flags & M_EXT || 9151541Srgrimes m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { 9161541Srgrimes /* just join the two chains */ 9171541Srgrimes m->m_next = n; 9181541Srgrimes return; 9191541Srgrimes } 9201541Srgrimes /* splat the data from one into the other */ 9211541Srgrimes bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 9221541Srgrimes (u_int)n->m_len); 9231541Srgrimes m->m_len += n->m_len; 9241541Srgrimes n = m_free(n); 9251541Srgrimes } 9261541Srgrimes} 9271541Srgrimes 9281549Srgrimesvoid 92972356Sbmilekicm_adj(struct mbuf *mp, int req_len) 9301541Srgrimes{ 93172356Sbmilekic int len = req_len; 93272356Sbmilekic struct mbuf *m; 93372356Sbmilekic int count; 9341541Srgrimes 9351541Srgrimes if ((m = mp) == NULL) 9361541Srgrimes return; 9371541Srgrimes if (len >= 0) { 9381541Srgrimes /* 9391541Srgrimes * Trim from head. 9401541Srgrimes */ 9411541Srgrimes while (m != NULL && len > 0) { 9421541Srgrimes if (m->m_len <= len) { 9431541Srgrimes len -= m->m_len; 9441541Srgrimes m->m_len = 0; 9451541Srgrimes m = m->m_next; 9461541Srgrimes } else { 9471541Srgrimes m->m_len -= len; 9481541Srgrimes m->m_data += len; 9491541Srgrimes len = 0; 9501541Srgrimes } 9511541Srgrimes } 9521541Srgrimes if (mp->m_flags & M_PKTHDR) 953207475Szec mp->m_pkthdr.len -= (req_len - len); 9541541Srgrimes } else { 9551541Srgrimes /* 9561541Srgrimes * Trim from tail. Scan the mbuf chain, 9571541Srgrimes * calculating its length and finding the last mbuf. 9581541Srgrimes * If the adjustment only affects this mbuf, then just 9591541Srgrimes * adjust and return. Otherwise, rescan and truncate 9601541Srgrimes * after the remaining size. 9611541Srgrimes */ 9621541Srgrimes len = -len; 9631541Srgrimes count = 0; 9641541Srgrimes for (;;) { 9651541Srgrimes count += m->m_len; 9661541Srgrimes if (m->m_next == (struct mbuf *)0) 9671541Srgrimes break; 9681541Srgrimes m = m->m_next; 9691541Srgrimes } 9701541Srgrimes if (m->m_len >= len) { 9711541Srgrimes m->m_len -= len; 9721541Srgrimes if (mp->m_flags & M_PKTHDR) 9731541Srgrimes mp->m_pkthdr.len -= len; 9741541Srgrimes return; 9751541Srgrimes } 9761541Srgrimes count -= len; 9771541Srgrimes if (count < 0) 9781541Srgrimes count = 0; 9791541Srgrimes /* 9801541Srgrimes * Correct length for chain is "count". 9811541Srgrimes * Find the mbuf with last data, adjust its length, 9821541Srgrimes * and toss data from remaining mbufs on chain. 9831541Srgrimes */ 9841541Srgrimes m = mp; 9851541Srgrimes if (m->m_flags & M_PKTHDR) 9861541Srgrimes m->m_pkthdr.len = count; 9871541Srgrimes for (; m; m = m->m_next) { 9881541Srgrimes if (m->m_len >= count) { 9891541Srgrimes m->m_len = count; 990142350Ssam if (m->m_next != NULL) { 991142350Ssam m_freem(m->m_next); 992142350Ssam m->m_next = NULL; 993142350Ssam } 9941541Srgrimes break; 9951541Srgrimes } 9961541Srgrimes count -= m->m_len; 9971541Srgrimes } 9981541Srgrimes } 9991541Srgrimes} 10001541Srgrimes 10011541Srgrimes/* 10021541Srgrimes * Rearange an mbuf chain so that len bytes are contiguous 10031541Srgrimes * and in the data area of an mbuf (so that mtod and dtom 10041541Srgrimes * will work for a structure of size len). Returns the resulting 10051541Srgrimes * mbuf chain on success, frees it and returns null on failure. 10061541Srgrimes * If there is room, it will add up to max_protohdr-len extra bytes to the 10071541Srgrimes * contiguous region in an attempt to avoid being called next time. 10081541Srgrimes */ 10091541Srgrimesstruct mbuf * 101072356Sbmilekicm_pullup(struct mbuf *n, int len) 10111541Srgrimes{ 101272356Sbmilekic struct mbuf *m; 101372356Sbmilekic int count; 10141541Srgrimes int space; 10151541Srgrimes 10161541Srgrimes /* 10171541Srgrimes * If first mbuf has no cluster, and has room for len bytes 10181541Srgrimes * without shifting current data, pullup into it, 10191541Srgrimes * otherwise allocate a new mbuf to prepend to the chain. 10201541Srgrimes */ 10211541Srgrimes if ((n->m_flags & M_EXT) == 0 && 10221541Srgrimes n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 10231541Srgrimes if (n->m_len >= len) 10241541Srgrimes return (n); 10251541Srgrimes m = n; 10261541Srgrimes n = n->m_next; 10271541Srgrimes len -= m->m_len; 10281541Srgrimes } else { 10291541Srgrimes if (len > MHLEN) 10301541Srgrimes goto bad; 1031111119Simp MGET(m, M_DONTWAIT, n->m_type); 103272356Sbmilekic if (m == NULL) 10331541Srgrimes goto bad; 10341541Srgrimes m->m_len = 0; 1035108466Ssam if (n->m_flags & M_PKTHDR) 1036108466Ssam M_MOVE_PKTHDR(m, n); 10371541Srgrimes } 10381541Srgrimes space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 10391541Srgrimes do { 10401541Srgrimes count = min(min(max(len, max_protohdr), space), n->m_len); 10411541Srgrimes bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 1042103569Sbmilekic (u_int)count); 10431541Srgrimes len -= count; 10441541Srgrimes m->m_len += count; 10451541Srgrimes n->m_len -= count; 10461541Srgrimes space -= count; 10471541Srgrimes if (n->m_len) 10481541Srgrimes n->m_data += count; 10491541Srgrimes else 10501541Srgrimes n = m_free(n); 10511541Srgrimes } while (len > 0 && n); 10521541Srgrimes if (len > 0) { 10531541Srgrimes (void) m_free(m); 10541541Srgrimes goto bad; 10551541Srgrimes } 10561541Srgrimes m->m_next = n; 10571541Srgrimes return (m); 10581541Srgrimesbad: 10591541Srgrimes m_freem(n); 106078592Sbmilekic mbstat.m_mpfail++; /* XXX: No consistency. */ 106172356Sbmilekic return (NULL); 10621541Srgrimes} 10631541Srgrimes 10641541Srgrimes/* 1065143761Sjmg * Like m_pullup(), except a new mbuf is always allocated, and we allow 1066143761Sjmg * the amount of empty space before the data in the new mbuf to be specified 1067143761Sjmg * (in the event that the caller expects to prepend later). 1068143761Sjmg */ 1069143761Sjmgint MSFail; 1070143761Sjmg 1071143761Sjmgstruct mbuf * 1072143761Sjmgm_copyup(struct mbuf *n, int len, int dstoff) 1073143761Sjmg{ 1074143761Sjmg struct mbuf *m; 1075143761Sjmg int count, space; 1076143761Sjmg 1077143761Sjmg if (len > (MHLEN - dstoff)) 1078143761Sjmg goto bad; 1079143761Sjmg MGET(m, M_DONTWAIT, n->m_type); 1080143761Sjmg if (m == NULL) 1081143761Sjmg goto bad; 1082143761Sjmg m->m_len = 0; 1083143761Sjmg if (n->m_flags & M_PKTHDR) 1084143761Sjmg M_MOVE_PKTHDR(m, n); 1085143761Sjmg m->m_data += dstoff; 1086143761Sjmg space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 1087143761Sjmg do { 1088143761Sjmg count = min(min(max(len, max_protohdr), space), n->m_len); 1089143761Sjmg memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), 1090143761Sjmg (unsigned)count); 1091143761Sjmg len -= count; 1092143761Sjmg m->m_len += count; 1093143761Sjmg n->m_len -= count; 1094143761Sjmg space -= count; 1095143761Sjmg if (n->m_len) 1096143761Sjmg n->m_data += count; 1097143761Sjmg else 1098143761Sjmg n = m_free(n); 1099143761Sjmg } while (len > 0 && n); 1100143761Sjmg if (len > 0) { 1101143761Sjmg (void) m_free(m); 1102143761Sjmg goto bad; 1103143761Sjmg } 1104143761Sjmg m->m_next = n; 1105143761Sjmg return (m); 1106143761Sjmg bad: 1107143761Sjmg m_freem(n); 1108143761Sjmg MSFail++; 1109143761Sjmg return (NULL); 1110143761Sjmg} 1111143761Sjmg 1112143761Sjmg/* 11131541Srgrimes * Partition an mbuf chain in two pieces, returning the tail -- 11141541Srgrimes * all but the first len0 bytes. In case of failure, it returns NULL and 11151541Srgrimes * attempts to restore the chain to its original state. 111697681Sarchie * 111797681Sarchie * Note that the resulting mbufs might be read-only, because the new 111897681Sarchie * mbuf can end up sharing an mbuf cluster with the original mbuf if 111997681Sarchie * the "breaking point" happens to lie within a cluster mbuf. Use the 112097681Sarchie * M_WRITABLE() macro to check for this case. 11211541Srgrimes */ 11221541Srgrimesstruct mbuf * 112372356Sbmilekicm_split(struct mbuf *m0, int len0, int wait) 11241541Srgrimes{ 112572356Sbmilekic struct mbuf *m, *n; 1126103569Sbmilekic u_int len = len0, remain; 11271541Srgrimes 1128132488Salfred MBUF_CHECKSLEEP(wait); 11291541Srgrimes for (m = m0; m && len > m->m_len; m = m->m_next) 11301541Srgrimes len -= m->m_len; 113172356Sbmilekic if (m == NULL) 113272356Sbmilekic return (NULL); 11331541Srgrimes remain = m->m_len - len; 11341541Srgrimes if (m0->m_flags & M_PKTHDR) { 11351541Srgrimes MGETHDR(n, wait, m0->m_type); 113672356Sbmilekic if (n == NULL) 113772356Sbmilekic return (NULL); 11381541Srgrimes n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 11391541Srgrimes n->m_pkthdr.len = m0->m_pkthdr.len - len0; 11401541Srgrimes m0->m_pkthdr.len = len0; 11411541Srgrimes if (m->m_flags & M_EXT) 11421541Srgrimes goto extpacket; 11431541Srgrimes if (remain > MHLEN) { 11441541Srgrimes /* m can't be the lead packet */ 11451541Srgrimes MH_ALIGN(n, 0); 11461541Srgrimes n->m_next = m_split(m, len, wait); 114772356Sbmilekic if (n->m_next == NULL) { 11481541Srgrimes (void) m_free(n); 114972356Sbmilekic return (NULL); 115094471Shsu } else { 115194471Shsu n->m_len = 0; 11521541Srgrimes return (n); 115394471Shsu } 11541541Srgrimes } else 11551541Srgrimes MH_ALIGN(n, remain); 11561541Srgrimes } else if (remain == 0) { 11571541Srgrimes n = m->m_next; 115872356Sbmilekic m->m_next = NULL; 11591541Srgrimes return (n); 11601541Srgrimes } else { 11611541Srgrimes MGET(n, wait, m->m_type); 116272356Sbmilekic if (n == NULL) 116372356Sbmilekic return (NULL); 11641541Srgrimes M_ALIGN(n, remain); 11651541Srgrimes } 11661541Srgrimesextpacket: 11671541Srgrimes if (m->m_flags & M_EXT) { 11681541Srgrimes n->m_data = m->m_data + len; 1169151976Sandre mb_dupcl(n, m); 11701541Srgrimes } else { 11711541Srgrimes bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); 11721541Srgrimes } 11731541Srgrimes n->m_len = remain; 11741541Srgrimes m->m_len = len; 11751541Srgrimes n->m_next = m->m_next; 117672356Sbmilekic m->m_next = NULL; 11771541Srgrimes return (n); 11781541Srgrimes} 11791541Srgrimes/* 11801541Srgrimes * Routine to copy from device local memory into mbufs. 118178508Sbmilekic * Note that `off' argument is offset into first mbuf of target chain from 118278508Sbmilekic * which to begin copying the data to. 11831541Srgrimes */ 11841541Srgrimesstruct mbuf * 118578508Sbmilekicm_devget(char *buf, int totlen, int off, struct ifnet *ifp, 1186169624Srwatson void (*copy)(char *from, caddr_t to, u_int len)) 11871541Srgrimes{ 118872356Sbmilekic struct mbuf *m; 1189129906Sbmilekic struct mbuf *top = NULL, **mp = ⊤ 119078508Sbmilekic int len; 11911541Srgrimes 119278508Sbmilekic if (off < 0 || off > MHLEN) 119378508Sbmilekic return (NULL); 119478508Sbmilekic 1195129906Sbmilekic while (totlen > 0) { 1196129906Sbmilekic if (top == NULL) { /* First one, must be PKTHDR */ 1197129906Sbmilekic if (totlen + off >= MINCLSIZE) { 1198129906Sbmilekic m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); 1199129906Sbmilekic len = MCLBYTES; 1200129906Sbmilekic } else { 1201129906Sbmilekic m = m_gethdr(M_DONTWAIT, MT_DATA); 1202129906Sbmilekic len = MHLEN; 12031541Srgrimes 1204129906Sbmilekic /* Place initial small packet/header at end of mbuf */ 1205129906Sbmilekic if (m && totlen + off + max_linkhdr <= MLEN) { 1206129906Sbmilekic m->m_data += max_linkhdr; 1207129906Sbmilekic len -= max_linkhdr; 1208129906Sbmilekic } 1209129906Sbmilekic } 1210129906Sbmilekic if (m == NULL) 1211129906Sbmilekic return NULL; 1212129906Sbmilekic m->m_pkthdr.rcvif = ifp; 1213129906Sbmilekic m->m_pkthdr.len = totlen; 1214129906Sbmilekic } else { 1215129906Sbmilekic if (totlen + off >= MINCLSIZE) { 1216129906Sbmilekic m = m_getcl(M_DONTWAIT, MT_DATA, 0); 1217129906Sbmilekic len = MCLBYTES; 1218129906Sbmilekic } else { 1219129906Sbmilekic m = m_get(M_DONTWAIT, MT_DATA); 1220129906Sbmilekic len = MLEN; 1221129906Sbmilekic } 122272356Sbmilekic if (m == NULL) { 12231541Srgrimes m_freem(top); 1224129906Sbmilekic return NULL; 12251541Srgrimes } 12261541Srgrimes } 122778508Sbmilekic if (off) { 122878508Sbmilekic m->m_data += off; 122978508Sbmilekic len -= off; 123078508Sbmilekic off = 0; 123178508Sbmilekic } 123278508Sbmilekic m->m_len = len = min(totlen, len); 12331541Srgrimes if (copy) 1234103569Sbmilekic copy(buf, mtod(m, caddr_t), (u_int)len); 12351541Srgrimes else 1236103569Sbmilekic bcopy(buf, mtod(m, caddr_t), (u_int)len); 123778508Sbmilekic buf += len; 12381541Srgrimes *mp = m; 12391541Srgrimes mp = &m->m_next; 12401541Srgrimes totlen -= len; 12411541Srgrimes } 12421541Srgrimes return (top); 12431541Srgrimes} 12443352Sphk 12453352Sphk/* 12463352Sphk * Copy data from a buffer back into the indicated mbuf chain, 12473352Sphk * starting "off" bytes from the beginning, extending the mbuf 12483352Sphk * chain if necessary. 12493352Sphk */ 12503352Sphkvoid 1251128402Sluigim_copyback(struct mbuf *m0, int off, int len, c_caddr_t cp) 12523352Sphk{ 125372356Sbmilekic int mlen; 125472356Sbmilekic struct mbuf *m = m0, *n; 12553352Sphk int totlen = 0; 12563352Sphk 125772356Sbmilekic if (m0 == NULL) 12583352Sphk return; 12593352Sphk while (off > (mlen = m->m_len)) { 12603352Sphk off -= mlen; 12613352Sphk totlen += mlen; 126272356Sbmilekic if (m->m_next == NULL) { 1263129906Sbmilekic n = m_get(M_DONTWAIT, m->m_type); 126472356Sbmilekic if (n == NULL) 12653352Sphk goto out; 1266129906Sbmilekic bzero(mtod(n, caddr_t), MLEN); 12673352Sphk n->m_len = min(MLEN, len + off); 12683352Sphk m->m_next = n; 12693352Sphk } 12703352Sphk m = m->m_next; 12713352Sphk } 12723352Sphk while (len > 0) { 1273187409Smav if (m->m_next == NULL && (len > m->m_len - off)) { 1274187409Smav m->m_len += min(len - (m->m_len - off), 1275187409Smav M_TRAILINGSPACE(m)); 1276187409Smav } 12773352Sphk mlen = min (m->m_len - off, len); 1278103569Sbmilekic bcopy(cp, off + mtod(m, caddr_t), (u_int)mlen); 12793352Sphk cp += mlen; 12803352Sphk len -= mlen; 12813352Sphk mlen += off; 12823352Sphk off = 0; 12833352Sphk totlen += mlen; 12843352Sphk if (len == 0) 12853352Sphk break; 128672356Sbmilekic if (m->m_next == NULL) { 1287111119Simp n = m_get(M_DONTWAIT, m->m_type); 128872356Sbmilekic if (n == NULL) 12893352Sphk break; 12903352Sphk n->m_len = min(MLEN, len); 12913352Sphk m->m_next = n; 12923352Sphk } 12933352Sphk m = m->m_next; 12943352Sphk } 12953352Sphkout: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) 12963352Sphk m->m_pkthdr.len = totlen; 12973352Sphk} 129852756Sphk 1299123557Sbms/* 1300138541Ssam * Append the specified data to the indicated mbuf chain, 1301138541Ssam * Extend the mbuf chain if the new data does not fit in 1302138541Ssam * existing space. 1303138541Ssam * 1304138541Ssam * Return 1 if able to complete the job; otherwise 0. 1305138541Ssam */ 1306138541Ssamint 1307138541Ssamm_append(struct mbuf *m0, int len, c_caddr_t cp) 1308138541Ssam{ 1309138541Ssam struct mbuf *m, *n; 1310138541Ssam int remainder, space; 1311138541Ssam 1312138541Ssam for (m = m0; m->m_next != NULL; m = m->m_next) 1313138541Ssam ; 1314138541Ssam remainder = len; 1315138541Ssam space = M_TRAILINGSPACE(m); 1316138541Ssam if (space > 0) { 1317138541Ssam /* 1318138541Ssam * Copy into available space. 1319138541Ssam */ 1320138541Ssam if (space > remainder) 1321138541Ssam space = remainder; 1322138541Ssam bcopy(cp, mtod(m, caddr_t) + m->m_len, space); 1323138541Ssam m->m_len += space; 1324138541Ssam cp += space, remainder -= space; 1325138541Ssam } 1326138541Ssam while (remainder > 0) { 1327138541Ssam /* 1328138541Ssam * Allocate a new mbuf; could check space 1329138541Ssam * and allocate a cluster instead. 1330138541Ssam */ 1331138541Ssam n = m_get(M_DONTWAIT, m->m_type); 1332138541Ssam if (n == NULL) 1333138541Ssam break; 1334138541Ssam n->m_len = min(MLEN, remainder); 1335138894Ssam bcopy(cp, mtod(n, caddr_t), n->m_len); 1336138894Ssam cp += n->m_len, remainder -= n->m_len; 1337138541Ssam m->m_next = n; 1338138541Ssam m = n; 1339138541Ssam } 1340138541Ssam if (m0->m_flags & M_PKTHDR) 1341138541Ssam m0->m_pkthdr.len += len - remainder; 1342138541Ssam return (remainder == 0); 1343138541Ssam} 1344138541Ssam 1345138541Ssam/* 1346123557Sbms * Apply function f to the data in an mbuf chain starting "off" bytes from 1347123557Sbms * the beginning, continuing for "len" bytes. 1348123557Sbms */ 1349123557Sbmsint 1350123557Sbmsm_apply(struct mbuf *m, int off, int len, 1351123564Sbms int (*f)(void *, void *, u_int), void *arg) 1352123557Sbms{ 1353123564Sbms u_int count; 1354123557Sbms int rval; 1355123557Sbms 1356123557Sbms KASSERT(off >= 0, ("m_apply, negative off %d", off)); 1357123557Sbms KASSERT(len >= 0, ("m_apply, negative len %d", len)); 1358123557Sbms while (off > 0) { 1359123557Sbms KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain")); 1360123557Sbms if (off < m->m_len) 1361123557Sbms break; 1362123557Sbms off -= m->m_len; 1363123557Sbms m = m->m_next; 1364123557Sbms } 1365123557Sbms while (len > 0) { 1366123557Sbms KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain")); 1367123557Sbms count = min(m->m_len - off, len); 1368123557Sbms rval = (*f)(arg, mtod(m, caddr_t) + off, count); 1369123557Sbms if (rval) 1370123557Sbms return (rval); 1371123557Sbms len -= count; 1372123557Sbms off = 0; 1373123557Sbms m = m->m_next; 1374123557Sbms } 1375123557Sbms return (0); 1376123557Sbms} 1377123557Sbms 1378123557Sbms/* 1379123557Sbms * Return a pointer to mbuf/offset of location in mbuf chain. 1380123557Sbms */ 1381123557Sbmsstruct mbuf * 1382123557Sbmsm_getptr(struct mbuf *m, int loc, int *off) 1383123557Sbms{ 1384123557Sbms 1385123557Sbms while (loc >= 0) { 1386123564Sbms /* Normal end of search. */ 1387123557Sbms if (m->m_len > loc) { 1388123557Sbms *off = loc; 1389123557Sbms return (m); 1390123557Sbms } else { 1391123557Sbms loc -= m->m_len; 1392123557Sbms if (m->m_next == NULL) { 1393123557Sbms if (loc == 0) { 1394123564Sbms /* Point at the end of valid data. */ 1395123557Sbms *off = m->m_len; 1396123557Sbms return (m); 1397123564Sbms } 1398123564Sbms return (NULL); 1399123564Sbms } 1400123564Sbms m = m->m_next; 1401123557Sbms } 1402123557Sbms } 1403123557Sbms return (NULL); 1404123557Sbms} 1405123557Sbms 140652756Sphkvoid 1407135904Sjmgm_print(const struct mbuf *m, int maxlen) 140852756Sphk{ 140952756Sphk int len; 1410135904Sjmg int pdata; 141154906Seivind const struct mbuf *m2; 141252756Sphk 1413230587Sken if (m == NULL) { 1414230587Sken printf("mbuf: %p\n", m); 1415230587Sken return; 1416230587Sken } 1417230587Sken 1418135904Sjmg if (m->m_flags & M_PKTHDR) 1419135904Sjmg len = m->m_pkthdr.len; 1420135904Sjmg else 1421135904Sjmg len = -1; 142252756Sphk m2 = m; 1423135904Sjmg while (m2 != NULL && (len == -1 || len)) { 1424135904Sjmg pdata = m2->m_len; 1425135904Sjmg if (maxlen != -1 && pdata > maxlen) 1426135904Sjmg pdata = maxlen; 1427135904Sjmg printf("mbuf: %p len: %d, next: %p, %b%s", m2, m2->m_len, 1428135904Sjmg m2->m_next, m2->m_flags, "\20\20freelist\17skipfw" 1429135904Sjmg "\11proto5\10proto4\7proto3\6proto2\5proto1\4rdonly" 1430135904Sjmg "\3eor\2pkthdr\1ext", pdata ? "" : "\n"); 1431135904Sjmg if (pdata) 1432156700Sjmg printf(", %*D\n", pdata, (u_char *)m2->m_data, "-"); 1433135904Sjmg if (len != -1) 1434135904Sjmg len -= m2->m_len; 143552756Sphk m2 = m2->m_next; 143652756Sphk } 1437135904Sjmg if (len > 0) 1438135904Sjmg printf("%d bytes unaccounted for.\n", len); 143952756Sphk return; 144052756Sphk} 1441103540Sphk 1442103569Sbmilekicu_int 1443103540Sphkm_fixhdr(struct mbuf *m0) 1444103540Sphk{ 1445103569Sbmilekic u_int len; 1446103540Sphk 1447103544Sphk len = m_length(m0, NULL); 1448103544Sphk m0->m_pkthdr.len = len; 1449103544Sphk return (len); 1450103544Sphk} 1451103544Sphk 1452103569Sbmilekicu_int 1453103544Sphkm_length(struct mbuf *m0, struct mbuf **last) 1454103544Sphk{ 1455103544Sphk struct mbuf *m; 1456103569Sbmilekic u_int len; 1457103544Sphk 1458103544Sphk len = 0; 1459103544Sphk for (m = m0; m != NULL; m = m->m_next) { 1460103540Sphk len += m->m_len; 1461103544Sphk if (m->m_next == NULL) 1462103544Sphk break; 1463103540Sphk } 1464103544Sphk if (last != NULL) 1465103544Sphk *last = m; 1466103544Sphk return (len); 1467103540Sphk} 1468112777Ssilby 1469112777Ssilby/* 1470112777Ssilby * Defragment a mbuf chain, returning the shortest possible 1471112777Ssilby * chain of mbufs and clusters. If allocation fails and 1472112777Ssilby * this cannot be completed, NULL will be returned, but 1473112777Ssilby * the passed in chain will be unchanged. Upon success, 1474112777Ssilby * the original chain will be freed, and the new chain 1475112777Ssilby * will be returned. 1476112777Ssilby * 1477112777Ssilby * If a non-packet header is passed in, the original 1478112777Ssilby * mbuf (chain?) will be returned unharmed. 1479112777Ssilby */ 1480112777Ssilbystruct mbuf * 1481112777Ssilbym_defrag(struct mbuf *m0, int how) 1482112777Ssilby{ 1483125472Ssilby struct mbuf *m_new = NULL, *m_final = NULL; 1484125472Ssilby int progress = 0, length; 1485112777Ssilby 1486132488Salfred MBUF_CHECKSLEEP(how); 1487112777Ssilby if (!(m0->m_flags & M_PKTHDR)) 1488112777Ssilby return (m0); 1489112777Ssilby 1490117770Ssilby m_fixhdr(m0); /* Needed sanity check */ 1491117770Ssilby 1492113490Ssilby#ifdef MBUF_STRESS_TEST 1493113490Ssilby if (m_defragrandomfailures) { 1494113490Ssilby int temp = arc4random() & 0xff; 1495113490Ssilby if (temp == 0xba) 1496113490Ssilby goto nospace; 1497113490Ssilby } 1498113490Ssilby#endif 1499112777Ssilby 1500112777Ssilby if (m0->m_pkthdr.len > MHLEN) 1501112777Ssilby m_final = m_getcl(how, MT_DATA, M_PKTHDR); 1502112777Ssilby else 1503112777Ssilby m_final = m_gethdr(how, MT_DATA); 1504112777Ssilby 1505112777Ssilby if (m_final == NULL) 1506112777Ssilby goto nospace; 1507112777Ssilby 1508123740Speter if (m_dup_pkthdr(m_final, m0, how) == 0) 1509112777Ssilby goto nospace; 1510112777Ssilby 1511112777Ssilby m_new = m_final; 1512112777Ssilby 1513112777Ssilby while (progress < m0->m_pkthdr.len) { 1514112777Ssilby length = m0->m_pkthdr.len - progress; 1515112777Ssilby if (length > MCLBYTES) 1516112777Ssilby length = MCLBYTES; 1517112777Ssilby 1518112777Ssilby if (m_new == NULL) { 1519112777Ssilby if (length > MLEN) 1520112777Ssilby m_new = m_getcl(how, MT_DATA, 0); 1521112777Ssilby else 1522112777Ssilby m_new = m_get(how, MT_DATA); 1523112777Ssilby if (m_new == NULL) 1524112777Ssilby goto nospace; 1525112777Ssilby } 1526112777Ssilby 1527112777Ssilby m_copydata(m0, progress, length, mtod(m_new, caddr_t)); 1528112777Ssilby progress += length; 1529112777Ssilby m_new->m_len = length; 1530112777Ssilby if (m_new != m_final) 1531112777Ssilby m_cat(m_final, m_new); 1532112777Ssilby m_new = NULL; 1533112777Ssilby } 1534116455Ssilby#ifdef MBUF_STRESS_TEST 1535112777Ssilby if (m0->m_next == NULL) 1536112777Ssilby m_defraguseless++; 1537116455Ssilby#endif 1538112777Ssilby m_freem(m0); 1539112777Ssilby m0 = m_final; 1540116455Ssilby#ifdef MBUF_STRESS_TEST 1541112777Ssilby m_defragpackets++; 1542112777Ssilby m_defragbytes += m0->m_pkthdr.len; 1543116455Ssilby#endif 1544112777Ssilby return (m0); 1545112777Ssilbynospace: 1546116455Ssilby#ifdef MBUF_STRESS_TEST 1547112777Ssilby m_defragfailure++; 1548116455Ssilby#endif 1549112777Ssilby if (m_final) 1550112777Ssilby m_freem(m_final); 1551112777Ssilby return (NULL); 1552112777Ssilby} 1553119644Ssilby 1554175414Ssam/* 1555175414Ssam * Defragment an mbuf chain, returning at most maxfrags separate 1556175414Ssam * mbufs+clusters. If this is not possible NULL is returned and 1557175414Ssam * the original mbuf chain is left in it's present (potentially 1558175414Ssam * modified) state. We use two techniques: collapsing consecutive 1559175414Ssam * mbufs and replacing consecutive mbufs by a cluster. 1560175414Ssam * 1561175414Ssam * NB: this should really be named m_defrag but that name is taken 1562175414Ssam */ 1563175414Ssamstruct mbuf * 1564175414Ssamm_collapse(struct mbuf *m0, int how, int maxfrags) 1565175414Ssam{ 1566175414Ssam struct mbuf *m, *n, *n2, **prev; 1567175414Ssam u_int curfrags; 1568175414Ssam 1569175414Ssam /* 1570175414Ssam * Calculate the current number of frags. 1571175414Ssam */ 1572175414Ssam curfrags = 0; 1573175414Ssam for (m = m0; m != NULL; m = m->m_next) 1574175414Ssam curfrags++; 1575175414Ssam /* 1576175414Ssam * First, try to collapse mbufs. Note that we always collapse 1577175414Ssam * towards the front so we don't need to deal with moving the 1578175414Ssam * pkthdr. This may be suboptimal if the first mbuf has much 1579175414Ssam * less data than the following. 1580175414Ssam */ 1581175414Ssam m = m0; 1582175414Ssamagain: 1583175414Ssam for (;;) { 1584175414Ssam n = m->m_next; 1585175414Ssam if (n == NULL) 1586175414Ssam break; 1587175414Ssam if ((m->m_flags & M_RDONLY) == 0 && 1588175414Ssam n->m_len < M_TRAILINGSPACE(m)) { 1589175414Ssam bcopy(mtod(n, void *), mtod(m, char *) + m->m_len, 1590175414Ssam n->m_len); 1591175414Ssam m->m_len += n->m_len; 1592175414Ssam m->m_next = n->m_next; 1593175414Ssam m_free(n); 1594175414Ssam if (--curfrags <= maxfrags) 1595175414Ssam return m0; 1596175414Ssam } else 1597175414Ssam m = n; 1598175414Ssam } 1599175414Ssam KASSERT(maxfrags > 1, 1600175414Ssam ("maxfrags %u, but normal collapse failed", maxfrags)); 1601175414Ssam /* 1602175414Ssam * Collapse consecutive mbufs to a cluster. 1603175414Ssam */ 1604175414Ssam prev = &m0->m_next; /* NB: not the first mbuf */ 1605175414Ssam while ((n = *prev) != NULL) { 1606175414Ssam if ((n2 = n->m_next) != NULL && 1607175414Ssam n->m_len + n2->m_len < MCLBYTES) { 1608175414Ssam m = m_getcl(how, MT_DATA, 0); 1609175414Ssam if (m == NULL) 1610175414Ssam goto bad; 1611175414Ssam bcopy(mtod(n, void *), mtod(m, void *), n->m_len); 1612175414Ssam bcopy(mtod(n2, void *), mtod(m, char *) + n->m_len, 1613175414Ssam n2->m_len); 1614175414Ssam m->m_len = n->m_len + n2->m_len; 1615175414Ssam m->m_next = n2->m_next; 1616175414Ssam *prev = m; 1617175414Ssam m_free(n); 1618175414Ssam m_free(n2); 1619175414Ssam if (--curfrags <= maxfrags) /* +1 cl -2 mbufs */ 1620175414Ssam return m0; 1621175414Ssam /* 1622175414Ssam * Still not there, try the normal collapse 1623175414Ssam * again before we allocate another cluster. 1624175414Ssam */ 1625175414Ssam goto again; 1626175414Ssam } 1627175414Ssam prev = &n->m_next; 1628175414Ssam } 1629175414Ssam /* 1630175414Ssam * No place where we can collapse to a cluster; punt. 1631175414Ssam * This can occur if, for example, you request 2 frags 1632175414Ssam * but the packet requires that both be clusters (we 1633175414Ssam * never reallocate the first mbuf to avoid moving the 1634175414Ssam * packet header). 1635175414Ssam */ 1636175414Ssambad: 1637175414Ssam return NULL; 1638175414Ssam} 1639175414Ssam 1640119644Ssilby#ifdef MBUF_STRESS_TEST 1641119644Ssilby 1642119644Ssilby/* 1643119644Ssilby * Fragment an mbuf chain. There's no reason you'd ever want to do 1644119644Ssilby * this in normal usage, but it's great for stress testing various 1645119644Ssilby * mbuf consumers. 1646119644Ssilby * 1647119644Ssilby * If fragmentation is not possible, the original chain will be 1648119644Ssilby * returned. 1649119644Ssilby * 1650119644Ssilby * Possible length values: 1651119644Ssilby * 0 no fragmentation will occur 1652119644Ssilby * > 0 each fragment will be of the specified length 1653119644Ssilby * -1 each fragment will be the same random value in length 1654119644Ssilby * -2 each fragment's length will be entirely random 1655119644Ssilby * (Random values range from 1 to 256) 1656119644Ssilby */ 1657119644Ssilbystruct mbuf * 1658119644Ssilbym_fragment(struct mbuf *m0, int how, int length) 1659119644Ssilby{ 1660125472Ssilby struct mbuf *m_new = NULL, *m_final = NULL; 1661125472Ssilby int progress = 0; 1662119644Ssilby 1663119644Ssilby if (!(m0->m_flags & M_PKTHDR)) 1664119644Ssilby return (m0); 1665119644Ssilby 1666119644Ssilby if ((length == 0) || (length < -2)) 1667119644Ssilby return (m0); 1668119644Ssilby 1669119644Ssilby m_fixhdr(m0); /* Needed sanity check */ 1670119644Ssilby 1671119644Ssilby m_final = m_getcl(how, MT_DATA, M_PKTHDR); 1672119644Ssilby 1673119644Ssilby if (m_final == NULL) 1674119644Ssilby goto nospace; 1675119644Ssilby 1676123823Ssilby if (m_dup_pkthdr(m_final, m0, how) == 0) 1677119644Ssilby goto nospace; 1678119644Ssilby 1679119644Ssilby m_new = m_final; 1680119644Ssilby 1681119644Ssilby if (length == -1) 1682119644Ssilby length = 1 + (arc4random() & 255); 1683119644Ssilby 1684119644Ssilby while (progress < m0->m_pkthdr.len) { 1685119644Ssilby int fraglen; 1686119644Ssilby 1687119644Ssilby if (length > 0) 1688119644Ssilby fraglen = length; 1689119644Ssilby else 1690119644Ssilby fraglen = 1 + (arc4random() & 255); 1691119644Ssilby if (fraglen > m0->m_pkthdr.len - progress) 1692119644Ssilby fraglen = m0->m_pkthdr.len - progress; 1693119644Ssilby 1694119644Ssilby if (fraglen > MCLBYTES) 1695119644Ssilby fraglen = MCLBYTES; 1696119644Ssilby 1697119644Ssilby if (m_new == NULL) { 1698119644Ssilby m_new = m_getcl(how, MT_DATA, 0); 1699119644Ssilby if (m_new == NULL) 1700119644Ssilby goto nospace; 1701119644Ssilby } 1702119644Ssilby 1703119644Ssilby m_copydata(m0, progress, fraglen, mtod(m_new, caddr_t)); 1704119644Ssilby progress += fraglen; 1705119644Ssilby m_new->m_len = fraglen; 1706119644Ssilby if (m_new != m_final) 1707119644Ssilby m_cat(m_final, m_new); 1708119644Ssilby m_new = NULL; 1709119644Ssilby } 1710119644Ssilby m_freem(m0); 1711119644Ssilby m0 = m_final; 1712119644Ssilby return (m0); 1713119644Ssilbynospace: 1714119644Ssilby if (m_final) 1715119644Ssilby m_freem(m_final); 1716119644Ssilby /* Return the original chain on failure */ 1717119644Ssilby return (m0); 1718119644Ssilby} 1719119644Ssilby 1720119644Ssilby#endif 1721125296Ssilby 1722163915Sandre/* 1723163915Sandre * Copy the contents of uio into a properly sized mbuf chain. 1724163915Sandre */ 1725125296Ssilbystruct mbuf * 1726163915Sandrem_uiotombuf(struct uio *uio, int how, int len, int align, int flags) 1727125296Ssilby{ 1728163915Sandre struct mbuf *m, *mb; 1729163915Sandre int error, length, total; 1730163915Sandre int progress = 0; 1731125296Ssilby 1732163915Sandre /* 1733163915Sandre * len can be zero or an arbitrary large value bound by 1734163915Sandre * the total data supplied by the uio. 1735163915Sandre */ 1736125296Ssilby if (len > 0) 1737125296Ssilby total = min(uio->uio_resid, len); 1738125296Ssilby else 1739125296Ssilby total = uio->uio_resid; 1740163915Sandre 1741163915Sandre /* 1742163915Sandre * The smallest unit returned by m_getm2() is a single mbuf 1743182777Sthompsa * with pkthdr. We can't align past it. 1744163915Sandre */ 1745145883Semax if (align >= MHLEN) 1746163915Sandre return (NULL); 1747163915Sandre 1748166171Sandre /* 1749166171Sandre * Give us the full allocation or nothing. 1750166171Sandre * If len is zero return the smallest empty mbuf. 1751166171Sandre */ 1752166171Sandre m = m_getm2(NULL, max(total + align, 1), how, MT_DATA, flags); 1753163915Sandre if (m == NULL) 1754163915Sandre return (NULL); 1755163915Sandre m->m_data += align; 1756163915Sandre 1757163915Sandre /* Fill all mbufs with uio data and update header information. */ 1758163915Sandre for (mb = m; mb != NULL; mb = mb->m_next) { 1759163915Sandre length = min(M_TRAILINGSPACE(mb), total - progress); 1760163915Sandre 1761163915Sandre error = uiomove(mtod(mb, void *), length, uio); 1762163915Sandre if (error) { 1763163915Sandre m_freem(m); 1764163915Sandre return (NULL); 1765125296Ssilby } 1766163915Sandre 1767163915Sandre mb->m_len = length; 1768125296Ssilby progress += length; 1769163915Sandre if (flags & M_PKTHDR) 1770163915Sandre m->m_pkthdr.len += length; 1771125296Ssilby } 1772163915Sandre KASSERT(progress == total, ("%s: progress != total", __func__)); 1773163915Sandre 1774163915Sandre return (m); 1775125296Ssilby} 1776148552Ssam 1777148552Ssam/* 1778194667Sandre * Copy an mbuf chain into a uio limited by len if set. 1779194667Sandre */ 1780194667Sandreint 1781194667Sandrem_mbuftouio(struct uio *uio, struct mbuf *m, int len) 1782194667Sandre{ 1783194667Sandre int error, length, total; 1784194667Sandre int progress = 0; 1785194667Sandre 1786194667Sandre if (len > 0) 1787194667Sandre total = min(uio->uio_resid, len); 1788194667Sandre else 1789194667Sandre total = uio->uio_resid; 1790194667Sandre 1791194667Sandre /* Fill the uio with data from the mbufs. */ 1792194667Sandre for (; m != NULL; m = m->m_next) { 1793194667Sandre length = min(m->m_len, total - progress); 1794194667Sandre 1795194667Sandre error = uiomove(mtod(m, void *), length, uio); 1796194667Sandre if (error) 1797194667Sandre return (error); 1798194667Sandre 1799194667Sandre progress += length; 1800194667Sandre } 1801194667Sandre 1802194667Sandre return (0); 1803194667Sandre} 1804194667Sandre 1805194667Sandre/* 1806148552Ssam * Set the m_data pointer of a newly-allocated mbuf 1807148552Ssam * to place an object of the specified size at the 1808148552Ssam * end of the mbuf, longword aligned. 1809148552Ssam */ 1810148552Ssamvoid 1811148552Ssamm_align(struct mbuf *m, int len) 1812148552Ssam{ 1813148552Ssam int adjust; 1814148552Ssam 1815148552Ssam if (m->m_flags & M_EXT) 1816148552Ssam adjust = m->m_ext.ext_size - len; 1817148552Ssam else if (m->m_flags & M_PKTHDR) 1818148552Ssam adjust = MHLEN - len; 1819148552Ssam else 1820148552Ssam adjust = MLEN - len; 1821148552Ssam m->m_data += adjust &~ (sizeof(long)-1); 1822148552Ssam} 1823156756Ssam 1824156756Ssam/* 1825156756Ssam * Create a writable copy of the mbuf chain. While doing this 1826156756Ssam * we compact the chain with a goal of producing a chain with 1827156756Ssam * at most two mbufs. The second mbuf in this chain is likely 1828156756Ssam * to be a cluster. The primary purpose of this work is to create 1829156756Ssam * a writable packet for encryption, compression, etc. The 1830156756Ssam * secondary goal is to linearize the data so the data can be 1831156756Ssam * passed to crypto hardware in the most efficient manner possible. 1832156756Ssam */ 1833156756Ssamstruct mbuf * 1834156756Ssamm_unshare(struct mbuf *m0, int how) 1835156756Ssam{ 1836156756Ssam struct mbuf *m, *mprev; 1837156756Ssam struct mbuf *n, *mfirst, *mlast; 1838156756Ssam int len, off; 1839156756Ssam 1840156756Ssam mprev = NULL; 1841156756Ssam for (m = m0; m != NULL; m = mprev->m_next) { 1842156756Ssam /* 1843156756Ssam * Regular mbufs are ignored unless there's a cluster 1844156756Ssam * in front of it that we can use to coalesce. We do 1845156756Ssam * the latter mainly so later clusters can be coalesced 1846156756Ssam * also w/o having to handle them specially (i.e. convert 1847156756Ssam * mbuf+cluster -> cluster). This optimization is heavily 1848156756Ssam * influenced by the assumption that we're running over 1849156756Ssam * Ethernet where MCLBYTES is large enough that the max 1850156756Ssam * packet size will permit lots of coalescing into a 1851156756Ssam * single cluster. This in turn permits efficient 1852156756Ssam * crypto operations, especially when using hardware. 1853156756Ssam */ 1854156756Ssam if ((m->m_flags & M_EXT) == 0) { 1855156756Ssam if (mprev && (mprev->m_flags & M_EXT) && 1856156756Ssam m->m_len <= M_TRAILINGSPACE(mprev)) { 1857156756Ssam /* XXX: this ignores mbuf types */ 1858156756Ssam memcpy(mtod(mprev, caddr_t) + mprev->m_len, 1859156756Ssam mtod(m, caddr_t), m->m_len); 1860156756Ssam mprev->m_len += m->m_len; 1861156756Ssam mprev->m_next = m->m_next; /* unlink from chain */ 1862156756Ssam m_free(m); /* reclaim mbuf */ 1863156756Ssam#if 0 1864156756Ssam newipsecstat.ips_mbcoalesced++; 1865156756Ssam#endif 1866156756Ssam } else { 1867156756Ssam mprev = m; 1868156756Ssam } 1869156756Ssam continue; 1870156756Ssam } 1871156756Ssam /* 1872156756Ssam * Writable mbufs are left alone (for now). 1873156756Ssam */ 1874156756Ssam if (M_WRITABLE(m)) { 1875156756Ssam mprev = m; 1876156756Ssam continue; 1877156756Ssam } 1878156756Ssam 1879156756Ssam /* 1880156756Ssam * Not writable, replace with a copy or coalesce with 1881156756Ssam * the previous mbuf if possible (since we have to copy 1882156756Ssam * it anyway, we try to reduce the number of mbufs and 1883156756Ssam * clusters so that future work is easier). 1884156756Ssam */ 1885156756Ssam KASSERT(m->m_flags & M_EXT, ("m_flags 0x%x", m->m_flags)); 1886156756Ssam /* NB: we only coalesce into a cluster or larger */ 1887156756Ssam if (mprev != NULL && (mprev->m_flags & M_EXT) && 1888156756Ssam m->m_len <= M_TRAILINGSPACE(mprev)) { 1889156756Ssam /* XXX: this ignores mbuf types */ 1890156756Ssam memcpy(mtod(mprev, caddr_t) + mprev->m_len, 1891156756Ssam mtod(m, caddr_t), m->m_len); 1892156756Ssam mprev->m_len += m->m_len; 1893156756Ssam mprev->m_next = m->m_next; /* unlink from chain */ 1894156756Ssam m_free(m); /* reclaim mbuf */ 1895156756Ssam#if 0 1896156756Ssam newipsecstat.ips_clcoalesced++; 1897156756Ssam#endif 1898156756Ssam continue; 1899156756Ssam } 1900156756Ssam 1901156756Ssam /* 1902156756Ssam * Allocate new space to hold the copy... 1903156756Ssam */ 1904156756Ssam /* XXX why can M_PKTHDR be set past the first mbuf? */ 1905156756Ssam if (mprev == NULL && (m->m_flags & M_PKTHDR)) { 1906156756Ssam /* 1907156756Ssam * NB: if a packet header is present we must 1908156756Ssam * allocate the mbuf separately from any cluster 1909156756Ssam * because M_MOVE_PKTHDR will smash the data 1910156756Ssam * pointer and drop the M_EXT marker. 1911156756Ssam */ 1912156756Ssam MGETHDR(n, how, m->m_type); 1913156756Ssam if (n == NULL) { 1914156756Ssam m_freem(m0); 1915156756Ssam return (NULL); 1916156756Ssam } 1917156756Ssam M_MOVE_PKTHDR(n, m); 1918156756Ssam MCLGET(n, how); 1919156756Ssam if ((n->m_flags & M_EXT) == 0) { 1920156756Ssam m_free(n); 1921156756Ssam m_freem(m0); 1922156756Ssam return (NULL); 1923156756Ssam } 1924156756Ssam } else { 1925156756Ssam n = m_getcl(how, m->m_type, m->m_flags); 1926156756Ssam if (n == NULL) { 1927156756Ssam m_freem(m0); 1928156756Ssam return (NULL); 1929156756Ssam } 1930156756Ssam } 1931156756Ssam /* 1932156756Ssam * ... and copy the data. We deal with jumbo mbufs 1933156756Ssam * (i.e. m_len > MCLBYTES) by splitting them into 1934156756Ssam * clusters. We could just malloc a buffer and make 1935156756Ssam * it external but too many device drivers don't know 1936156756Ssam * how to break up the non-contiguous memory when 1937156756Ssam * doing DMA. 1938156756Ssam */ 1939156756Ssam len = m->m_len; 1940156756Ssam off = 0; 1941156756Ssam mfirst = n; 1942156756Ssam mlast = NULL; 1943156756Ssam for (;;) { 1944156756Ssam int cc = min(len, MCLBYTES); 1945156756Ssam memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, cc); 1946156756Ssam n->m_len = cc; 1947156756Ssam if (mlast != NULL) 1948156756Ssam mlast->m_next = n; 1949156756Ssam mlast = n; 1950156756Ssam#if 0 1951156756Ssam newipsecstat.ips_clcopied++; 1952156756Ssam#endif 1953156756Ssam 1954156756Ssam len -= cc; 1955156756Ssam if (len <= 0) 1956156756Ssam break; 1957156756Ssam off += cc; 1958156756Ssam 1959156756Ssam n = m_getcl(how, m->m_type, m->m_flags); 1960156756Ssam if (n == NULL) { 1961156756Ssam m_freem(mfirst); 1962156756Ssam m_freem(m0); 1963156756Ssam return (NULL); 1964156756Ssam } 1965156756Ssam } 1966156756Ssam n->m_next = m->m_next; 1967156756Ssam if (mprev == NULL) 1968156756Ssam m0 = mfirst; /* new head of chain */ 1969156756Ssam else 1970156756Ssam mprev->m_next = mfirst; /* replace old mbuf */ 1971156756Ssam m_free(m); /* release old mbuf */ 1972156756Ssam mprev = mfirst; 1973156756Ssam } 1974156756Ssam return (m0); 1975156756Ssam} 1976178674Sjulian 1977178674Sjulian#ifdef MBUF_PROFILING 1978178674Sjulian 1979178674Sjulian#define MP_BUCKETS 32 /* don't just change this as things may overflow.*/ 1980178674Sjulianstruct mbufprofile { 1981178700Sjulian uintmax_t wasted[MP_BUCKETS]; 1982178700Sjulian uintmax_t used[MP_BUCKETS]; 1983178700Sjulian uintmax_t segments[MP_BUCKETS]; 1984178674Sjulian} mbprof; 1985178674Sjulian 1986178674Sjulian#define MP_MAXDIGITS 21 /* strlen("16,000,000,000,000,000,000") == 21 */ 1987178674Sjulian#define MP_NUMLINES 6 1988178674Sjulian#define MP_NUMSPERLINE 16 1989178674Sjulian#define MP_EXTRABYTES 64 /* > strlen("used:\nwasted:\nsegments:\n") */ 1990178674Sjulian/* work out max space needed and add a bit of spare space too */ 1991178674Sjulian#define MP_MAXLINE ((MP_MAXDIGITS+1) * MP_NUMSPERLINE) 1992178674Sjulian#define MP_BUFSIZE ((MP_MAXLINE * MP_NUMLINES) + 1 + MP_EXTRABYTES) 1993178674Sjulian 1994178674Sjulianchar mbprofbuf[MP_BUFSIZE]; 1995178674Sjulian 1996178674Sjulianvoid 1997178674Sjulianm_profile(struct mbuf *m) 1998178674Sjulian{ 1999178674Sjulian int segments = 0; 2000178674Sjulian int used = 0; 2001178674Sjulian int wasted = 0; 2002178674Sjulian 2003178674Sjulian while (m) { 2004178674Sjulian segments++; 2005178674Sjulian used += m->m_len; 2006178674Sjulian if (m->m_flags & M_EXT) { 2007178674Sjulian wasted += MHLEN - sizeof(m->m_ext) + 2008178674Sjulian m->m_ext.ext_size - m->m_len; 2009178674Sjulian } else { 2010178674Sjulian if (m->m_flags & M_PKTHDR) 2011178674Sjulian wasted += MHLEN - m->m_len; 2012178674Sjulian else 2013178674Sjulian wasted += MLEN - m->m_len; 2014178674Sjulian } 2015178674Sjulian m = m->m_next; 2016178674Sjulian } 2017178674Sjulian /* be paranoid.. it helps */ 2018178674Sjulian if (segments > MP_BUCKETS - 1) 2019178674Sjulian segments = MP_BUCKETS - 1; 2020178674Sjulian if (used > 100000) 2021178674Sjulian used = 100000; 2022178674Sjulian if (wasted > 100000) 2023178674Sjulian wasted = 100000; 2024178674Sjulian /* store in the appropriate bucket */ 2025178674Sjulian /* don't bother locking. if it's slightly off, so what? */ 2026178674Sjulian mbprof.segments[segments]++; 2027178674Sjulian mbprof.used[fls(used)]++; 2028178674Sjulian mbprof.wasted[fls(wasted)]++; 2029178674Sjulian} 2030178674Sjulian 2031178674Sjulianstatic void 2032178674Sjulianmbprof_textify(void) 2033178674Sjulian{ 2034178674Sjulian int offset; 2035178674Sjulian char *c; 2036209390Sed uint64_t *p; 2037178674Sjulian 2038178674Sjulian 2039178674Sjulian p = &mbprof.wasted[0]; 2040178674Sjulian c = mbprofbuf; 2041178674Sjulian offset = snprintf(c, MP_MAXLINE + 10, 2042178674Sjulian "wasted:\n" 2043178700Sjulian "%ju %ju %ju %ju %ju %ju %ju %ju " 2044178700Sjulian "%ju %ju %ju %ju %ju %ju %ju %ju\n", 2045178674Sjulian p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], 2046178674Sjulian p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]); 2047178674Sjulian#ifdef BIG_ARRAY 2048178674Sjulian p = &mbprof.wasted[16]; 2049178674Sjulian c += offset; 2050178674Sjulian offset = snprintf(c, MP_MAXLINE, 2051178700Sjulian "%ju %ju %ju %ju %ju %ju %ju %ju " 2052178700Sjulian "%ju %ju %ju %ju %ju %ju %ju %ju\n", 2053178674Sjulian p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], 2054178674Sjulian p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]); 2055178674Sjulian#endif 2056178674Sjulian p = &mbprof.used[0]; 2057178674Sjulian c += offset; 2058178674Sjulian offset = snprintf(c, MP_MAXLINE + 10, 2059178674Sjulian "used:\n" 2060178700Sjulian "%ju %ju %ju %ju %ju %ju %ju %ju " 2061178700Sjulian "%ju %ju %ju %ju %ju %ju %ju %ju\n", 2062178674Sjulian p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], 2063178674Sjulian p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]); 2064178674Sjulian#ifdef BIG_ARRAY 2065178674Sjulian p = &mbprof.used[16]; 2066178674Sjulian c += offset; 2067178674Sjulian offset = snprintf(c, MP_MAXLINE, 2068178700Sjulian "%ju %ju %ju %ju %ju %ju %ju %ju " 2069178700Sjulian "%ju %ju %ju %ju %ju %ju %ju %ju\n", 2070178674Sjulian p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], 2071178674Sjulian p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]); 2072178674Sjulian#endif 2073178674Sjulian p = &mbprof.segments[0]; 2074178674Sjulian c += offset; 2075178674Sjulian offset = snprintf(c, MP_MAXLINE + 10, 2076178674Sjulian "segments:\n" 2077178700Sjulian "%ju %ju %ju %ju %ju %ju %ju %ju " 2078178700Sjulian "%ju %ju %ju %ju %ju %ju %ju %ju\n", 2079178674Sjulian p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], 2080178674Sjulian p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]); 2081178674Sjulian#ifdef BIG_ARRAY 2082178674Sjulian p = &mbprof.segments[16]; 2083178674Sjulian c += offset; 2084178674Sjulian offset = snprintf(c, MP_MAXLINE, 2085178700Sjulian "%ju %ju %ju %ju %ju %ju %ju %ju " 2086178700Sjulian "%ju %ju %ju %ju %ju %ju %ju %jju", 2087178674Sjulian p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], 2088178674Sjulian p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]); 2089178674Sjulian#endif 2090178674Sjulian} 2091178674Sjulian 2092178674Sjulianstatic int 2093178674Sjulianmbprof_handler(SYSCTL_HANDLER_ARGS) 2094178674Sjulian{ 2095178674Sjulian int error; 2096178674Sjulian 2097178674Sjulian mbprof_textify(); 2098178674Sjulian error = SYSCTL_OUT(req, mbprofbuf, strlen(mbprofbuf) + 1); 2099178674Sjulian return (error); 2100178674Sjulian} 2101178674Sjulian 2102178674Sjulianstatic int 2103178674Sjulianmbprof_clr_handler(SYSCTL_HANDLER_ARGS) 2104178674Sjulian{ 2105178674Sjulian int clear, error; 2106178674Sjulian 2107178674Sjulian clear = 0; 2108178674Sjulian error = sysctl_handle_int(oidp, &clear, 0, req); 2109178674Sjulian if (error || !req->newptr) 2110178674Sjulian return (error); 2111178674Sjulian 2112178674Sjulian if (clear) { 2113178674Sjulian bzero(&mbprof, sizeof(mbprof)); 2114178674Sjulian } 2115178674Sjulian 2116178674Sjulian return (error); 2117178674Sjulian} 2118178674Sjulian 2119178674Sjulian 2120178674SjulianSYSCTL_PROC(_kern_ipc, OID_AUTO, mbufprofile, CTLTYPE_STRING|CTLFLAG_RD, 2121178674Sjulian NULL, 0, mbprof_handler, "A", "mbuf profiling statistics"); 2122178674Sjulian 2123178674SjulianSYSCTL_PROC(_kern_ipc, OID_AUTO, mbufprofileclr, CTLTYPE_INT|CTLFLAG_RW, 2124178674Sjulian NULL, 0, mbprof_clr_handler, "I", "clear mbuf profiling statistics"); 2125178674Sjulian#endif 2126178674Sjulian 2127