1/* 2 * Copyright (c) 1999-2014 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */ 29/* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */ 30/* 31 * Mach Operating System 32 * Copyright (c) 1987 Carnegie-Mellon University 33 * All rights reserved. The CMU software License Agreement specifies 34 * the terms and conditions for use and redistribution. 35 */ 36/* 37 * Copyright (c) 1994 NeXT Computer, Inc. All rights reserved. 38 * 39 * Copyright (c) 1982, 1986, 1988 Regents of the University of California. 40 * All rights reserved. 41 * 42 * Redistribution and use in source and binary forms, with or without 43 * modification, are permitted provided that the following conditions 44 * are met: 45 * 1. Redistributions of source code must retain the above copyright 46 * notice, this list of conditions and the following disclaimer. 47 * 2. Redistributions in binary form must reproduce the above copyright 48 * notice, this list of conditions and the following disclaimer in the 49 * documentation and/or other materials provided with the distribution. 50 * 3. All advertising materials mentioning features or use of this software 51 * must display the following acknowledgement: 52 * This product includes software developed by the University of 53 * California, Berkeley and its contributors. 54 * 4. Neither the name of the University nor the names of its contributors 55 * may be used to endorse or promote products derived from this software 56 * without specific prior written permission. 57 * 58 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 59 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 60 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 61 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 62 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 63 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 64 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 65 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 66 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 67 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 68 * SUCH DAMAGE. 69 * 70 * @(#)mbuf.h 8.3 (Berkeley) 1/21/94 71 */ 72/* 73 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce 74 * support for mandatory and extensible security protections. This notice 75 * is included in support of clause 2.2 (b) of the Apple Public License, 76 * Version 2.0. 77 */ 78 79#ifndef _SYS_MBUF_H_ 80#define _SYS_MBUF_H_ 81 82#include <sys/cdefs.h> 83#include <sys/appleapiopts.h> 84 85#ifdef XNU_KERNEL_PRIVATE 86 87#include <sys/lock.h> 88#include <sys/queue.h> 89#include <machine/endian.h> 90/* 91 * Mbufs are of a single size, MSIZE (machine/param.h), which 92 * includes overhead. An mbuf may add a single "mbuf cluster" of size 93 * MCLBYTES/MBIGCLBYTES/M16KCLBYTES (also in machine/param.h), which has 94 * no additional overhead and is used instead of the internal data area; 95 * this is done when at least MINCLSIZE of data must be stored. 96 */ 97 98/* 99 * The following _MLEN and _MHLEN macros are private to xnu. Private code 100 * that are outside of xnu must use the mbuf_get_{mlen,mhlen} routines since 101 * the sizes of the structures are dependent upon specific xnu configs. 102 */ 103#define _MLEN (MSIZE - sizeof(struct m_hdr)) /* normal data len */ 104#define _MHLEN (_MLEN - sizeof(struct pkthdr)) /* data len w/pkthdr */ 105 106#define NMBPBGSHIFT (MBIGCLSHIFT - MSIZESHIFT) 107#define NMBPBG (1 << NMBPBGSHIFT) /* # of mbufs per big cl */ 108 109#define NCLPBGSHIFT (MBIGCLSHIFT - MCLSHIFT) 110#define NCLPBG (1 << NCLPBGSHIFT) /* # of cl per big cl */ 111 112#define NMBPCLSHIFT (NMBPBGSHIFT - NCLPBGSHIFT) 113#define NMBPCL (1 << NMBPCLSHIFT) /* # of mbufs per cl */ 114 115#define NCLPJCLSHIFT ((M16KCLSHIFT - MBIGCLSHIFT) + NCLPBGSHIFT) 116#define NCLPJCL (1 << NCLPJCLSHIFT) /* # of cl per jumbo cl */ 117 118/* 119 * Macros for type conversion 120 * mtod(m,t) - convert mbuf pointer to data pointer of correct type 121 * dtom(x) - convert data pointer within mbuf to mbuf pointer (XXX) 122 */ 123#define mtod(m, t) ((t)m_mtod(m)) 124#define dtom(x) m_dtom(x) 125 126/* header at beginning of each mbuf: */ 127struct m_hdr { 128 struct mbuf *mh_next; /* next buffer in chain */ 129 struct mbuf *mh_nextpkt; /* next chain in queue/record */ 130 caddr_t mh_data; /* location of data */ 131 int32_t mh_len; /* amount of data in this mbuf */ 132 u_int16_t mh_type; /* type of data in this mbuf */ 133 u_int16_t mh_flags; /* flags; see below */ 134}; 135 136/* 137 * Packet tag structure (see below for details). 138 */ 139struct m_tag { 140 u_int64_t m_tag_cookie; /* Error checking */ 141#ifndef __LP64__ 142 u_int32_t pad; /* For structure alignment */ 143#endif /* !__LP64__ */ 144 SLIST_ENTRY(m_tag) m_tag_link; /* List of packet tags */ 145 u_int16_t m_tag_type; /* Module specific type */ 146 u_int16_t m_tag_len; /* Length of data */ 147 u_int32_t m_tag_id; /* Module ID */ 148}; 149 150#define M_TAG_ALIGN(len) \ 151 (P2ROUNDUP(len, sizeof (u_int64_t)) + sizeof (struct m_tag)) 152 153#define M_TAG_VALID_PATTERN 0xfeedfacefeedfaceULL 154#define M_TAG_FREE_PATTERN 0xdeadbeefdeadbeefULL 155 156/* 157 * Packet tag header structure (at the top of mbuf). Pointers are 158 * 32-bit in ILP32; m_tag needs 64-bit alignment, hence padded. 159 */ 160struct m_taghdr { 161#ifndef __LP64__ 162 u_int32_t pad; /* For structure alignment */ 163#endif /* !__LP64__ */ 164 u_int64_t refcnt; /* Number of tags in this mbuf */ 165}; 166 167/* 168 * Driver auxiliary metadata tag (KERNEL_TAG_TYPE_DRVAUX). 169 */ 170struct m_drvaux_tag { 171 u_int32_t da_family; /* IFNET_FAMILY values */ 172 u_int32_t da_subfamily; /* IFNET_SUBFAMILY values */ 173 u_int32_t da_reserved; /* for future */ 174 u_int32_t da_length; /* length of following data */ 175}; 176 177/* Values for pftag_flags (16-bit wide) */ 178#define PF_TAG_GENERATED 0x1 /* pkt generated by PF */ 179#define PF_TAG_FRAGCACHE 0x2 180#define PF_TAG_TRANSLATE_LOCALHOST 0x4 181#if PF_ECN 182#define PF_TAG_HDR_INET 0x8 /* hdr points to IPv4 */ 183#define PF_TAG_HDR_INET6 0x10 /* hdr points to IPv6 */ 184#endif /* PF_ECN */ 185/* 186 * PF mbuf tag 187 */ 188struct pf_mtag { 189 u_int16_t pftag_flags; /* PF_TAG flags */ 190 u_int16_t pftag_rtableid; /* alternate routing table id */ 191 u_int16_t pftag_tag; 192 u_int16_t pftag_routed; 193#if PF_ALTQ 194 u_int32_t pftag_qid; 195#endif /* PF_ALTQ */ 196#if PF_ECN 197 void *pftag_hdr; /* saved hdr pos in mbuf, for ECN */ 198#endif /* PF_ECN */ 199}; 200 201/* 202 * TCP mbuf tag 203 */ 204struct tcp_pktinfo { 205 union { 206 struct { 207 u_int32_t segsz; /* segment size (actual MSS) */ 208 } __tx; 209 struct { 210 u_int16_t lro_pktlen; /* max seg size encountered */ 211 u_int8_t lro_npkts; /* # of coalesced TCP pkts */ 212 u_int8_t lro_timediff; /* time spent in LRO */ 213 } __rx; 214 } __offload; 215 union { 216 u_int32_t pri; /* send msg priority */ 217 u_int32_t seq; /* recv msg sequence # */ 218 } __msgattr; 219#define tso_segsz proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.segsz 220#define lro_pktlen proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_pktlen 221#define lro_npkts proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_npkts 222#define lro_elapsed proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_timediff 223#define msg_pri proto_mtag.__pr_u.tcp.tm_tcp.__msgattr.pri 224#define msg_seq proto_mtag.__pr_u.tcp.tm_tcp.__msgattr.seq 225}; 226 227/* 228 * MPTCP mbuf tag 229 */ 230struct mptcp_pktinfo { 231 u_int64_t mtpi_dsn; /* MPTCP Data Sequence Number */ 232 union { 233 u_int64_t mtpi_dan; /* MPTCP Data Ack Number */ 234 struct { 235 u_int32_t mtpi_rel_seq; /* Relative Seq Number */ 236 u_int32_t mtpi_length; /* Length of mapping */ 237 } mtpi_subf; 238 }; 239#define mp_dsn proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_dsn 240#define mp_rseq proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_rel_seq 241#define mp_rlen proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_length 242#define mp_dack proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_dan 243}; 244 245/* 246 * TCP specific mbuf tag. Note that the current implementation uses 247 * MPTCP metadata strictly between MPTCP and the TCP subflow layers, 248 * hence tm_tcp and tm_mptcp are mutually exclusive. This also means 249 * that TCP messages functionality is currently incompatible with MPTCP. 250 */ 251struct tcp_mtag { 252 union { 253 struct tcp_pktinfo tm_tcp; /* TCP and below */ 254 struct mptcp_pktinfo tm_mptcp; /* MPTCP-TCP only */ 255 }; 256}; 257 258/* 259 * Protocol specific mbuf tag (at most one protocol metadata per mbuf). 260 * 261 * Care must be taken to ensure that they are mutually exclusive, e.g. 262 * IPSec policy ID implies no TCP segment offload (which is fine given 263 * that the former is used on the virtual ipsec interface that does 264 * not advertise the TSO capability.) 265 */ 266struct proto_mtag { 267 union { 268 struct tcp_mtag tcp; /* TCP specific */ 269 } __pr_u; 270}; 271 272/* 273 * NECP specific mbuf tag. 274 */ 275struct necp_mtag { 276 uint32_t necp_policy_id; 277 uint32_t necp_last_interface_index; 278}; 279 280/* 281 * Record/packet header in first mbuf of chain; valid only if M_PKTHDR set. 282 */ 283struct pkthdr { 284 struct ifnet *rcvif; /* rcv interface */ 285 /* variables for ip and tcp reassembly */ 286 void *pkt_hdr; /* pointer to packet header */ 287 int32_t len; /* total packet length */ 288 /* variables for hardware checksum */ 289 /* Note: csum_flags is used for hardware checksum and VLAN */ 290 u_int32_t csum_flags; /* flags regarding checksum */ 291 union { 292 struct { 293 u_int16_t val; /* checksum value */ 294 u_int16_t start; /* checksum start offset */ 295 } _csum_rx; 296#define csum_rx_val _csum_rx.val 297#define csum_rx_start _csum_rx.start 298 struct { 299 u_int16_t start; /* checksum start offset */ 300 u_int16_t stuff; /* checksum stuff offset */ 301 } _csum_tx; 302#define csum_tx_start _csum_tx.start 303#define csum_tx_stuff _csum_tx.stuff 304 u_int32_t csum_data; /* data field used by csum routines */ 305 }; 306 u_int16_t vlan_tag; /* VLAN tag, host byte order */ 307 /* 308 * Packet classifier info 309 * 310 * PKTF_FLOW_ID set means valid flow ID. A non-zero flow ID value 311 * means the packet has been classified by one of the flow sources. 312 * It is also a prerequisite for flow control advisory, which is 313 * enabled by additionally setting PKTF_FLOW_ADV. 314 * 315 * The protocol value is a best-effort representation of the payload. 316 * It is opportunistically updated and used only for optimization. 317 * It is not a substitute for parsing the protocol header(s); use it 318 * only as a hint. 319 * 320 * If PKTF_IFAINFO is set, pkt_ifainfo contains one or both of the 321 * indices of interfaces which own the source and/or destination 322 * addresses of the packet. For the local/loopback case (PKTF_LOOP), 323 * both should be valid, and thus allows for the receiving end to 324 * quickly determine the actual interfaces used by the the addresses; 325 * they may not necessarily be the same or refer to the loopback 326 * interface. Otherwise, in the non-local/loopback case, the indices 327 * are opportunistically set, and because of that only one may be set 328 * (0 means the index has not been determined.) In addition, the 329 * interface address flags are also recorded. This allows us to avoid 330 * storing the corresponding {in,in6}_ifaddr in an mbuf tag. Ideally 331 * this would be a superset of {ia,ia6}_flags, but the namespaces are 332 * overlapping at present, so we'll need a new set of values in future 333 * to achieve this. For now, we will just rely on the address family 334 * related code paths examining this mbuf to interpret the flags. 335 */ 336 u_int8_t pkt_proto; /* IPPROTO value */ 337 u_int8_t pkt_flowsrc; /* FLOWSRC values */ 338 u_int32_t pkt_flowid; /* flow ID */ 339 u_int32_t pkt_flags; /* PKTF flags (see below) */ 340 u_int32_t pkt_svc; /* MBUF_SVC value */ 341 union { 342 struct { 343 u_int16_t src; /* ifindex of src addr i/f */ 344 u_int16_t src_flags; /* src PKT_IFAIFF flags */ 345 u_int16_t dst; /* ifindex of dst addr i/f */ 346 u_int16_t dst_flags; /* dst PKT_IFAIFF flags */ 347 } _pkt_iaif; 348#define src_ifindex _pkt_iaif.src 349#define src_iff _pkt_iaif.src_flags 350#define dst_ifindex _pkt_iaif.dst 351#define dst_iff _pkt_iaif.dst_flags 352 u_int64_t pkt_ifainfo; /* data field used by ifainfo */ 353 }; 354#if MEASURE_BW 355 u_int64_t pkt_bwseq; /* sequence # */ 356#endif /* MEASURE_BW */ 357 u_int64_t pkt_enqueue_ts; /* enqueue time */ 358 /* 359 * Tags (external and built-in) 360 */ 361 SLIST_HEAD(packet_tags, m_tag) tags; /* list of external tags */ 362 struct proto_mtag proto_mtag; /* built-in protocol-specific tag */ 363 struct pf_mtag pf_mtag; /* built-in PF tag */ 364 struct necp_mtag necp_mtag; /* built-in NECP tag */ 365 /* 366 * Module private scratch space (32-bit aligned), currently 16-bytes 367 * large. Anything stored here is not guaranteed to survive across 368 * modules. This should be the penultimate structure right before 369 * the red zone. Add new fields above this. 370 */ 371 struct { 372 union { 373 u_int8_t __mpriv8[16]; 374 u_int16_t __mpriv16[8]; 375 struct { 376 union { 377 u_int8_t __val8[4]; 378 u_int16_t __val16[2]; 379 u_int32_t __val32; 380 } __mpriv32_u; 381 } __mpriv32[4]; 382 u_int64_t __mpriv64[2]; 383 } __mpriv_u; 384 } pkt_mpriv __attribute__((aligned(4))); 385 u_int32_t redzone; /* red zone */ 386}; 387 388/* 389 * Flow data source type. A data source module is responsible for generating 390 * a unique flow ID and associating it to each data flow as pkt_flowid. 391 * This is required for flow control/advisory, as it allows the output queue 392 * to identify the data source object and inform that it can resume its 393 * transmission (in the event it was flow controlled.) 394 */ 395#define FLOWSRC_INPCB 1 /* flow ID generated by INPCB */ 396#define FLOWSRC_IFNET 2 /* flow ID generated by interface */ 397#define FLOWSRC_PF 3 /* flow ID generated by PF */ 398 399/* 400 * Packet flags. Unlike m_flags, all packet flags are copied along when 401 * copying m_pkthdr, i.e. no equivalent of M_COPYFLAGS here. These flags 402 * (and other classifier info) will be cleared during DLIL input. 403 * 404 * Some notes about M_LOOP and PKTF_LOOP: 405 * 406 * - M_LOOP flag is overloaded, and its use is discouraged. Historically, 407 * that flag was used by the KAME implementation for allowing certain 408 * certain exceptions to be made in the IP6_EXTHDR_CHECK() logic; this 409 * was originally meant to be set as the packet is looped back to the 410 * system, and in some circumstances temporarily set in ip6_output(). 411 * Over time, this flag was used by the pre-output routines to indicate 412 * to the DLIL frameout and output routines, that the packet may be 413 * looped back to the system under the right conditions. In addition, 414 * this is an mbuf flag rather than an mbuf packet header flag. 415 * 416 * - PKTF_LOOP is an mbuf packet header flag, which is set if and only 417 * if the packet was looped back to the system. This flag should be 418 * used instead for newer code. 419 */ 420#define PKTF_FLOW_ID 0x1 /* pkt has valid flowid value */ 421#define PKTF_FLOW_ADV 0x2 /* pkt triggers local flow advisory */ 422#define PKTF_FLOW_LOCALSRC 0x4 /* pkt is locally originated */ 423#define PKTF_FLOW_RAWSOCK 0x8 /* pkt locally generated by raw sock */ 424#define PKTF_PRIO_PRIVILEGED 0x10 /* packet priority is privileged */ 425#define PKTF_PROXY_DST 0x20 /* processed but not locally destined */ 426#define PKTF_INET_RESOLVE 0x40 /* IPv4 resolver packet */ 427#define PKTF_INET6_RESOLVE 0x80 /* IPv6 resolver packet */ 428#define PKTF_RESOLVE_RTR 0x100 /* pkt is for resolving router */ 429#define PKTF_SW_LRO_PKT 0x200 /* pkt is a large coalesced pkt */ 430#define PKTF_SW_LRO_DID_CSUM 0x400 /* IP and TCP checksums done by LRO */ 431#define PKTF_MPTCP 0x800 /* TCP with MPTCP metadata */ 432#define PKTF_MPSO 0x1000 /* MPTCP socket meta data */ 433#define PKTF_LOOP 0x2000 /* loopbacked packet */ 434#define PKTF_IFAINFO 0x4000 /* pkt has valid interface addr info */ 435#define PKTF_SO_BACKGROUND 0x8000 /* data is from background source */ 436#define PKTF_FORWARDED 0x10000 /* pkt was forwarded from another i/f */ 437#define PKTF_PRIV_GUARDED 0x20000 /* pkt_mpriv area guard enabled */ 438#define PKTF_KEEPALIVE 0x40000 /* pkt is kernel-generated keepalive */ 439/* flags related to flow control/advisory and identification */ 440#define PKTF_FLOW_MASK \ 441 (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC | PKTF_FLOW_RAWSOCK) 442 443/* 444 * Description of external storage mapped into mbuf, valid only if M_EXT set. 445 */ 446struct m_ext { 447 caddr_t ext_buf; /* start of buffer */ 448 void (*ext_free) /* free routine if not the usual */ 449 (caddr_t, u_int, caddr_t); 450 u_int ext_size; /* size of buffer, for ext_free */ 451 caddr_t ext_arg; /* additional ext_free argument */ 452 struct ext_refsq { /* references held */ 453 struct ext_refsq *forward, *backward; 454 } ext_refs; 455 struct ext_ref { 456 u_int32_t refcnt; 457 u_int32_t flags; 458 } *ext_refflags; 459}; 460 461/* define m_ext to a type since it gets redefined below */ 462typedef struct m_ext _m_ext_t; 463 464/* 465 * The mbuf object 466 */ 467struct mbuf { 468 struct m_hdr m_hdr; 469 union { 470 struct { 471 struct pkthdr MH_pkthdr; /* M_PKTHDR set */ 472 union { 473 struct m_ext MH_ext; /* M_EXT set */ 474 char MH_databuf[_MHLEN]; 475 } MH_dat; 476 } MH; 477 char M_databuf[_MLEN]; /* !M_PKTHDR, !M_EXT */ 478 } M_dat; 479}; 480 481#define m_next m_hdr.mh_next 482#define m_len m_hdr.mh_len 483#define m_data m_hdr.mh_data 484#define m_type m_hdr.mh_type 485#define m_flags m_hdr.mh_flags 486#define m_nextpkt m_hdr.mh_nextpkt 487#define m_act m_nextpkt 488#define m_pkthdr M_dat.MH.MH_pkthdr 489#define m_ext M_dat.MH.MH_dat.MH_ext 490#define m_pktdat M_dat.MH.MH_dat.MH_databuf 491#define m_dat M_dat.M_databuf 492#define m_pktlen(_m) ((_m)->m_pkthdr.len) 493#define m_pftag(_m) (&(_m)->m_pkthdr.pf_mtag) 494 495/* mbuf flags (private) */ 496#define M_EXT 0x0001 /* has associated external storage */ 497#define M_PKTHDR 0x0002 /* start of record */ 498#define M_EOR 0x0004 /* end of record */ 499#define M_PROTO1 0x0008 /* protocol-specific */ 500#define M_PROTO2 0x0010 /* protocol-specific */ 501#define M_PROTO3 0x0020 /* protocol-specific */ 502#define M_LOOP 0x0040 /* packet is looped back (also see PKTF_LOOP) */ 503#define M_PROTO5 0x0080 /* protocol-specific */ 504 505/* mbuf pkthdr flags, also in m_flags (private) */ 506#define M_BCAST 0x0100 /* send/received as link-level broadcast */ 507#define M_MCAST 0x0200 /* send/received as link-level multicast */ 508#define M_FRAG 0x0400 /* packet is a fragment of a larger packet */ 509#define M_FIRSTFRAG 0x0800 /* packet is first fragment */ 510#define M_LASTFRAG 0x1000 /* packet is last fragment */ 511#define M_PROMISC 0x2000 /* packet is promiscuous (shouldn't go to stack) */ 512#define M_HASFCS 0x4000 /* packet has FCS */ 513#define M_TAGHDR 0x8000 /* m_tag hdr structure at top of mbuf data */ 514 515/* 516 * Flags to purge when crossing layers. 517 */ 518#define M_PROTOFLAGS \ 519 (M_PROTO1|M_PROTO2|M_PROTO3|M_PROTO5) 520 521/* flags copied when copying m_pkthdr */ 522#define M_COPYFLAGS \ 523 (M_PKTHDR|M_EOR|M_PROTO1|M_PROTO2|M_PROTO3 | \ 524 M_LOOP|M_PROTO5|M_BCAST|M_MCAST|M_FRAG | \ 525 M_FIRSTFRAG|M_LASTFRAG|M_PROMISC|M_HASFCS) 526 527/* flags indicating hw checksum support and sw checksum requirements */ 528#define CSUM_IP 0x0001 /* will csum IP */ 529#define CSUM_TCP 0x0002 /* will csum TCP */ 530#define CSUM_UDP 0x0004 /* will csum UDP */ 531#define CSUM_IP_FRAGS 0x0008 /* will csum IP fragments */ 532#define CSUM_FRAGMENT 0x0010 /* will do IP fragmentation */ 533#define CSUM_TCPIPV6 0x0020 /* will csum TCP for IPv6 */ 534#define CSUM_UDPIPV6 0x0040 /* will csum UDP for IPv6 */ 535#define CSUM_FRAGMENT_IPV6 0x0080 /* will do IPv6 fragmentation */ 536 537#define CSUM_IP_CHECKED 0x0100 /* did csum IP */ 538#define CSUM_IP_VALID 0x0200 /* ... the csum is valid */ 539#define CSUM_DATA_VALID 0x0400 /* csum_data field is valid */ 540#define CSUM_PSEUDO_HDR 0x0800 /* csum_data has pseudo hdr */ 541#define CSUM_PARTIAL 0x1000 /* simple Sum16 computation */ 542 543#define CSUM_DELAY_DATA (CSUM_TCP | CSUM_UDP) 544#define CSUM_DELAY_IP (CSUM_IP) /* IPv4 only: no IPv6 IP cksum */ 545#define CSUM_DELAY_IPV6_DATA (CSUM_TCPIPV6 | CSUM_UDPIPV6) 546#define CSUM_DATA_IPV6_VALID CSUM_DATA_VALID /* csum_data field is valid */ 547 548#define CSUM_TX_FLAGS \ 549 (CSUM_DELAY_IP | CSUM_DELAY_DATA | CSUM_DELAY_IPV6_DATA | \ 550 CSUM_DATA_VALID | CSUM_PARTIAL) 551 552#define CSUM_RX_FLAGS \ 553 (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_PSEUDO_HDR | \ 554 CSUM_DATA_VALID | CSUM_PARTIAL) 555 556/* 557 * Note: see also IF_HWASSIST_CSUM defined in <net/if_var.h> 558 */ 559 560/* VLAN tag present */ 561#define CSUM_VLAN_TAG_VALID 0x10000 /* vlan_tag field is valid */ 562 563/* TCP Segment Offloading requested on this mbuf */ 564#define CSUM_TSO_IPV4 0x100000 /* This mbuf needs to be segmented by the NIC */ 565#define CSUM_TSO_IPV6 0x200000 /* This mbuf needs to be segmented by the NIC */ 566 567#define TSO_IPV4_OK(_ifp, _m) \ 568 (((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \ 569 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \ 570 571#define TSO_IPV4_NOTOK(_ifp, _m) \ 572 (!((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \ 573 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \ 574 575#define TSO_IPV6_OK(_ifp, _m) \ 576 (((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \ 577 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \ 578 579#define TSO_IPV6_NOTOK(_ifp, _m) \ 580 (!((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \ 581 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \ 582 583#endif /* XNU_KERNEL_PRIVATE */ 584 585/* mbuf types */ 586#define MT_FREE 0 /* should be on free list */ 587#define MT_DATA 1 /* dynamic (data) allocation */ 588#define MT_HEADER 2 /* packet header */ 589#define MT_SOCKET 3 /* socket structure */ 590#define MT_PCB 4 /* protocol control block */ 591#define MT_RTABLE 5 /* routing tables */ 592#define MT_HTABLE 6 /* IMP host tables */ 593#define MT_ATABLE 7 /* address resolution tables */ 594#define MT_SONAME 8 /* socket name */ 595#define MT_SOOPTS 10 /* socket options */ 596#define MT_FTABLE 11 /* fragment reassembly header */ 597#define MT_RIGHTS 12 /* access rights */ 598#define MT_IFADDR 13 /* interface address */ 599#define MT_CONTROL 14 /* extra-data protocol message */ 600#define MT_OOBDATA 15 /* expedited data */ 601#define MT_TAG 16 /* volatile metadata associated to pkts */ 602#define MT_MAX 32 /* enough? */ 603 604#ifdef XNU_KERNEL_PRIVATE 605/* 606 * mbuf allocation/deallocation macros: 607 * 608 * MGET(struct mbuf *m, int how, int type) 609 * allocates an mbuf and initializes it to contain internal data. 610 * 611 * MGETHDR(struct mbuf *m, int how, int type) 612 * allocates an mbuf and initializes it to contain a packet header 613 * and internal data. 614 */ 615 616#if 1 617#define MCHECK(m) m_mcheck(m) 618#else 619#define MCHECK(m) 620#endif 621 622#define MGET(m, how, type) ((m) = m_get((how), (type))) 623 624#define MGETHDR(m, how, type) ((m) = m_gethdr((how), (type))) 625 626/* 627 * Mbuf cluster macros. 628 * MCLALLOC(caddr_t p, int how) allocates an mbuf cluster. 629 * MCLGET adds such clusters to a normal mbuf; 630 * the flag M_EXT is set upon success. 631 * MCLFREE releases a reference to a cluster allocated by MCLALLOC, 632 * freeing the cluster if the reference count has reached 0. 633 * 634 * Normal mbuf clusters are normally treated as character arrays 635 * after allocation, but use the first word of the buffer as a free list 636 * pointer while on the free list. 637 */ 638union mcluster { 639 union mcluster *mcl_next; 640 char mcl_buf[MCLBYTES]; 641}; 642 643#define MCLALLOC(p, how) ((p) = m_mclalloc(how)) 644 645#define MCLFREE(p) m_mclfree(p) 646 647#define MCLGET(m, how) ((m) = m_mclget(m, how)) 648 649/* 650 * Mbuf big cluster 651 */ 652union mbigcluster { 653 union mbigcluster *mbc_next; 654 char mbc_buf[MBIGCLBYTES]; 655}; 656 657/* 658 * Mbuf jumbo cluster 659 */ 660union m16kcluster { 661 union m16kcluster *m16kcl_next; 662 char m16kcl_buf[M16KCLBYTES]; 663}; 664 665#define MCLHASREFERENCE(m) m_mclhasreference(m) 666 667/* 668 * MFREE(struct mbuf *m, struct mbuf *n) 669 * Free a single mbuf and associated external storage. 670 * Place the successor, if any, in n. 671 */ 672 673#define MFREE(m, n) ((n) = m_free(m)) 674 675/* 676 * Copy mbuf pkthdr from from to to. 677 * from must have M_PKTHDR set, and to must be empty. 678 * aux pointer will be moved to `to'. 679 */ 680#define M_COPY_PKTHDR(to, from) m_copy_pkthdr(to, from) 681 682#define M_COPY_PFTAG(to, from) m_copy_pftag(to, from) 683 684#define M_COPY_CLASSIFIER(to, from) m_copy_classifier(to, from) 685 686/* 687 * Set the m_data pointer of a newly-allocated mbuf (m_get/MGET) to place 688 * an object of the specified size at the end of the mbuf, longword aligned. 689 */ 690#define M_ALIGN(m, len) \ 691do { \ 692 (m)->m_data += (MLEN - (len)) &~ (sizeof (long) - 1); \ 693} while (0) 694 695/* 696 * As above, for mbufs allocated with m_gethdr/MGETHDR 697 * or initialized by M_COPY_PKTHDR. 698 */ 699#define MH_ALIGN(m, len) \ 700do { \ 701 (m)->m_data += (MHLEN - (len)) &~ (sizeof (long) - 1); \ 702} while (0) 703 704/* 705 * Compute the amount of space available 706 * before the current start of data in an mbuf. 707 * Subroutine - data not available if certain references. 708 */ 709#define M_LEADINGSPACE(m) m_leadingspace(m) 710 711/* 712 * Compute the amount of space available 713 * after the end of data in an mbuf. 714 * Subroutine - data not available if certain references. 715 */ 716#define M_TRAILINGSPACE(m) m_trailingspace(m) 717 718/* 719 * Arrange to prepend space of size plen to mbuf m. 720 * If a new mbuf must be allocated, how specifies whether to wait. 721 * If how is M_DONTWAIT and allocation fails, the original mbuf chain 722 * is freed and m is set to NULL. 723 */ 724#define M_PREPEND(m, plen, how) ((m) = m_prepend_2((m), (plen), (how))) 725 726/* change mbuf to new type */ 727#define MCHTYPE(m, t) m_mchtype(m, t) 728 729/* compatiblity with 4.3 */ 730#define m_copy(m, o, l) m_copym((m), (o), (l), M_DONTWAIT) 731 732#define MBSHIFT 20 /* 1MB */ 733#define MBSIZE (1 << MBSHIFT) 734#define GBSHIFT 30 /* 1GB */ 735#define GBSIZE (1 << GBSHIFT) 736 737/* 738 * M_STRUCT_GET ensures that intermediate protocol header (from "off" to 739 * "off+len") is located in single mbuf, on contiguous memory region. 740 * The pointer to the region will be returned to pointer variable "val", 741 * with type "typ". 742 * 743 * M_STRUCT_GET0 does the same, except that it aligns the structure at 744 * very top of mbuf. GET0 is likely to make memory copy than GET. 745 */ 746#define M_STRUCT_GET(val, typ, m, off, len) \ 747do { \ 748 struct mbuf *t; \ 749 int tmp; \ 750 \ 751 if ((m)->m_len >= (off) + (len)) { \ 752 (val) = (typ)(mtod((m), caddr_t) + (off)); \ 753 } else { \ 754 t = m_pulldown((m), (off), (len), &tmp); \ 755 if (t != NULL) { \ 756 if (t->m_len < tmp + (len)) \ 757 panic("m_pulldown malfunction"); \ 758 (val) = (typ)(mtod(t, caddr_t) + tmp); \ 759 } else { \ 760 (val) = (typ)NULL; \ 761 (m) = NULL; \ 762 } \ 763 } \ 764} while (0) 765 766#define M_STRUCT_GET0(val, typ, m, off, len) \ 767do { \ 768 struct mbuf *t; \ 769 \ 770 if ((off) == 0 && ((m)->m_len >= (len))) { \ 771 (val) = (typ)(void *)mtod(m, caddr_t); \ 772 } else { \ 773 t = m_pulldown((m), (off), (len), NULL); \ 774 if (t != NULL) { \ 775 if (t->m_len < (len)) \ 776 panic("m_pulldown malfunction"); \ 777 (val) = (typ)(void *)mtod(t, caddr_t); \ 778 } else { \ 779 (val) = (typ)NULL; \ 780 (m) = NULL; \ 781 } \ 782 } \ 783} while (0) 784 785#define MBUF_INPUT_CHECK(m, rcvif) \ 786do { \ 787 if (!(m->m_flags & MBUF_PKTHDR) || \ 788 m->m_len < 0 || \ 789 m->m_len > ((njcl > 0) ? njclbytes : MBIGCLBYTES) || \ 790 m->m_type == MT_FREE || \ 791 ((m->m_flags & M_EXT) != 0 && m->m_ext.ext_buf == NULL)) { \ 792 panic_plain("Failed mbuf validity check: mbuf %p len %d " \ 793 "type %d flags 0x%x data %p rcvif %s ifflags 0x%x", \ 794 m, m->m_len, m->m_type, m->m_flags, \ 795 ((m->m_flags & M_EXT) ? m->m_ext.ext_buf : m->m_data), \ 796 if_name(rcvif), \ 797 (rcvif->if_flags & 0xffff)); \ 798 } \ 799} while (0) 800 801/* 802 * Simple mbuf queueing system 803 * 804 * This is basically a SIMPLEQ adapted to mbuf use (i.e. using 805 * m_nextpkt instead of field.sqe_next). 806 * 807 * m_next is ignored, so queueing chains of mbufs is possible 808 */ 809#define MBUFQ_HEAD(name) \ 810struct name { \ 811 struct mbuf *mq_first; /* first packet */ \ 812 struct mbuf **mq_last; /* addr of last next packet */ \ 813} 814 815#define MBUFQ_INIT(q) do { \ 816 MBUFQ_FIRST(q) = NULL; \ 817 (q)->mq_last = &MBUFQ_FIRST(q); \ 818} while (0) 819 820#define MBUFQ_PREPEND(q, m) do { \ 821 if ((MBUFQ_NEXT(m) = MBUFQ_FIRST(q)) == NULL) \ 822 (q)->mq_last = &MBUFQ_NEXT(m); \ 823 MBUFQ_FIRST(q) = (m); \ 824} while (0) 825 826#define MBUFQ_ENQUEUE(q, m) do { \ 827 MBUFQ_NEXT(m) = NULL; \ 828 *(q)->mq_last = (m); \ 829 (q)->mq_last = &MBUFQ_NEXT(m); \ 830} while (0) 831 832#define MBUFQ_ENQUEUE_MULTI(q, m, n) do { \ 833 MBUFQ_NEXT(n) = NULL; \ 834 *(q)->mq_last = (m); \ 835 (q)->mq_last = &MBUFQ_NEXT(n); \ 836} while (0) 837 838#define MBUFQ_DEQUEUE(q, m) do { \ 839 if (((m) = MBUFQ_FIRST(q)) != NULL) { \ 840 if ((MBUFQ_FIRST(q) = MBUFQ_NEXT(m)) == NULL) \ 841 (q)->mq_last = &MBUFQ_FIRST(q); \ 842 else \ 843 MBUFQ_NEXT(m) = NULL; \ 844 } \ 845} while (0) 846 847#define MBUFQ_REMOVE(q, m) do { \ 848 if (MBUFQ_FIRST(q) == (m)) { \ 849 MBUFQ_DEQUEUE(q, m); \ 850 } else { \ 851 struct mbuf *_m = MBUFQ_FIRST(q); \ 852 while (MBUFQ_NEXT(_m) != (m)) \ 853 _m = MBUFQ_NEXT(_m); \ 854 if ((MBUFQ_NEXT(_m) = \ 855 MBUFQ_NEXT(MBUFQ_NEXT(_m))) == NULL) \ 856 (q)->mq_last = &MBUFQ_NEXT(_m); \ 857 } \ 858} while (0) 859 860#define MBUFQ_DRAIN(q) do { \ 861 struct mbuf *__m0; \ 862 while ((__m0 = MBUFQ_FIRST(q)) != NULL) { \ 863 MBUFQ_FIRST(q) = MBUFQ_NEXT(__m0); \ 864 MBUFQ_NEXT(__m0) = NULL; \ 865 m_freem(__m0); \ 866 } \ 867 (q)->mq_last = &MBUFQ_FIRST(q); \ 868} while (0) 869 870#define MBUFQ_FOREACH(m, q) \ 871 for ((m) = MBUFQ_FIRST(q); \ 872 (m); \ 873 (m) = MBUFQ_NEXT(m)) 874 875#define MBUFQ_FOREACH_SAFE(m, q, tvar) \ 876 for ((m) = MBUFQ_FIRST(q); \ 877 (m) && ((tvar) = MBUFQ_NEXT(m), 1); \ 878 (m) = (tvar)) 879 880#define MBUFQ_EMPTY(q) ((q)->mq_first == NULL) 881#define MBUFQ_FIRST(q) ((q)->mq_first) 882#define MBUFQ_NEXT(m) ((m)->m_nextpkt) 883#define MBUFQ_LAST(q) (*(q)->mq_last) 884 885#define max_linkhdr P2ROUNDUP(_max_linkhdr, sizeof (u_int32_t)) 886#define max_protohdr P2ROUNDUP(_max_protohdr, sizeof (u_int32_t)) 887#endif /* XNU_KERNEL_PRIVATE */ 888 889/* 890 * Mbuf statistics (legacy). 891 */ 892struct mbstat { 893 u_int32_t m_mbufs; /* mbufs obtained from page pool */ 894 u_int32_t m_clusters; /* clusters obtained from page pool */ 895 u_int32_t m_spare; /* spare field */ 896 u_int32_t m_clfree; /* free clusters */ 897 u_int32_t m_drops; /* times failed to find space */ 898 u_int32_t m_wait; /* times waited for space */ 899 u_int32_t m_drain; /* times drained protocols for space */ 900 u_short m_mtypes[256]; /* type specific mbuf allocations */ 901 u_int32_t m_mcfail; /* times m_copym failed */ 902 u_int32_t m_mpfail; /* times m_pullup failed */ 903 u_int32_t m_msize; /* length of an mbuf */ 904 u_int32_t m_mclbytes; /* length of an mbuf cluster */ 905 u_int32_t m_minclsize; /* min length of data to allocate a cluster */ 906 u_int32_t m_mlen; /* length of data in an mbuf */ 907 u_int32_t m_mhlen; /* length of data in a header mbuf */ 908 u_int32_t m_bigclusters; /* clusters obtained from page pool */ 909 u_int32_t m_bigclfree; /* free clusters */ 910 u_int32_t m_bigmclbytes; /* length of an mbuf cluster */ 911}; 912 913/* Compatibillity with 10.3 */ 914struct ombstat { 915 u_int32_t m_mbufs; /* mbufs obtained from page pool */ 916 u_int32_t m_clusters; /* clusters obtained from page pool */ 917 u_int32_t m_spare; /* spare field */ 918 u_int32_t m_clfree; /* free clusters */ 919 u_int32_t m_drops; /* times failed to find space */ 920 u_int32_t m_wait; /* times waited for space */ 921 u_int32_t m_drain; /* times drained protocols for space */ 922 u_short m_mtypes[256]; /* type specific mbuf allocations */ 923 u_int32_t m_mcfail; /* times m_copym failed */ 924 u_int32_t m_mpfail; /* times m_pullup failed */ 925 u_int32_t m_msize; /* length of an mbuf */ 926 u_int32_t m_mclbytes; /* length of an mbuf cluster */ 927 u_int32_t m_minclsize; /* min length of data to allocate a cluster */ 928 u_int32_t m_mlen; /* length of data in an mbuf */ 929 u_int32_t m_mhlen; /* length of data in a header mbuf */ 930}; 931 932/* 933 * mbuf class statistics. 934 */ 935#define MAX_MBUF_CNAME 15 936 937#if defined(XNU_KERNEL_PRIVATE) 938/* For backwards compatibility with 32-bit userland process */ 939struct omb_class_stat { 940 char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */ 941 u_int32_t mbcl_size; /* buffer size */ 942 u_int32_t mbcl_total; /* # of buffers created */ 943 u_int32_t mbcl_active; /* # of active buffers */ 944 u_int32_t mbcl_infree; /* # of available buffers */ 945 u_int32_t mbcl_slab_cnt; /* # of available slabs */ 946 u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */ 947 u_int64_t mbcl_free_cnt; /* # of times free is called */ 948 u_int64_t mbcl_notified; /* # of notified wakeups */ 949 u_int64_t mbcl_purge_cnt; /* # of purges so far */ 950 u_int64_t mbcl_fail_cnt; /* # of allocation failures */ 951 u_int32_t mbcl_ctotal; /* total only for this class */ 952 u_int32_t mbcl_release_cnt; /* amount of memory returned */ 953 /* 954 * Cache layer statistics 955 */ 956 u_int32_t mbcl_mc_state; /* cache state (see below) */ 957 u_int32_t mbcl_mc_cached; /* # of cached buffers */ 958 u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */ 959 u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */ 960 u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */ 961 u_int64_t mbcl_reserved[4]; /* for future use */ 962} __attribute__((__packed__)); 963#endif /* XNU_KERNEL_PRIVATE */ 964 965typedef struct mb_class_stat { 966 char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */ 967 u_int32_t mbcl_size; /* buffer size */ 968 u_int32_t mbcl_total; /* # of buffers created */ 969 u_int32_t mbcl_active; /* # of active buffers */ 970 u_int32_t mbcl_infree; /* # of available buffers */ 971 u_int32_t mbcl_slab_cnt; /* # of available slabs */ 972#if defined(KERNEL) || defined(__LP64__) 973 u_int32_t mbcl_pad; /* padding */ 974#endif /* KERNEL || __LP64__ */ 975 u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */ 976 u_int64_t mbcl_free_cnt; /* # of times free is called */ 977 u_int64_t mbcl_notified; /* # of notified wakeups */ 978 u_int64_t mbcl_purge_cnt; /* # of purges so far */ 979 u_int64_t mbcl_fail_cnt; /* # of allocation failures */ 980 u_int32_t mbcl_ctotal; /* total only for this class */ 981 u_int32_t mbcl_release_cnt; /* amount of memory returned */ 982 /* 983 * Cache layer statistics 984 */ 985 u_int32_t mbcl_mc_state; /* cache state (see below) */ 986 u_int32_t mbcl_mc_cached; /* # of cached buffers */ 987 u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */ 988 u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */ 989 u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */ 990 u_int32_t mbcl_peak_reported; /* last usage peak reported */ 991 u_int32_t mbcl_reserved[7]; /* for future use */ 992} mb_class_stat_t; 993 994#define MCS_DISABLED 0 /* cache is permanently disabled */ 995#define MCS_ONLINE 1 /* cache is online */ 996#define MCS_PURGING 2 /* cache is being purged */ 997#define MCS_OFFLINE 3 /* cache is offline (resizing) */ 998 999#if defined(XNU_KERNEL_PRIVATE) 1000/* For backwards compatibility with 32-bit userland process */ 1001struct omb_stat { 1002 u_int32_t mbs_cnt; /* number of classes */ 1003 struct omb_class_stat mbs_class[1]; /* class array */ 1004} __attribute__((__packed__)); 1005#endif /* XNU_KERNEL_PRIVATE */ 1006 1007typedef struct mb_stat { 1008 u_int32_t mbs_cnt; /* number of classes */ 1009#if defined(KERNEL) || defined(__LP64__) 1010 u_int32_t mbs_pad; /* padding */ 1011#endif /* KERNEL || __LP64__ */ 1012 mb_class_stat_t mbs_class[1]; /* class array */ 1013} mb_stat_t; 1014 1015#ifdef PRIVATE 1016#define MLEAK_STACK_DEPTH 16 /* Max PC stack depth */ 1017 1018typedef struct mleak_trace_stat { 1019 u_int64_t mltr_collisions; 1020 u_int64_t mltr_hitcount; 1021 u_int64_t mltr_allocs; 1022 u_int64_t mltr_depth; 1023 u_int64_t mltr_addr[MLEAK_STACK_DEPTH]; 1024} mleak_trace_stat_t; 1025 1026typedef struct mleak_stat { 1027 u_int32_t ml_isaddr64; /* 64-bit KVA? */ 1028 u_int32_t ml_cnt; /* number of traces */ 1029 mleak_trace_stat_t ml_trace[1]; /* trace array */ 1030} mleak_stat_t; 1031 1032struct mleak_table { 1033 u_int32_t mleak_capture; /* sampling capture counter */ 1034 u_int32_t mleak_sample_factor; /* sample factor */ 1035 1036 /* Times two active records want to occupy the same spot */ 1037 u_int64_t alloc_collisions; 1038 u_int64_t trace_collisions; 1039 1040 /* Times new record lands on spot previously occupied by freed alloc */ 1041 u_int64_t alloc_overwrites; 1042 u_int64_t trace_overwrites; 1043 1044 /* Times a new alloc or trace is put into the hash table */ 1045 u_int64_t alloc_recorded; 1046 u_int64_t trace_recorded; 1047 1048 /* Total number of outstanding allocs */ 1049 u_int64_t outstanding_allocs; 1050 1051 /* Times mleak_log returned false because couldn't acquire the lock */ 1052 u_int64_t total_conflicts; 1053}; 1054#endif /* PRIVATE */ 1055 1056#ifdef KERNEL_PRIVATE 1057__BEGIN_DECLS 1058 1059/* 1060 * Exported (private) 1061 */ 1062 1063extern struct mbstat mbstat; /* statistics */ 1064 1065__END_DECLS 1066#endif /* KERNEL_PRIVATE */ 1067 1068#ifdef XNU_KERNEL_PRIVATE 1069__BEGIN_DECLS 1070 1071/* 1072 * Not exported (xnu private) 1073 */ 1074 1075/* flags to m_get/MGET */ 1076/* Need to include malloc.h to get right options for malloc */ 1077#include <sys/malloc.h> 1078 1079struct mbuf; 1080 1081/* length to m_copy to copy all */ 1082#define M_COPYALL 1000000000 1083 1084#define M_DONTWAIT M_NOWAIT 1085#define M_WAIT M_WAITOK 1086 1087/* modes for m_copym and variants */ 1088#define M_COPYM_NOOP_HDR 0 /* don't copy/move pkthdr contents */ 1089#define M_COPYM_COPY_HDR 1 /* copy pkthdr from old to new */ 1090#define M_COPYM_MOVE_HDR 2 /* move pkthdr from old to new */ 1091#define M_COPYM_MUST_COPY_HDR 3 /* MUST copy pkthdr from old to new */ 1092#define M_COPYM_MUST_MOVE_HDR 4 /* MUST move pkthdr from old to new */ 1093 1094/* 1095 * These macros are mapped to the appropriate KPIs, so that private code 1096 * can be simply recompiled in order to be forward-compatible with future 1097 * changes toward the struture sizes. 1098 */ 1099#define MLEN mbuf_get_mlen() /* normal data len */ 1100#define MHLEN mbuf_get_mhlen() /* data len w/pkthdr */ 1101 1102#define MINCLSIZE mbuf_get_minclsize() /* cluster usage threshold */ 1103 1104extern void m_freem(struct mbuf *); 1105extern u_int64_t mcl_to_paddr(char *); 1106extern void m_adj(struct mbuf *, int); 1107extern void m_cat(struct mbuf *, struct mbuf *); 1108extern void m_copydata(struct mbuf *, int, int, void *); 1109extern struct mbuf *m_copym(struct mbuf *, int, int, int); 1110extern struct mbuf *m_copym_mode(struct mbuf *, int, int, int, uint32_t); 1111extern struct mbuf *m_get(int, int); 1112extern struct mbuf *m_gethdr(int, int); 1113extern struct mbuf *m_getpacket(void); 1114extern struct mbuf *m_getpackets(int, int, int); 1115extern struct mbuf *m_mclget(struct mbuf *, int); 1116extern void *m_mtod(struct mbuf *); 1117extern struct mbuf *m_prepend_2(struct mbuf *, int, int); 1118extern struct mbuf *m_pullup(struct mbuf *, int); 1119extern struct mbuf *m_split(struct mbuf *, int, int); 1120extern void m_mclfree(caddr_t p); 1121 1122/* 1123 * On platforms which require strict alignment (currently for anything but 1124 * i386 or x86_64), this macro checks whether the data pointer of an mbuf 1125 * is 32-bit aligned (this is the expected minimum alignment for protocol 1126 * headers), and assert otherwise. 1127 */ 1128#if defined(__i386__) || defined(__x86_64__) 1129#define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) 1130#else /* !__i386__ && !__x86_64__ */ 1131#define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) do { \ 1132 if (!IS_P2ALIGNED((_m)->m_data, sizeof (u_int32_t))) { \ 1133 if (((_m)->m_flags & M_PKTHDR) && \ 1134 (_m)->m_pkthdr.rcvif != NULL) { \ 1135 panic_plain("\n%s: mbuf %p data ptr %p is not " \ 1136 "32-bit aligned [%s: alignerrs=%lld]\n", \ 1137 __func__, (_m), (_m)->m_data, \ 1138 if_name((_m)->m_pkthdr.rcvif), \ 1139 (_m)->m_pkthdr.rcvif->if_alignerrs); \ 1140 } else { \ 1141 panic_plain("\n%s: mbuf %p data ptr %p is not " \ 1142 "32-bit aligned\n", \ 1143 __func__, (_m), (_m)->m_data); \ 1144 } \ 1145 } \ 1146} while (0) 1147#endif /* !__i386__ && !__x86_64__ */ 1148 1149/* Maximum number of MBUF_SC values (excluding MBUF_SC_UNSPEC) */ 1150#define MBUF_SC_MAX_CLASSES 10 1151 1152/* 1153 * These conversion macros rely on the corresponding MBUF_SC and 1154 * MBUF_TC values in order to establish the following mapping: 1155 * 1156 * MBUF_SC_BK_SYS ] ==> MBUF_TC_BK 1157 * MBUF_SC_BK ] 1158 * 1159 * MBUF_SC_BE ] ==> MBUF_TC_BE 1160 * MBUF_SC_RD ] 1161 * MBUF_SC_OAM ] 1162 * 1163 * MBUF_SC_AV ] ==> MBUF_TC_VI 1164 * MBUF_SC_RV ] 1165 * MBUF_SC_VI ] 1166 * 1167 * MBUF_SC_VO ] ==> MBUF_TC_VO 1168 * MBUF_SC_CTL ] 1169 * 1170 * The values assigned to each service class allows for a fast mapping to 1171 * the corresponding MBUF_TC traffic class values, as well as to retrieve the 1172 * assigned index; therefore care must be taken when comparing against these 1173 * values. Use the corresponding class and index macros to retrieve the 1174 * corresponding portion, and never assume that a higher class corresponds 1175 * to a higher index. 1176 */ 1177#define MBUF_SCVAL(x) ((x) & 0xffff) 1178#define MBUF_SCIDX(x) ((((x) >> 16) & 0xff) >> 3) 1179#define MBUF_SC2TC(_sc) (MBUF_SCVAL(_sc) >> 7) 1180#define MBUF_TC2SCVAL(_tc) ((_tc) << 7) 1181#define IS_MBUF_SC_BACKGROUND(_sc) (((_sc) == MBUF_SC_BK_SYS) || \ 1182 ((_sc) == MBUF_SC_BK)) 1183 1184#define SCIDX_BK_SYS MBUF_SCIDX(MBUF_SC_BK_SYS) 1185#define SCIDX_BK MBUF_SCIDX(MBUF_SC_BK) 1186#define SCIDX_BE MBUF_SCIDX(MBUF_SC_BE) 1187#define SCIDX_RD MBUF_SCIDX(MBUF_SC_RD) 1188#define SCIDX_OAM MBUF_SCIDX(MBUF_SC_OAM) 1189#define SCIDX_AV MBUF_SCIDX(MBUF_SC_AV) 1190#define SCIDX_RV MBUF_SCIDX(MBUF_SC_RV) 1191#define SCIDX_VI MBUF_SCIDX(MBUF_SC_VI) 1192#define SCIDX_VO MBUF_SCIDX(MBUF_SC_VO) 1193#define SCIDX_CTL MBUF_SCIDX(MBUF_SC_CTL) 1194 1195#define SCVAL_BK_SYS MBUF_SCVAL(MBUF_SC_BK_SYS) 1196#define SCVAL_BK MBUF_SCVAL(MBUF_SC_BK) 1197#define SCVAL_BE MBUF_SCVAL(MBUF_SC_BE) 1198#define SCVAL_RD MBUF_SCVAL(MBUF_SC_RD) 1199#define SCVAL_OAM MBUF_SCVAL(MBUF_SC_OAM) 1200#define SCVAL_AV MBUF_SCVAL(MBUF_SC_AV) 1201#define SCVAL_RV MBUF_SCVAL(MBUF_SC_RV) 1202#define SCVAL_VI MBUF_SCVAL(MBUF_SC_VI) 1203#define SCVAL_VO MBUF_SCVAL(MBUF_SC_VO) 1204#define SCVAL_CTL MBUF_SCVAL(MBUF_SC_CTL) 1205 1206#define MBUF_VALID_SC(c) \ 1207 (c == MBUF_SC_BK_SYS || c == MBUF_SC_BK || c == MBUF_SC_BE || \ 1208 c == MBUF_SC_RD || c == MBUF_SC_OAM || c == MBUF_SC_AV || \ 1209 c == MBUF_SC_RV || c == MBUF_SC_VI || c == MBUF_SC_VO || \ 1210 c == MBUF_SC_CTL) 1211 1212#define MBUF_VALID_SCIDX(c) \ 1213 (c == SCIDX_BK_SYS || c == SCIDX_BK || c == SCIDX_BE || \ 1214 c == SCIDX_RD || c == SCIDX_OAM || c == SCIDX_AV || \ 1215 c == SCIDX_RV || c == SCIDX_VI || c == SCIDX_VO || \ 1216 c == SCIDX_CTL) 1217 1218#define MBUF_VALID_SCVAL(c) \ 1219 (c == SCVAL_BK_SYS || c == SCVAL_BK || c == SCVAL_BE || \ 1220 c == SCVAL_RD || c == SCVAL_OAM || c == SCVAL_AV || \ 1221 c == SCVAL_RV || c == SCVAL_VI || c == SCVAL_VO || \ 1222 c == SCVAL_CTL) 1223 1224extern union mbigcluster *mbutl; /* start VA of mbuf pool */ 1225extern union mbigcluster *embutl; /* end VA of mbuf pool */ 1226extern unsigned int nmbclusters; /* number of mapped clusters */ 1227extern int njcl; /* # of jumbo clusters */ 1228extern int njclbytes; /* size of a jumbo cluster */ 1229extern int max_hdr; /* largest link+protocol header */ 1230extern int max_datalen; /* MHLEN - max_hdr */ 1231 1232/* Use max_linkhdr instead of _max_linkhdr */ 1233extern int _max_linkhdr; /* largest link-level header */ 1234 1235/* Use max_protohdr instead of _max_protohdr */ 1236extern int _max_protohdr; /* largest protocol header */ 1237 1238__private_extern__ unsigned int mbuf_default_ncl(int, u_int64_t); 1239__private_extern__ void mbinit(void); 1240__private_extern__ struct mbuf *m_clattach(struct mbuf *, int, caddr_t, 1241 void (*)(caddr_t, u_int, caddr_t), u_int, caddr_t, int); 1242__private_extern__ caddr_t m_bigalloc(int); 1243__private_extern__ void m_bigfree(caddr_t, u_int, caddr_t); 1244__private_extern__ struct mbuf *m_mbigget(struct mbuf *, int); 1245__private_extern__ caddr_t m_16kalloc(int); 1246__private_extern__ void m_16kfree(caddr_t, u_int, caddr_t); 1247__private_extern__ struct mbuf *m_m16kget(struct mbuf *, int); 1248__private_extern__ int m_reinit(struct mbuf *, int); 1249__private_extern__ struct mbuf *m_free(struct mbuf *); 1250__private_extern__ struct mbuf *m_getclr(int, int); 1251__private_extern__ struct mbuf *m_getptr(struct mbuf *, int, int *); 1252__private_extern__ unsigned int m_length(struct mbuf *); 1253__private_extern__ unsigned int m_length2(struct mbuf *, struct mbuf **); 1254__private_extern__ unsigned int m_fixhdr(struct mbuf *); 1255__private_extern__ struct mbuf *m_defrag(struct mbuf *, int); 1256__private_extern__ struct mbuf *m_defrag_offset(struct mbuf *, u_int32_t, int); 1257__private_extern__ struct mbuf *m_prepend(struct mbuf *, int, int); 1258__private_extern__ struct mbuf *m_copyup(struct mbuf *, int, int); 1259__private_extern__ struct mbuf *m_retry(int, int); 1260__private_extern__ struct mbuf *m_retryhdr(int, int); 1261__private_extern__ int m_freem_list(struct mbuf *); 1262__private_extern__ int m_append(struct mbuf *, int, caddr_t); 1263__private_extern__ struct mbuf *m_last(struct mbuf *); 1264__private_extern__ struct mbuf *m_devget(char *, int, int, struct ifnet *, 1265 void (*)(const void *, void *, size_t)); 1266__private_extern__ struct mbuf *m_pulldown(struct mbuf *, int, int, int *); 1267 1268__private_extern__ struct mbuf *m_getcl(int, int, int); 1269__private_extern__ caddr_t m_mclalloc(int); 1270__private_extern__ int m_mclhasreference(struct mbuf *); 1271__private_extern__ void m_copy_pkthdr(struct mbuf *, struct mbuf *); 1272__private_extern__ void m_copy_pftag(struct mbuf *, struct mbuf *); 1273__private_extern__ void m_copy_classifier(struct mbuf *, struct mbuf *); 1274 1275__private_extern__ struct mbuf *m_dtom(void *); 1276__private_extern__ int m_mtocl(void *); 1277__private_extern__ union mcluster *m_cltom(int); 1278 1279__private_extern__ int m_trailingspace(struct mbuf *); 1280__private_extern__ int m_leadingspace(struct mbuf *); 1281 1282__private_extern__ struct mbuf *m_normalize(struct mbuf *m); 1283__private_extern__ void m_mchtype(struct mbuf *m, int t); 1284__private_extern__ void m_mcheck(struct mbuf *); 1285 1286__private_extern__ void m_copyback(struct mbuf *, int, int, const void *); 1287__private_extern__ struct mbuf *m_copyback_cow(struct mbuf *, int, int, 1288 const void *, int); 1289__private_extern__ int m_makewritable(struct mbuf **, int, int, int); 1290__private_extern__ struct mbuf *m_dup(struct mbuf *m, int how); 1291__private_extern__ struct mbuf *m_copym_with_hdrs(struct mbuf *, int, int, int, 1292 struct mbuf **, int *, uint32_t); 1293__private_extern__ struct mbuf *m_getpackethdrs(int, int); 1294__private_extern__ struct mbuf *m_getpacket_how(int); 1295__private_extern__ struct mbuf *m_getpackets_internal(unsigned int *, int, 1296 int, int, size_t); 1297__private_extern__ struct mbuf *m_allocpacket_internal(unsigned int *, size_t, 1298 unsigned int *, int, int, size_t); 1299 1300__private_extern__ void m_drain(void); 1301 1302/* 1303 * Packets may have annotations attached by affixing a list of "packet 1304 * tags" to the pkthdr structure. Packet tags are dynamically allocated 1305 * semi-opaque data structures that have a fixed header (struct m_tag) 1306 * that specifies the size of the memory block and an <id,type> pair that 1307 * identifies it. The id identifies the module and the type identifies the 1308 * type of data for that module. The id of zero is reserved for the kernel. 1309 * 1310 * Note that the packet tag returned by m_tag_allocate has the default 1311 * memory alignment implemented by malloc. To reference private data one 1312 * can use a construct like: 1313 * 1314 * struct m_tag *mtag = m_tag_allocate(...); 1315 * struct foo *p = (struct foo *)(mtag+1); 1316 * 1317 * if the alignment of struct m_tag is sufficient for referencing members 1318 * of struct foo. Otherwise it is necessary to embed struct m_tag within 1319 * the private data structure to insure proper alignment; e.g. 1320 * 1321 * struct foo { 1322 * struct m_tag tag; 1323 * ... 1324 * }; 1325 * struct foo *p = (struct foo *) m_tag_allocate(...); 1326 * struct m_tag *mtag = &p->tag; 1327 */ 1328 1329#define KERNEL_MODULE_TAG_ID 0 1330 1331enum { 1332 KERNEL_TAG_TYPE_NONE = 0, 1333 KERNEL_TAG_TYPE_DUMMYNET = 1, 1334 KERNEL_TAG_TYPE_DIVERT = 2, 1335 KERNEL_TAG_TYPE_IPFORWARD = 3, 1336 KERNEL_TAG_TYPE_IPFILT = 4, 1337 KERNEL_TAG_TYPE_MACLABEL = 5, 1338 KERNEL_TAG_TYPE_MAC_POLICY_LABEL = 6, 1339 KERNEL_TAG_TYPE_ENCAP = 8, 1340 KERNEL_TAG_TYPE_INET6 = 9, 1341 KERNEL_TAG_TYPE_IPSEC = 10, 1342 KERNEL_TAG_TYPE_DRVAUX = 11, 1343}; 1344 1345/* Packet tag routines */ 1346__private_extern__ struct m_tag *m_tag_alloc(u_int32_t, u_int16_t, int, int); 1347__private_extern__ struct m_tag *m_tag_create(u_int32_t, u_int16_t, int, int, 1348 struct mbuf *); 1349__private_extern__ void m_tag_free(struct m_tag *); 1350__private_extern__ void m_tag_prepend(struct mbuf *, struct m_tag *); 1351__private_extern__ void m_tag_unlink(struct mbuf *, struct m_tag *); 1352__private_extern__ void m_tag_delete(struct mbuf *, struct m_tag *); 1353__private_extern__ void m_tag_delete_chain(struct mbuf *, struct m_tag *); 1354__private_extern__ struct m_tag *m_tag_locate(struct mbuf *, u_int32_t, 1355 u_int16_t, struct m_tag *); 1356__private_extern__ struct m_tag *m_tag_copy(struct m_tag *, int); 1357__private_extern__ int m_tag_copy_chain(struct mbuf *, struct mbuf *, int); 1358__private_extern__ void m_tag_init(struct mbuf *, int); 1359__private_extern__ struct m_tag *m_tag_first(struct mbuf *); 1360__private_extern__ struct m_tag *m_tag_next(struct mbuf *, struct m_tag *); 1361 1362__END_DECLS 1363#endif /* XNU_KERNEL_PRIVATE */ 1364#ifdef KERNEL 1365#include <sys/kpi_mbuf.h> 1366#ifdef XNU_KERNEL_PRIVATE 1367__BEGIN_DECLS 1368 1369__private_extern__ void m_scratch_init(struct mbuf *); 1370__private_extern__ u_int32_t m_scratch_get(struct mbuf *, u_int8_t **); 1371 1372__private_extern__ void m_classifier_init(struct mbuf *, uint32_t); 1373 1374__private_extern__ int m_set_service_class(struct mbuf *, mbuf_svc_class_t); 1375__private_extern__ mbuf_svc_class_t m_get_service_class(struct mbuf *); 1376__private_extern__ mbuf_svc_class_t m_service_class_from_idx(u_int32_t); 1377__private_extern__ mbuf_svc_class_t m_service_class_from_val(u_int32_t); 1378__private_extern__ int m_set_traffic_class(struct mbuf *, mbuf_traffic_class_t); 1379__private_extern__ mbuf_traffic_class_t m_get_traffic_class(struct mbuf *); 1380 1381#define ADDCARRY(_x) do { \ 1382 while (((_x) >> 16) != 0) \ 1383 (_x) = ((_x) >> 16) + ((_x) & 0xffff); \ 1384} while (0) 1385 1386__private_extern__ u_int16_t m_adj_sum16(struct mbuf *, u_int32_t, 1387 u_int32_t, u_int32_t); 1388__private_extern__ u_int16_t m_sum16(struct mbuf *, u_int32_t, u_int32_t); 1389 1390__END_DECLS 1391#endif /* XNU_KERNEL_PRIVATE */ 1392#endif /* KERNEL */ 1393#endif /* !_SYS_MBUF_H_ */ 1394