1/*
2 * Copyright (c) 1999-2014 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */
29/* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
30/*
31 * Mach Operating System
32 * Copyright (c) 1987 Carnegie-Mellon University
33 * All rights reserved.  The CMU software License Agreement specifies
34 * the terms and conditions for use and redistribution.
35 */
36/*
37 * Copyright (c) 1994 NeXT Computer, Inc. All rights reserved.
38 *
39 * Copyright (c) 1982, 1986, 1988 Regents of the University of California.
40 * All rights reserved.
41 *
42 * Redistribution and use in source and binary forms, with or without
43 * modification, are permitted provided that the following conditions
44 * are met:
45 * 1. Redistributions of source code must retain the above copyright
46 *    notice, this list of conditions and the following disclaimer.
47 * 2. Redistributions in binary form must reproduce the above copyright
48 *    notice, this list of conditions and the following disclaimer in the
49 *    documentation and/or other materials provided with the distribution.
50 * 3. All advertising materials mentioning features or use of this software
51 *    must display the following acknowledgement:
52 *      This product includes software developed by the University of
53 *      California, Berkeley and its contributors.
54 * 4. Neither the name of the University nor the names of its contributors
55 *    may be used to endorse or promote products derived from this software
56 *    without specific prior written permission.
57 *
58 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
59 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
60 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
61 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
62 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
63 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
64 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
65 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
66 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
67 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
68 * SUCH DAMAGE.
69 *
70 *	@(#)mbuf.h	8.3 (Berkeley) 1/21/94
71 */
72/*
73 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
74 * support for mandatory and extensible security protections.  This notice
75 * is included in support of clause 2.2 (b) of the Apple Public License,
76 * Version 2.0.
77 */
78
79#ifndef	_SYS_MBUF_H_
80#define	_SYS_MBUF_H_
81
82#include <sys/cdefs.h>
83#include <sys/appleapiopts.h>
84
85#ifdef XNU_KERNEL_PRIVATE
86
87#include <sys/lock.h>
88#include <sys/queue.h>
89#include <machine/endian.h>
90/*
91 * Mbufs are of a single size, MSIZE (machine/param.h), which
92 * includes overhead.  An mbuf may add a single "mbuf cluster" of size
93 * MCLBYTES/MBIGCLBYTES/M16KCLBYTES (also in machine/param.h), which has
94 * no additional overhead and is used instead of the internal data area;
95 * this is done when at least MINCLSIZE of data must be stored.
96 */
97
98/*
99 * The following _MLEN and _MHLEN macros are private to xnu.  Private code
100 * that are outside of xnu must use the mbuf_get_{mlen,mhlen} routines since
101 * the sizes of the structures are dependent upon specific xnu configs.
102 */
103#define	_MLEN		(MSIZE - sizeof(struct m_hdr))	/* normal data len */
104#define	_MHLEN		(_MLEN - sizeof(struct pkthdr))	/* data len w/pkthdr */
105
106#define	NMBPBGSHIFT	(MBIGCLSHIFT - MSIZESHIFT)
107#define	NMBPBG		(1 << NMBPBGSHIFT)	/* # of mbufs per big cl */
108
109#define	NCLPBGSHIFT	(MBIGCLSHIFT - MCLSHIFT)
110#define	NCLPBG		(1 << NCLPBGSHIFT)	/* # of cl per big cl */
111
112#define	NMBPCLSHIFT	(NMBPBGSHIFT - NCLPBGSHIFT)
113#define	NMBPCL		(1 << NMBPCLSHIFT)	/* # of mbufs per cl */
114
115#define	NCLPJCLSHIFT	((M16KCLSHIFT - MBIGCLSHIFT) + NCLPBGSHIFT)
116#define	NCLPJCL		(1 << NCLPJCLSHIFT)	/* # of cl per jumbo cl */
117
118/*
119 * Macros for type conversion
120 * mtod(m,t) -	convert mbuf pointer to data pointer of correct type
121 * dtom(x) -	convert data pointer within mbuf to mbuf pointer (XXX)
122 */
123#define	mtod(m, t)	((t)m_mtod(m))
124#define	dtom(x)		m_dtom(x)
125
126/* header at beginning of each mbuf: */
127struct m_hdr {
128	struct mbuf	*mh_next;	/* next buffer in chain */
129	struct mbuf	*mh_nextpkt;	/* next chain in queue/record */
130	caddr_t		mh_data;	/* location of data */
131	int32_t		mh_len;		/* amount of data in this mbuf */
132	u_int16_t	mh_type;	/* type of data in this mbuf */
133	u_int16_t	mh_flags;	/* flags; see below */
134};
135
136/*
137 * Packet tag structure (see below for details).
138 */
139struct m_tag {
140	u_int64_t		m_tag_cookie;	/* Error checking */
141#ifndef __LP64__
142	u_int32_t		pad;		/* For structure alignment */
143#endif /* !__LP64__ */
144	SLIST_ENTRY(m_tag)	m_tag_link;	/* List of packet tags */
145	u_int16_t		m_tag_type;	/* Module specific type */
146	u_int16_t		m_tag_len;	/* Length of data */
147	u_int32_t		m_tag_id;	/* Module ID */
148};
149
150#define	M_TAG_ALIGN(len) \
151	(P2ROUNDUP(len, sizeof (u_int64_t)) + sizeof (struct m_tag))
152
153#define	M_TAG_VALID_PATTERN	0xfeedfacefeedfaceULL
154#define	M_TAG_FREE_PATTERN	0xdeadbeefdeadbeefULL
155
156/*
157 * Packet tag header structure (at the top of mbuf).  Pointers are
158 * 32-bit in ILP32; m_tag needs 64-bit alignment, hence padded.
159 */
160struct m_taghdr {
161#ifndef __LP64__
162	u_int32_t		pad;		/* For structure alignment */
163#endif /* !__LP64__ */
164	u_int64_t		refcnt;		/* Number of tags in this mbuf */
165};
166
167/*
168 * Driver auxiliary metadata tag (KERNEL_TAG_TYPE_DRVAUX).
169 */
170struct m_drvaux_tag {
171	u_int32_t	da_family;	/* IFNET_FAMILY values */
172	u_int32_t	da_subfamily;	/* IFNET_SUBFAMILY values */
173	u_int32_t	da_reserved;	/* for future */
174	u_int32_t	da_length;	/* length of following data */
175};
176
177/* Values for pftag_flags (16-bit wide) */
178#define	PF_TAG_GENERATED		0x1	/* pkt generated by PF */
179#define	PF_TAG_FRAGCACHE		0x2
180#define	PF_TAG_TRANSLATE_LOCALHOST	0x4
181#if PF_ECN
182#define	PF_TAG_HDR_INET			0x8	/* hdr points to IPv4 */
183#define	PF_TAG_HDR_INET6		0x10	/* hdr points to IPv6 */
184#endif /* PF_ECN */
185/*
186 * PF mbuf tag
187 */
188struct pf_mtag {
189	u_int16_t	pftag_flags;	/* PF_TAG flags */
190	u_int16_t	pftag_rtableid;	/* alternate routing table id */
191	u_int16_t	pftag_tag;
192	u_int16_t	pftag_routed;
193#if PF_ALTQ
194	u_int32_t	pftag_qid;
195#endif /* PF_ALTQ */
196#if PF_ECN
197	void		*pftag_hdr;	/* saved hdr pos in mbuf, for ECN */
198#endif /* PF_ECN */
199};
200
201/*
202 * TCP mbuf tag
203 */
204struct tcp_pktinfo {
205	union {
206		struct {
207			u_int32_t segsz;	/* segment size (actual MSS) */
208		} __tx;
209		struct {
210			u_int16_t lro_pktlen;	/* max seg size encountered */
211			u_int8_t  lro_npkts;	/* # of coalesced TCP pkts */
212			u_int8_t  lro_timediff;	/* time spent in LRO */
213		} __rx;
214	} __offload;
215	union {
216		u_int32_t	pri;		/* send msg priority */
217		u_int32_t	seq;		/* recv msg sequence # */
218	} __msgattr;
219#define tso_segsz	proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.segsz
220#define lro_pktlen	proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_pktlen
221#define lro_npkts	proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_npkts
222#define lro_elapsed	proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_timediff
223#define msg_pri		proto_mtag.__pr_u.tcp.tm_tcp.__msgattr.pri
224#define msg_seq		proto_mtag.__pr_u.tcp.tm_tcp.__msgattr.seq
225};
226
227/*
228 * MPTCP mbuf tag
229 */
230struct mptcp_pktinfo {
231	u_int64_t	mtpi_dsn;	/* MPTCP Data Sequence Number */
232	union {
233		u_int64_t	mtpi_dan;	/* MPTCP Data Ack Number */
234		struct {
235			u_int32_t mtpi_rel_seq;	/* Relative Seq Number */
236			u_int32_t mtpi_length;	/* Length of mapping */
237		} mtpi_subf;
238	};
239#define	mp_dsn		proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_dsn
240#define	mp_rseq		proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_rel_seq
241#define	mp_rlen		proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_length
242#define	mp_dack		proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_dan
243};
244
245/*
246 * TCP specific mbuf tag.  Note that the current implementation uses
247 * MPTCP metadata strictly between MPTCP and the TCP subflow layers,
248 * hence tm_tcp and tm_mptcp are mutually exclusive.  This also means
249 * that TCP messages functionality is currently incompatible with MPTCP.
250 */
251struct tcp_mtag {
252	union {
253		struct tcp_pktinfo	tm_tcp;		/* TCP and below */
254		struct mptcp_pktinfo	tm_mptcp;	/* MPTCP-TCP only */
255	};
256};
257
258/*
259 * Protocol specific mbuf tag (at most one protocol metadata per mbuf).
260 *
261 * Care must be taken to ensure that they are mutually exclusive, e.g.
262 * IPSec policy ID implies no TCP segment offload (which is fine given
263 * that the former is used on the virtual ipsec interface that does
264 * not advertise the TSO capability.)
265 */
266struct proto_mtag {
267	union {
268		struct tcp_mtag	tcp;		/* TCP specific */
269	} __pr_u;
270};
271
272/*
273 * NECP specific mbuf tag.
274 */
275struct necp_mtag {
276	uint32_t	necp_policy_id;
277	uint32_t	necp_last_interface_index;
278};
279
280/*
281 * Record/packet header in first mbuf of chain; valid only if M_PKTHDR set.
282 */
283struct	pkthdr {
284	struct	ifnet *rcvif;		/* rcv interface */
285	/* variables for ip and tcp reassembly */
286	void	*pkt_hdr;		/* pointer to packet header */
287	int32_t	len;			/* total packet length */
288	/* variables for hardware checksum */
289	/* Note: csum_flags is used for hardware checksum and VLAN */
290	u_int32_t csum_flags;		/* flags regarding checksum */
291	union {
292		struct {
293			u_int16_t val;	 /* checksum value */
294			u_int16_t start; /* checksum start offset */
295		} _csum_rx;
296#define	csum_rx_val	_csum_rx.val
297#define	csum_rx_start	_csum_rx.start
298		struct {
299			u_int16_t start; /* checksum start offset */
300			u_int16_t stuff; /* checksum stuff offset */
301		} _csum_tx;
302#define	csum_tx_start	_csum_tx.start
303#define	csum_tx_stuff	_csum_tx.stuff
304		u_int32_t csum_data;	/* data field used by csum routines */
305	};
306	u_int16_t vlan_tag;		/* VLAN tag, host byte order */
307	/*
308	 * Packet classifier info
309	 *
310	 * PKTF_FLOW_ID set means valid flow ID.  A non-zero flow ID value
311	 * means the packet has been classified by one of the flow sources.
312	 * It is also a prerequisite for flow control advisory, which is
313	 * enabled by additionally setting PKTF_FLOW_ADV.
314	 *
315	 * The protocol value is a best-effort representation of the payload.
316	 * It is opportunistically updated and used only for optimization.
317	 * It is not a substitute for parsing the protocol header(s); use it
318	 * only as a hint.
319	 *
320	 * If PKTF_IFAINFO is set, pkt_ifainfo contains one or both of the
321	 * indices of interfaces which own the source and/or destination
322	 * addresses of the packet.  For the local/loopback case (PKTF_LOOP),
323	 * both should be valid, and thus allows for the receiving end to
324	 * quickly determine the actual interfaces used by the the addresses;
325	 * they may not necessarily be the same or refer to the loopback
326	 * interface.  Otherwise, in the non-local/loopback case, the indices
327	 * are opportunistically set, and because of that only one may be set
328	 * (0 means the index has not been determined.)  In addition, the
329	 * interface address flags are also recorded.  This allows us to avoid
330	 * storing the corresponding {in,in6}_ifaddr in an mbuf tag.  Ideally
331	 * this would be a superset of {ia,ia6}_flags, but the namespaces are
332	 * overlapping at present, so we'll need a new set of values in future
333	 * to achieve this.  For now, we will just rely on the address family
334	 * related code paths examining this mbuf to interpret the flags.
335	 */
336	u_int8_t pkt_proto;		/* IPPROTO value */
337	u_int8_t pkt_flowsrc;		/* FLOWSRC values */
338	u_int32_t pkt_flowid;		/* flow ID */
339	u_int32_t pkt_flags;		/* PKTF flags (see below) */
340	u_int32_t pkt_svc;		/* MBUF_SVC value */
341	union {
342		struct {
343			u_int16_t src;		/* ifindex of src addr i/f */
344			u_int16_t src_flags;	/* src PKT_IFAIFF flags */
345			u_int16_t dst;		/* ifindex of dst addr i/f */
346			u_int16_t dst_flags;	/* dst PKT_IFAIFF flags */
347		} _pkt_iaif;
348#define	src_ifindex	_pkt_iaif.src
349#define	src_iff		_pkt_iaif.src_flags
350#define	dst_ifindex	_pkt_iaif.dst
351#define	dst_iff		_pkt_iaif.dst_flags
352		u_int64_t pkt_ifainfo;	/* data field used by ifainfo */
353	};
354#if MEASURE_BW
355	u_int64_t pkt_bwseq;		/* sequence # */
356#endif /* MEASURE_BW */
357	u_int64_t pkt_enqueue_ts;	/* enqueue time */
358	/*
359	 * Tags (external and built-in)
360	 */
361	SLIST_HEAD(packet_tags, m_tag) tags; /* list of external tags */
362	struct proto_mtag proto_mtag;	/* built-in protocol-specific tag */
363	struct pf_mtag	pf_mtag;	/* built-in PF tag */
364	struct necp_mtag necp_mtag; /* built-in NECP tag */
365	/*
366	 * Module private scratch space (32-bit aligned), currently 16-bytes
367	 * large.  Anything stored here is not guaranteed to survive across
368	 * modules.  This should be the penultimate structure right before
369	 * the red zone.  Add new fields above this.
370	 */
371	struct {
372		union {
373			u_int8_t	__mpriv8[16];
374			u_int16_t	__mpriv16[8];
375			struct {
376				union {
377					u_int8_t	__val8[4];
378					u_int16_t	__val16[2];
379					u_int32_t	__val32;
380				} __mpriv32_u;
381			}		__mpriv32[4];
382			u_int64_t	__mpriv64[2];
383		} __mpriv_u;
384	} pkt_mpriv __attribute__((aligned(4)));
385	u_int32_t redzone;		/* red zone */
386};
387
388/*
389 * Flow data source type.  A data source module is responsible for generating
390 * a unique flow ID and associating it to each data flow as pkt_flowid.
391 * This is required for flow control/advisory, as it allows the output queue
392 * to identify the data source object and inform that it can resume its
393 * transmission (in the event it was flow controlled.)
394 */
395#define	FLOWSRC_INPCB		1	/* flow ID generated by INPCB */
396#define	FLOWSRC_IFNET		2	/* flow ID generated by interface */
397#define	FLOWSRC_PF		3	/* flow ID generated by PF */
398
399/*
400 * Packet flags.  Unlike m_flags, all packet flags are copied along when
401 * copying m_pkthdr, i.e. no equivalent of M_COPYFLAGS here.  These flags
402 * (and other classifier info) will be cleared during DLIL input.
403 *
404 * Some notes about M_LOOP and PKTF_LOOP:
405 *
406 *    - M_LOOP flag is overloaded, and its use is discouraged.  Historically,
407 *	that flag was used by the KAME implementation for allowing certain
408 *	certain exceptions to be made in the IP6_EXTHDR_CHECK() logic; this
409 *	was originally meant to be set as the packet is looped back to the
410 *	system, and in some circumstances temporarily set in ip6_output().
411 *	Over time, this flag was used by the pre-output routines to indicate
412 *	to the DLIL frameout and output routines, that the packet may be
413 *	looped back to the system under the right conditions.  In addition,
414 *	this is an mbuf flag rather than an mbuf packet header flag.
415 *
416 *    - PKTF_LOOP is an mbuf packet header flag, which is set if and only
417 *	if the packet was looped back to the system.  This flag should be
418 *	used instead for newer code.
419 */
420#define	PKTF_FLOW_ID		0x1	/* pkt has valid flowid value */
421#define	PKTF_FLOW_ADV		0x2	/* pkt triggers local flow advisory */
422#define	PKTF_FLOW_LOCALSRC	0x4	/* pkt is locally originated  */
423#define	PKTF_FLOW_RAWSOCK	0x8	/* pkt locally generated by raw sock */
424#define	PKTF_PRIO_PRIVILEGED	0x10	/* packet priority is privileged */
425#define	PKTF_PROXY_DST		0x20	/* processed but not locally destined */
426#define	PKTF_INET_RESOLVE	0x40	/* IPv4 resolver packet */
427#define	PKTF_INET6_RESOLVE	0x80	/* IPv6 resolver packet */
428#define	PKTF_RESOLVE_RTR	0x100	/* pkt is for resolving router */
429#define	PKTF_SW_LRO_PKT		0x200	/* pkt is a large coalesced pkt */
430#define	PKTF_SW_LRO_DID_CSUM	0x400	/* IP and TCP checksums done by LRO */
431#define	PKTF_MPTCP		0x800	/* TCP with MPTCP metadata */
432#define	PKTF_MPSO		0x1000	/* MPTCP socket meta data */
433#define	PKTF_LOOP		0x2000	/* loopbacked packet */
434#define	PKTF_IFAINFO		0x4000	/* pkt has valid interface addr info */
435#define	PKTF_SO_BACKGROUND	0x8000	/* data is from background source */
436#define	PKTF_FORWARDED		0x10000	/* pkt was forwarded from another i/f */
437#define	PKTF_PRIV_GUARDED	0x20000	/* pkt_mpriv area guard enabled */
438#define	PKTF_KEEPALIVE		0x40000	/* pkt is kernel-generated keepalive */
439/* flags related to flow control/advisory and identification */
440#define	PKTF_FLOW_MASK	\
441	(PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC | PKTF_FLOW_RAWSOCK)
442
443/*
444 * Description of external storage mapped into mbuf, valid only if M_EXT set.
445 */
446struct m_ext {
447	caddr_t	ext_buf;		/* start of buffer */
448	void	(*ext_free)		/* free routine if not the usual */
449		    (caddr_t, u_int, caddr_t);
450	u_int	ext_size;		/* size of buffer, for ext_free */
451	caddr_t	ext_arg;		/* additional ext_free argument */
452	struct	ext_refsq {		/* references held */
453		struct ext_refsq *forward, *backward;
454	} ext_refs;
455	struct ext_ref {
456		u_int32_t refcnt;
457		u_int32_t flags;
458	} *ext_refflags;
459};
460
461/* define m_ext to a type since it gets redefined below */
462typedef struct m_ext _m_ext_t;
463
464/*
465 * The mbuf object
466 */
467struct mbuf {
468	struct	m_hdr m_hdr;
469	union {
470		struct {
471			struct	pkthdr MH_pkthdr;	/* M_PKTHDR set */
472			union {
473				struct	m_ext MH_ext;	/* M_EXT set */
474				char	MH_databuf[_MHLEN];
475			} MH_dat;
476		} MH;
477		char	M_databuf[_MLEN];		/* !M_PKTHDR, !M_EXT */
478	} M_dat;
479};
480
481#define	m_next		m_hdr.mh_next
482#define	m_len		m_hdr.mh_len
483#define	m_data		m_hdr.mh_data
484#define	m_type		m_hdr.mh_type
485#define	m_flags		m_hdr.mh_flags
486#define	m_nextpkt	m_hdr.mh_nextpkt
487#define	m_act		m_nextpkt
488#define	m_pkthdr	M_dat.MH.MH_pkthdr
489#define	m_ext		M_dat.MH.MH_dat.MH_ext
490#define	m_pktdat	M_dat.MH.MH_dat.MH_databuf
491#define	m_dat		M_dat.M_databuf
492#define	m_pktlen(_m)	((_m)->m_pkthdr.len)
493#define	m_pftag(_m)	(&(_m)->m_pkthdr.pf_mtag)
494
495/* mbuf flags (private) */
496#define	M_EXT		0x0001	/* has associated external storage */
497#define	M_PKTHDR	0x0002	/* start of record */
498#define	M_EOR		0x0004	/* end of record */
499#define	M_PROTO1	0x0008	/* protocol-specific */
500#define	M_PROTO2	0x0010	/* protocol-specific */
501#define	M_PROTO3	0x0020	/* protocol-specific */
502#define	M_LOOP		0x0040	/* packet is looped back (also see PKTF_LOOP) */
503#define	M_PROTO5	0x0080	/* protocol-specific */
504
505/* mbuf pkthdr flags, also in m_flags (private) */
506#define	M_BCAST		0x0100	/* send/received as link-level broadcast */
507#define	M_MCAST		0x0200	/* send/received as link-level multicast */
508#define	M_FRAG		0x0400	/* packet is a fragment of a larger packet */
509#define	M_FIRSTFRAG	0x0800	/* packet is first fragment */
510#define	M_LASTFRAG	0x1000	/* packet is last fragment */
511#define	M_PROMISC	0x2000	/* packet is promiscuous (shouldn't go to stack) */
512#define	M_HASFCS	0x4000	/* packet has FCS */
513#define	M_TAGHDR	0x8000	/* m_tag hdr structure at top of mbuf data */
514
515/*
516 * Flags to purge when crossing layers.
517 */
518#define	M_PROTOFLAGS \
519	(M_PROTO1|M_PROTO2|M_PROTO3|M_PROTO5)
520
521/* flags copied when copying m_pkthdr */
522#define	M_COPYFLAGS							\
523	(M_PKTHDR|M_EOR|M_PROTO1|M_PROTO2|M_PROTO3 |			\
524	M_LOOP|M_PROTO5|M_BCAST|M_MCAST|M_FRAG |			\
525	M_FIRSTFRAG|M_LASTFRAG|M_PROMISC|M_HASFCS)
526
527/* flags indicating hw checksum support and sw checksum requirements */
528#define	CSUM_IP			0x0001		/* will csum IP */
529#define	CSUM_TCP		0x0002		/* will csum TCP */
530#define	CSUM_UDP		0x0004		/* will csum UDP */
531#define	CSUM_IP_FRAGS		0x0008		/* will csum IP fragments */
532#define	CSUM_FRAGMENT		0x0010		/* will do IP fragmentation */
533#define	CSUM_TCPIPV6		0x0020		/* will csum TCP for IPv6 */
534#define	CSUM_UDPIPV6		0x0040		/* will csum UDP for IPv6 */
535#define	CSUM_FRAGMENT_IPV6	0x0080		/* will do IPv6 fragmentation */
536
537#define	CSUM_IP_CHECKED		0x0100		/* did csum IP */
538#define	CSUM_IP_VALID		0x0200		/*   ... the csum is valid */
539#define	CSUM_DATA_VALID		0x0400		/* csum_data field is valid */
540#define	CSUM_PSEUDO_HDR		0x0800		/* csum_data has pseudo hdr */
541#define	CSUM_PARTIAL		0x1000		/* simple Sum16 computation */
542
543#define	CSUM_DELAY_DATA		(CSUM_TCP | CSUM_UDP)
544#define	CSUM_DELAY_IP		(CSUM_IP)	/* IPv4 only: no IPv6 IP cksum */
545#define	CSUM_DELAY_IPV6_DATA	(CSUM_TCPIPV6 | CSUM_UDPIPV6)
546#define	CSUM_DATA_IPV6_VALID    CSUM_DATA_VALID	/* csum_data field is valid */
547
548#define	CSUM_TX_FLAGS							\
549	(CSUM_DELAY_IP | CSUM_DELAY_DATA | CSUM_DELAY_IPV6_DATA |	\
550	CSUM_DATA_VALID | CSUM_PARTIAL)
551
552#define	CSUM_RX_FLAGS							\
553	(CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_PSEUDO_HDR |		\
554	CSUM_DATA_VALID | CSUM_PARTIAL)
555
556/*
557 * Note: see also IF_HWASSIST_CSUM defined in <net/if_var.h>
558 */
559
560/* VLAN tag present */
561#define	CSUM_VLAN_TAG_VALID	0x10000		/* vlan_tag field is valid */
562
563/* TCP Segment Offloading requested on this mbuf */
564#define	CSUM_TSO_IPV4		0x100000	/* This mbuf needs to be segmented by the NIC */
565#define	CSUM_TSO_IPV6		0x200000	/* This mbuf needs to be segmented by the NIC */
566
567#define	TSO_IPV4_OK(_ifp, _m)						\
568    (((_ifp)->if_hwassist & IFNET_TSO_IPV4) &&				\
569    ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4))			\
570
571#define	TSO_IPV4_NOTOK(_ifp, _m)					\
572    (!((_ifp)->if_hwassist & IFNET_TSO_IPV4) &&				\
573    ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4))			\
574
575#define	TSO_IPV6_OK(_ifp, _m)						\
576    (((_ifp)->if_hwassist & IFNET_TSO_IPV6) &&				\
577    ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6))			\
578
579#define	TSO_IPV6_NOTOK(_ifp, _m)					\
580    (!((_ifp)->if_hwassist & IFNET_TSO_IPV6) &&				\
581    ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6))			\
582
583#endif /* XNU_KERNEL_PRIVATE */
584
585/* mbuf types */
586#define	MT_FREE		0	/* should be on free list */
587#define	MT_DATA		1	/* dynamic (data) allocation */
588#define	MT_HEADER	2	/* packet header */
589#define	MT_SOCKET	3	/* socket structure */
590#define	MT_PCB		4	/* protocol control block */
591#define	MT_RTABLE	5	/* routing tables */
592#define	MT_HTABLE	6	/* IMP host tables */
593#define	MT_ATABLE	7	/* address resolution tables */
594#define	MT_SONAME	8	/* socket name */
595#define	MT_SOOPTS	10	/* socket options */
596#define	MT_FTABLE	11	/* fragment reassembly header */
597#define	MT_RIGHTS	12	/* access rights */
598#define	MT_IFADDR	13	/* interface address */
599#define	MT_CONTROL	14	/* extra-data protocol message */
600#define	MT_OOBDATA	15	/* expedited data  */
601#define	MT_TAG		16	/* volatile metadata associated to pkts */
602#define	MT_MAX		32	/* enough? */
603
604#ifdef XNU_KERNEL_PRIVATE
605/*
606 * mbuf allocation/deallocation macros:
607 *
608 *	MGET(struct mbuf *m, int how, int type)
609 * allocates an mbuf and initializes it to contain internal data.
610 *
611 *	MGETHDR(struct mbuf *m, int how, int type)
612 * allocates an mbuf and initializes it to contain a packet header
613 * and internal data.
614 */
615
616#if 1
617#define	MCHECK(m) m_mcheck(m)
618#else
619#define	MCHECK(m)
620#endif
621
622#define	MGET(m, how, type) ((m) = m_get((how), (type)))
623
624#define	MGETHDR(m, how, type)	((m) = m_gethdr((how), (type)))
625
626/*
627 * Mbuf cluster macros.
628 * MCLALLOC(caddr_t p, int how) allocates an mbuf cluster.
629 * MCLGET adds such clusters to a normal mbuf;
630 * the flag M_EXT is set upon success.
631 * MCLFREE releases a reference to a cluster allocated by MCLALLOC,
632 * freeing the cluster if the reference count has reached 0.
633 *
634 * Normal mbuf clusters are normally treated as character arrays
635 * after allocation, but use the first word of the buffer as a free list
636 * pointer while on the free list.
637 */
638union mcluster {
639	union	mcluster *mcl_next;
640	char	mcl_buf[MCLBYTES];
641};
642
643#define	MCLALLOC(p, how)	((p) = m_mclalloc(how))
644
645#define	MCLFREE(p)		m_mclfree(p)
646
647#define	MCLGET(m, how)		((m) = m_mclget(m, how))
648
649/*
650 * Mbuf big cluster
651 */
652union mbigcluster {
653	union mbigcluster	*mbc_next;
654	char			mbc_buf[MBIGCLBYTES];
655};
656
657/*
658 * Mbuf jumbo cluster
659 */
660union m16kcluster {
661	union m16kcluster	*m16kcl_next;
662	char			m16kcl_buf[M16KCLBYTES];
663};
664
665#define	MCLHASREFERENCE(m)	m_mclhasreference(m)
666
667/*
668 * MFREE(struct mbuf *m, struct mbuf *n)
669 * Free a single mbuf and associated external storage.
670 * Place the successor, if any, in n.
671 */
672
673#define	MFREE(m, n) ((n) = m_free(m))
674
675/*
676 * Copy mbuf pkthdr from from to to.
677 * from must have M_PKTHDR set, and to must be empty.
678 * aux pointer will be moved to `to'.
679 */
680#define	M_COPY_PKTHDR(to, from)		m_copy_pkthdr(to, from)
681
682#define M_COPY_PFTAG(to, from)		m_copy_pftag(to, from)
683
684#define	M_COPY_CLASSIFIER(to, from)	m_copy_classifier(to, from)
685
686/*
687 * Set the m_data pointer of a newly-allocated mbuf (m_get/MGET) to place
688 * an object of the specified size at the end of the mbuf, longword aligned.
689 */
690#define	M_ALIGN(m, len)							\
691do {									\
692	(m)->m_data += (MLEN - (len)) &~ (sizeof (long) - 1);		\
693} while (0)
694
695/*
696 * As above, for mbufs allocated with m_gethdr/MGETHDR
697 * or initialized by M_COPY_PKTHDR.
698 */
699#define	MH_ALIGN(m, len)						\
700do {									\
701	(m)->m_data += (MHLEN - (len)) &~ (sizeof (long) - 1);		\
702} while (0)
703
704/*
705 * Compute the amount of space available
706 * before the current start of data in an mbuf.
707 * Subroutine - data not available if certain references.
708 */
709#define	M_LEADINGSPACE(m)	m_leadingspace(m)
710
711/*
712 * Compute the amount of space available
713 * after the end of data in an mbuf.
714 * Subroutine - data not available if certain references.
715 */
716#define	M_TRAILINGSPACE(m)	m_trailingspace(m)
717
718/*
719 * Arrange to prepend space of size plen to mbuf m.
720 * If a new mbuf must be allocated, how specifies whether to wait.
721 * If how is M_DONTWAIT and allocation fails, the original mbuf chain
722 * is freed and m is set to NULL.
723 */
724#define	M_PREPEND(m, plen, how)	((m) = m_prepend_2((m), (plen), (how)))
725
726/* change mbuf to new type */
727#define	MCHTYPE(m, t)		m_mchtype(m, t)
728
729/* compatiblity with 4.3 */
730#define	m_copy(m, o, l)		m_copym((m), (o), (l), M_DONTWAIT)
731
732#define	MBSHIFT		20				/* 1MB */
733#define	MBSIZE		(1 << MBSHIFT)
734#define	GBSHIFT		30				/* 1GB */
735#define	GBSIZE		(1 << GBSHIFT)
736
737/*
738 * M_STRUCT_GET ensures that intermediate protocol header (from "off" to
739 * "off+len") is located in single mbuf, on contiguous memory region.
740 * The pointer to the region will be returned to pointer variable "val",
741 * with type "typ".
742 *
743 * M_STRUCT_GET0 does the same, except that it aligns the structure at
744 * very top of mbuf.  GET0 is likely to make memory copy than GET.
745 */
746#define	M_STRUCT_GET(val, typ, m, off, len)				\
747do {									\
748	struct mbuf *t;							\
749	int tmp;							\
750									\
751	if ((m)->m_len >= (off) + (len)) {				\
752		(val) = (typ)(mtod((m), caddr_t) + (off));		\
753	} else {							\
754		t = m_pulldown((m), (off), (len), &tmp);		\
755		if (t != NULL) {					\
756			if (t->m_len < tmp + (len))			\
757				panic("m_pulldown malfunction");	\
758			(val) = (typ)(mtod(t, caddr_t) + tmp);		\
759		} else {						\
760			(val) = (typ)NULL;				\
761			(m) = NULL;					\
762		}							\
763	}								\
764} while (0)
765
766#define	M_STRUCT_GET0(val, typ, m, off, len)				\
767do {									\
768	struct mbuf *t;							\
769									\
770	if ((off) == 0 && ((m)->m_len >= (len))) {			\
771		(val) = (typ)(void *)mtod(m, caddr_t);			\
772	} else {							\
773		t = m_pulldown((m), (off), (len), NULL);		\
774		if (t != NULL) {					\
775			if (t->m_len < (len))				\
776				panic("m_pulldown malfunction");	\
777			(val) = (typ)(void *)mtod(t, caddr_t);		\
778		} else {						\
779			(val) = (typ)NULL;				\
780			(m) = NULL;					\
781		}							\
782	}								\
783} while (0)
784
785#define	MBUF_INPUT_CHECK(m, rcvif)					\
786do {									\
787	if (!(m->m_flags & MBUF_PKTHDR) ||				\
788	    m->m_len < 0 ||						\
789	    m->m_len > ((njcl > 0) ? njclbytes : MBIGCLBYTES) ||	\
790	    m->m_type == MT_FREE ||					\
791	    ((m->m_flags & M_EXT) != 0 && m->m_ext.ext_buf == NULL)) {	\
792		panic_plain("Failed mbuf validity check: mbuf %p len %d "  \
793		    "type %d flags 0x%x data %p rcvif %s ifflags 0x%x",  \
794		    m, m->m_len, m->m_type, m->m_flags,			   \
795		    ((m->m_flags & M_EXT) ? m->m_ext.ext_buf : m->m_data), \
796		    if_name(rcvif),					\
797		    (rcvif->if_flags & 0xffff));			\
798	}								\
799} while (0)
800
801/*
802 * Simple mbuf queueing system
803 *
804 * This is basically a SIMPLEQ adapted to mbuf use (i.e. using
805 * m_nextpkt instead of field.sqe_next).
806 *
807 * m_next is ignored, so queueing chains of mbufs is possible
808 */
809#define MBUFQ_HEAD(name)					\
810struct name {							\
811	struct mbuf *mq_first;	/* first packet */		\
812	struct mbuf **mq_last;	/* addr of last next packet */	\
813}
814
815#define MBUFQ_INIT(q)		do {				\
816	MBUFQ_FIRST(q) = NULL;					\
817	(q)->mq_last = &MBUFQ_FIRST(q);				\
818} while (0)
819
820#define MBUFQ_PREPEND(q, m)	do {				\
821	if ((MBUFQ_NEXT(m) = MBUFQ_FIRST(q)) == NULL)		\
822		(q)->mq_last = &MBUFQ_NEXT(m);			\
823	MBUFQ_FIRST(q) = (m);					\
824} while (0)
825
826#define MBUFQ_ENQUEUE(q, m)	do {				\
827	MBUFQ_NEXT(m) = NULL;					\
828	*(q)->mq_last = (m);					\
829	(q)->mq_last = &MBUFQ_NEXT(m);				\
830} while (0)
831
832#define MBUFQ_ENQUEUE_MULTI(q, m, n)	do {			\
833	MBUFQ_NEXT(n) = NULL;					\
834	*(q)->mq_last = (m);					\
835	(q)->mq_last = &MBUFQ_NEXT(n);				\
836} while (0)
837
838#define MBUFQ_DEQUEUE(q, m)	do {				\
839	if (((m) = MBUFQ_FIRST(q)) != NULL) {			\
840		if ((MBUFQ_FIRST(q) = MBUFQ_NEXT(m)) == NULL)	\
841			(q)->mq_last = &MBUFQ_FIRST(q);		\
842		else						\
843			MBUFQ_NEXT(m) = NULL;			\
844	}							\
845} while (0)
846
847#define	MBUFQ_REMOVE(q, m)	do {				\
848	if (MBUFQ_FIRST(q) == (m)) {				\
849		MBUFQ_DEQUEUE(q, m);				\
850	} else {						\
851		struct mbuf *_m = MBUFQ_FIRST(q);		\
852		while (MBUFQ_NEXT(_m) != (m))			\
853			_m = MBUFQ_NEXT(_m);			\
854		if ((MBUFQ_NEXT(_m) =				\
855		    MBUFQ_NEXT(MBUFQ_NEXT(_m))) == NULL)	\
856			(q)->mq_last = &MBUFQ_NEXT(_m);		\
857	}							\
858} while (0)
859
860#define MBUFQ_DRAIN(q)		do {				\
861	struct mbuf *__m0;					\
862	while ((__m0 = MBUFQ_FIRST(q)) != NULL) {		\
863		MBUFQ_FIRST(q) = MBUFQ_NEXT(__m0);		\
864		MBUFQ_NEXT(__m0) = NULL;			\
865		m_freem(__m0);					\
866	}							\
867	(q)->mq_last = &MBUFQ_FIRST(q);				\
868} while (0)
869
870#define	MBUFQ_FOREACH(m, q)					\
871	for ((m) = MBUFQ_FIRST(q);				\
872	    (m);						\
873	    (m) = MBUFQ_NEXT(m))
874
875#define	MBUFQ_FOREACH_SAFE(m, q, tvar)				\
876	for ((m) = MBUFQ_FIRST(q);				\
877	    (m) && ((tvar) = MBUFQ_NEXT(m), 1);			\
878	    (m) = (tvar))
879
880#define	MBUFQ_EMPTY(q)		((q)->mq_first == NULL)
881#define MBUFQ_FIRST(q)		((q)->mq_first)
882#define MBUFQ_NEXT(m)		((m)->m_nextpkt)
883#define MBUFQ_LAST(q)		(*(q)->mq_last)
884
885#define	max_linkhdr	P2ROUNDUP(_max_linkhdr, sizeof (u_int32_t))
886#define	max_protohdr	P2ROUNDUP(_max_protohdr, sizeof (u_int32_t))
887#endif /* XNU_KERNEL_PRIVATE */
888
889/*
890 * Mbuf statistics (legacy).
891 */
892struct mbstat {
893	u_int32_t	m_mbufs;	/* mbufs obtained from page pool */
894	u_int32_t	m_clusters;	/* clusters obtained from page pool */
895	u_int32_t	m_spare;	/* spare field */
896	u_int32_t	m_clfree;	/* free clusters */
897	u_int32_t	m_drops;	/* times failed to find space */
898	u_int32_t	m_wait;		/* times waited for space */
899	u_int32_t	m_drain;	/* times drained protocols for space */
900	u_short		m_mtypes[256];	/* type specific mbuf allocations */
901	u_int32_t	m_mcfail;	/* times m_copym failed */
902	u_int32_t	m_mpfail;	/* times m_pullup failed */
903	u_int32_t	m_msize;	/* length of an mbuf */
904	u_int32_t	m_mclbytes;	/* length of an mbuf cluster */
905	u_int32_t	m_minclsize;	/* min length of data to allocate a cluster */
906	u_int32_t	m_mlen;		/* length of data in an mbuf */
907	u_int32_t	m_mhlen;	/* length of data in a header mbuf */
908	u_int32_t	m_bigclusters;	/* clusters obtained from page pool */
909	u_int32_t	m_bigclfree;	/* free clusters */
910	u_int32_t	m_bigmclbytes;	/* length of an mbuf cluster */
911};
912
913/* Compatibillity with 10.3 */
914struct ombstat {
915	u_int32_t	m_mbufs;	/* mbufs obtained from page pool */
916	u_int32_t	m_clusters;	/* clusters obtained from page pool */
917	u_int32_t	m_spare;	/* spare field */
918	u_int32_t	m_clfree;	/* free clusters */
919	u_int32_t	m_drops;	/* times failed to find space */
920	u_int32_t	m_wait;		/* times waited for space */
921	u_int32_t	m_drain;	/* times drained protocols for space */
922	u_short		m_mtypes[256];	/* type specific mbuf allocations */
923	u_int32_t	m_mcfail;	/* times m_copym failed */
924	u_int32_t	m_mpfail;	/* times m_pullup failed */
925	u_int32_t	m_msize;	/* length of an mbuf */
926	u_int32_t	m_mclbytes;	/* length of an mbuf cluster */
927	u_int32_t	m_minclsize;	/* min length of data to allocate a cluster */
928	u_int32_t	m_mlen;		/* length of data in an mbuf */
929	u_int32_t	m_mhlen;	/* length of data in a header mbuf */
930};
931
932/*
933 * mbuf class statistics.
934 */
935#define	MAX_MBUF_CNAME	15
936
937#if defined(XNU_KERNEL_PRIVATE)
938/* For backwards compatibility with 32-bit userland process */
939struct omb_class_stat {
940	char		mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */
941	u_int32_t	mbcl_size;	/* buffer size */
942	u_int32_t	mbcl_total;	/* # of buffers created */
943	u_int32_t	mbcl_active;	/* # of active buffers */
944	u_int32_t	mbcl_infree;	/* # of available buffers */
945	u_int32_t	mbcl_slab_cnt;	/* # of available slabs */
946	u_int64_t	mbcl_alloc_cnt;	/* # of times alloc is called */
947	u_int64_t	mbcl_free_cnt;	/* # of times free is called */
948	u_int64_t	mbcl_notified;	/* # of notified wakeups */
949	u_int64_t	mbcl_purge_cnt;	/* # of purges so far */
950	u_int64_t	mbcl_fail_cnt;	/* # of allocation failures */
951	u_int32_t	mbcl_ctotal;	/* total only for this class */
952	u_int32_t	mbcl_release_cnt; /* amount of memory returned */
953	/*
954	 * Cache layer statistics
955	 */
956	u_int32_t	mbcl_mc_state;	/* cache state (see below) */
957	u_int32_t	mbcl_mc_cached;	/* # of cached buffers */
958	u_int32_t	mbcl_mc_waiter_cnt;  /* # waiters on the cache */
959	u_int32_t	mbcl_mc_wretry_cnt;  /* # of wait retries */
960	u_int32_t	mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */
961	u_int64_t	mbcl_reserved[4];    /* for future use */
962} __attribute__((__packed__));
963#endif /* XNU_KERNEL_PRIVATE */
964
965typedef struct mb_class_stat {
966	char		mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */
967	u_int32_t	mbcl_size;	/* buffer size */
968	u_int32_t	mbcl_total;	/* # of buffers created */
969	u_int32_t	mbcl_active;	/* # of active buffers */
970	u_int32_t	mbcl_infree;	/* # of available buffers */
971	u_int32_t	mbcl_slab_cnt;	/* # of available slabs */
972#if defined(KERNEL) || defined(__LP64__)
973	u_int32_t	mbcl_pad;	/* padding */
974#endif /* KERNEL || __LP64__ */
975	u_int64_t	mbcl_alloc_cnt;	/* # of times alloc is called */
976	u_int64_t	mbcl_free_cnt;	/* # of times free is called */
977	u_int64_t	mbcl_notified;	/* # of notified wakeups */
978	u_int64_t	mbcl_purge_cnt;	/* # of purges so far */
979	u_int64_t	mbcl_fail_cnt;	/* # of allocation failures */
980	u_int32_t	mbcl_ctotal;	/* total only for this class */
981	u_int32_t	mbcl_release_cnt; /* amount of memory returned */
982	/*
983	 * Cache layer statistics
984	 */
985	u_int32_t	mbcl_mc_state;	/* cache state (see below) */
986	u_int32_t	mbcl_mc_cached;	/* # of cached buffers */
987	u_int32_t	mbcl_mc_waiter_cnt;  /* # waiters on the cache */
988	u_int32_t	mbcl_mc_wretry_cnt;  /* # of wait retries */
989	u_int32_t	mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */
990	u_int32_t	mbcl_peak_reported; /* last usage peak reported */
991	u_int32_t	mbcl_reserved[7];    /* for future use */
992} mb_class_stat_t;
993
994#define	MCS_DISABLED	0	/* cache is permanently disabled */
995#define	MCS_ONLINE	1	/* cache is online */
996#define	MCS_PURGING	2	/* cache is being purged */
997#define	MCS_OFFLINE	3	/* cache is offline (resizing) */
998
999#if defined(XNU_KERNEL_PRIVATE)
1000/* For backwards compatibility with 32-bit userland process */
1001struct omb_stat {
1002	u_int32_t		mbs_cnt;	/* number of classes */
1003	struct omb_class_stat	mbs_class[1];	/* class array */
1004} __attribute__((__packed__));
1005#endif /* XNU_KERNEL_PRIVATE */
1006
1007typedef struct mb_stat {
1008	u_int32_t	mbs_cnt;	/* number of classes */
1009#if defined(KERNEL) || defined(__LP64__)
1010	u_int32_t	mbs_pad;	/* padding */
1011#endif /* KERNEL || __LP64__ */
1012	mb_class_stat_t	mbs_class[1];	/* class array */
1013} mb_stat_t;
1014
1015#ifdef PRIVATE
1016#define	MLEAK_STACK_DEPTH	16	/* Max PC stack depth */
1017
1018typedef struct mleak_trace_stat {
1019	u_int64_t	mltr_collisions;
1020	u_int64_t	mltr_hitcount;
1021	u_int64_t	mltr_allocs;
1022	u_int64_t	mltr_depth;
1023	u_int64_t	mltr_addr[MLEAK_STACK_DEPTH];
1024} mleak_trace_stat_t;
1025
1026typedef struct mleak_stat {
1027	u_int32_t		ml_isaddr64;	/* 64-bit KVA? */
1028	u_int32_t		ml_cnt;		/* number of traces */
1029	mleak_trace_stat_t	ml_trace[1];	/* trace array */
1030} mleak_stat_t;
1031
1032struct mleak_table {
1033	u_int32_t mleak_capture;	/* sampling capture counter */
1034	u_int32_t mleak_sample_factor;	/* sample factor */
1035
1036	/* Times two active records want to occupy the same spot */
1037	u_int64_t alloc_collisions;
1038	u_int64_t trace_collisions;
1039
1040	/* Times new record lands on spot previously occupied by freed alloc */
1041	u_int64_t alloc_overwrites;
1042	u_int64_t trace_overwrites;
1043
1044	/* Times a new alloc or trace is put into the hash table */
1045	u_int64_t alloc_recorded;
1046	u_int64_t trace_recorded;
1047
1048	/* Total number of outstanding allocs */
1049	u_int64_t outstanding_allocs;
1050
1051	/* Times mleak_log returned false because couldn't acquire the lock */
1052	u_int64_t total_conflicts;
1053};
1054#endif /* PRIVATE */
1055
1056#ifdef KERNEL_PRIVATE
1057__BEGIN_DECLS
1058
1059/*
1060 * Exported (private)
1061 */
1062
1063extern struct mbstat mbstat;			/* statistics */
1064
1065__END_DECLS
1066#endif /* KERNEL_PRIVATE */
1067
1068#ifdef XNU_KERNEL_PRIVATE
1069__BEGIN_DECLS
1070
1071/*
1072 * Not exported (xnu private)
1073 */
1074
1075/* flags to m_get/MGET */
1076/* Need to include malloc.h to get right options for malloc  */
1077#include	<sys/malloc.h>
1078
1079struct mbuf;
1080
1081/* length to m_copy to copy all */
1082#define	M_COPYALL	1000000000
1083
1084#define	M_DONTWAIT	M_NOWAIT
1085#define	M_WAIT		M_WAITOK
1086
1087/* modes for m_copym and variants */
1088#define	M_COPYM_NOOP_HDR	0	/* don't copy/move pkthdr contents */
1089#define	M_COPYM_COPY_HDR	1	/* copy pkthdr from old to new */
1090#define	M_COPYM_MOVE_HDR	2	/* move pkthdr from old to new */
1091#define	M_COPYM_MUST_COPY_HDR	3	/* MUST copy pkthdr from old to new */
1092#define	M_COPYM_MUST_MOVE_HDR	4	/* MUST move pkthdr from old to new */
1093
1094/*
1095 * These macros are mapped to the appropriate KPIs, so that private code
1096 * can be simply recompiled in order to be forward-compatible with future
1097 * changes toward the struture sizes.
1098 */
1099#define	MLEN		mbuf_get_mlen()		/* normal data len */
1100#define	MHLEN		mbuf_get_mhlen()	/* data len w/pkthdr */
1101
1102#define	MINCLSIZE	mbuf_get_minclsize()	/* cluster usage threshold */
1103
1104extern void m_freem(struct mbuf *);
1105extern u_int64_t mcl_to_paddr(char *);
1106extern void m_adj(struct mbuf *, int);
1107extern void m_cat(struct mbuf *, struct mbuf *);
1108extern void m_copydata(struct mbuf *, int, int, void *);
1109extern struct mbuf *m_copym(struct mbuf *, int, int, int);
1110extern struct mbuf *m_copym_mode(struct mbuf *, int, int, int, uint32_t);
1111extern struct mbuf *m_get(int, int);
1112extern struct mbuf *m_gethdr(int, int);
1113extern struct mbuf *m_getpacket(void);
1114extern struct mbuf *m_getpackets(int, int, int);
1115extern struct mbuf *m_mclget(struct mbuf *, int);
1116extern void *m_mtod(struct mbuf *);
1117extern struct mbuf *m_prepend_2(struct mbuf *, int, int);
1118extern struct mbuf *m_pullup(struct mbuf *, int);
1119extern struct mbuf *m_split(struct mbuf *, int, int);
1120extern void m_mclfree(caddr_t p);
1121
1122/*
1123 * On platforms which require strict alignment (currently for anything but
1124 * i386 or x86_64), this macro checks whether the data pointer of an mbuf
1125 * is 32-bit aligned (this is the expected minimum alignment for protocol
1126 * headers), and assert otherwise.
1127 */
1128#if defined(__i386__) || defined(__x86_64__)
1129#define	MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m)
1130#else /* !__i386__ && !__x86_64__ */
1131#define	MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) do {			\
1132	if (!IS_P2ALIGNED((_m)->m_data, sizeof (u_int32_t))) {		\
1133		if (((_m)->m_flags & M_PKTHDR) &&			\
1134		    (_m)->m_pkthdr.rcvif != NULL) {			\
1135			panic_plain("\n%s: mbuf %p data ptr %p is not "	\
1136			    "32-bit aligned [%s: alignerrs=%lld]\n",	\
1137			    __func__, (_m), (_m)->m_data,		\
1138			    if_name((_m)->m_pkthdr.rcvif),		\
1139			    (_m)->m_pkthdr.rcvif->if_alignerrs);	\
1140		} else {						\
1141			panic_plain("\n%s: mbuf %p data ptr %p is not "	\
1142			    "32-bit aligned\n",				\
1143			    __func__, (_m), (_m)->m_data);		\
1144		}							\
1145	}								\
1146} while (0)
1147#endif /* !__i386__ && !__x86_64__ */
1148
1149/* Maximum number of MBUF_SC values (excluding MBUF_SC_UNSPEC) */
1150#define	MBUF_SC_MAX_CLASSES	10
1151
1152/*
1153 * These conversion macros rely on the corresponding MBUF_SC and
1154 * MBUF_TC values in order to establish the following mapping:
1155 *
1156 *	MBUF_SC_BK_SYS	] ==>	MBUF_TC_BK
1157 *	MBUF_SC_BK	]
1158 *
1159 *	MBUF_SC_BE	] ==>	MBUF_TC_BE
1160 *	MBUF_SC_RD	]
1161 *	MBUF_SC_OAM	]
1162 *
1163 *	MBUF_SC_AV	] ==>	MBUF_TC_VI
1164 *	MBUF_SC_RV	]
1165 *	MBUF_SC_VI	]
1166 *
1167 *	MBUF_SC_VO	] ==>	MBUF_TC_VO
1168 *	MBUF_SC_CTL	]
1169 *
1170 * The values assigned to each service class allows for a fast mapping to
1171 * the corresponding MBUF_TC traffic class values, as well as to retrieve the
1172 * assigned index; therefore care must be taken when comparing against these
1173 * values.  Use the corresponding class and index macros to retrieve the
1174 * corresponding portion, and never assume that a higher class corresponds
1175 * to a higher index.
1176 */
1177#define	MBUF_SCVAL(x)		((x) & 0xffff)
1178#define	MBUF_SCIDX(x)		((((x) >> 16) & 0xff) >> 3)
1179#define	MBUF_SC2TC(_sc)		(MBUF_SCVAL(_sc) >> 7)
1180#define	MBUF_TC2SCVAL(_tc)	((_tc) << 7)
1181#define IS_MBUF_SC_BACKGROUND(_sc) (((_sc) == MBUF_SC_BK_SYS) || \
1182	((_sc) == MBUF_SC_BK))
1183
1184#define	SCIDX_BK_SYS		MBUF_SCIDX(MBUF_SC_BK_SYS)
1185#define	SCIDX_BK		MBUF_SCIDX(MBUF_SC_BK)
1186#define	SCIDX_BE		MBUF_SCIDX(MBUF_SC_BE)
1187#define	SCIDX_RD		MBUF_SCIDX(MBUF_SC_RD)
1188#define	SCIDX_OAM		MBUF_SCIDX(MBUF_SC_OAM)
1189#define	SCIDX_AV		MBUF_SCIDX(MBUF_SC_AV)
1190#define	SCIDX_RV		MBUF_SCIDX(MBUF_SC_RV)
1191#define	SCIDX_VI		MBUF_SCIDX(MBUF_SC_VI)
1192#define	SCIDX_VO		MBUF_SCIDX(MBUF_SC_VO)
1193#define	SCIDX_CTL		MBUF_SCIDX(MBUF_SC_CTL)
1194
1195#define	SCVAL_BK_SYS		MBUF_SCVAL(MBUF_SC_BK_SYS)
1196#define	SCVAL_BK		MBUF_SCVAL(MBUF_SC_BK)
1197#define	SCVAL_BE		MBUF_SCVAL(MBUF_SC_BE)
1198#define	SCVAL_RD		MBUF_SCVAL(MBUF_SC_RD)
1199#define	SCVAL_OAM		MBUF_SCVAL(MBUF_SC_OAM)
1200#define	SCVAL_AV		MBUF_SCVAL(MBUF_SC_AV)
1201#define	SCVAL_RV		MBUF_SCVAL(MBUF_SC_RV)
1202#define	SCVAL_VI		MBUF_SCVAL(MBUF_SC_VI)
1203#define	SCVAL_VO		MBUF_SCVAL(MBUF_SC_VO)
1204#define	SCVAL_CTL		MBUF_SCVAL(MBUF_SC_CTL)
1205
1206#define	MBUF_VALID_SC(c)						\
1207	(c == MBUF_SC_BK_SYS || c == MBUF_SC_BK || c == MBUF_SC_BE ||	\
1208	c == MBUF_SC_RD || c == MBUF_SC_OAM || c == MBUF_SC_AV ||	\
1209	c == MBUF_SC_RV || c == MBUF_SC_VI || c == MBUF_SC_VO ||	\
1210	c == MBUF_SC_CTL)
1211
1212#define	MBUF_VALID_SCIDX(c)						\
1213	(c == SCIDX_BK_SYS || c == SCIDX_BK || c == SCIDX_BE ||		\
1214	c == SCIDX_RD || c == SCIDX_OAM || c == SCIDX_AV ||		\
1215	c == SCIDX_RV || c == SCIDX_VI || c == SCIDX_VO ||		\
1216	c == SCIDX_CTL)
1217
1218#define	MBUF_VALID_SCVAL(c)						\
1219	(c == SCVAL_BK_SYS || c == SCVAL_BK || c == SCVAL_BE ||		\
1220	c == SCVAL_RD || c == SCVAL_OAM || c == SCVAL_AV ||		\
1221	c == SCVAL_RV || c == SCVAL_VI || c == SCVAL_VO ||		\
1222	c == SCVAL_CTL)
1223
1224extern union mbigcluster *mbutl;	/* start VA of mbuf pool */
1225extern union mbigcluster *embutl;	/* end VA of mbuf pool */
1226extern unsigned int nmbclusters;	/* number of mapped clusters */
1227extern int njcl;		/* # of jumbo clusters  */
1228extern int njclbytes;	/* size of a jumbo cluster */
1229extern int max_hdr;		/* largest link+protocol header */
1230extern int max_datalen;	/* MHLEN - max_hdr */
1231
1232/* Use max_linkhdr instead of _max_linkhdr */
1233extern int _max_linkhdr;	/* largest link-level header */
1234
1235/* Use max_protohdr instead of _max_protohdr */
1236extern int _max_protohdr;	/* largest protocol header */
1237
1238__private_extern__ unsigned int mbuf_default_ncl(int, u_int64_t);
1239__private_extern__ void mbinit(void);
1240__private_extern__ struct mbuf *m_clattach(struct mbuf *, int, caddr_t,
1241    void (*)(caddr_t, u_int, caddr_t), u_int, caddr_t, int);
1242__private_extern__ caddr_t m_bigalloc(int);
1243__private_extern__ void m_bigfree(caddr_t, u_int, caddr_t);
1244__private_extern__ struct mbuf *m_mbigget(struct mbuf *, int);
1245__private_extern__ caddr_t m_16kalloc(int);
1246__private_extern__ void m_16kfree(caddr_t, u_int, caddr_t);
1247__private_extern__ struct mbuf *m_m16kget(struct mbuf *, int);
1248__private_extern__ int m_reinit(struct mbuf *, int);
1249__private_extern__ struct mbuf *m_free(struct mbuf *);
1250__private_extern__ struct mbuf *m_getclr(int, int);
1251__private_extern__ struct mbuf *m_getptr(struct mbuf *, int, int *);
1252__private_extern__ unsigned int m_length(struct mbuf *);
1253__private_extern__ unsigned int m_length2(struct mbuf *, struct mbuf **);
1254__private_extern__ unsigned int m_fixhdr(struct mbuf *);
1255__private_extern__ struct mbuf *m_defrag(struct mbuf *, int);
1256__private_extern__ struct mbuf *m_defrag_offset(struct mbuf *, u_int32_t, int);
1257__private_extern__ struct mbuf *m_prepend(struct mbuf *, int, int);
1258__private_extern__ struct mbuf *m_copyup(struct mbuf *, int, int);
1259__private_extern__ struct mbuf *m_retry(int, int);
1260__private_extern__ struct mbuf *m_retryhdr(int, int);
1261__private_extern__ int m_freem_list(struct mbuf *);
1262__private_extern__ int m_append(struct mbuf *, int, caddr_t);
1263__private_extern__ struct mbuf *m_last(struct mbuf *);
1264__private_extern__ struct mbuf *m_devget(char *, int, int, struct ifnet *,
1265    void (*)(const void *, void *, size_t));
1266__private_extern__ struct mbuf *m_pulldown(struct mbuf *, int, int, int *);
1267
1268__private_extern__ struct mbuf *m_getcl(int, int, int);
1269__private_extern__ caddr_t m_mclalloc(int);
1270__private_extern__ int m_mclhasreference(struct mbuf *);
1271__private_extern__ void m_copy_pkthdr(struct mbuf *, struct mbuf *);
1272__private_extern__ void m_copy_pftag(struct mbuf *, struct mbuf *);
1273__private_extern__ void m_copy_classifier(struct mbuf *, struct mbuf *);
1274
1275__private_extern__ struct mbuf *m_dtom(void *);
1276__private_extern__ int m_mtocl(void *);
1277__private_extern__ union mcluster *m_cltom(int);
1278
1279__private_extern__ int m_trailingspace(struct mbuf *);
1280__private_extern__ int m_leadingspace(struct mbuf *);
1281
1282__private_extern__ struct mbuf *m_normalize(struct mbuf *m);
1283__private_extern__ void m_mchtype(struct mbuf *m, int t);
1284__private_extern__ void m_mcheck(struct mbuf *);
1285
1286__private_extern__ void m_copyback(struct mbuf *, int, int, const void *);
1287__private_extern__ struct mbuf *m_copyback_cow(struct mbuf *, int, int,
1288    const void *, int);
1289__private_extern__ int m_makewritable(struct mbuf **, int, int, int);
1290__private_extern__ struct mbuf *m_dup(struct mbuf *m, int how);
1291__private_extern__ struct mbuf *m_copym_with_hdrs(struct mbuf *, int, int, int,
1292    struct mbuf **, int *, uint32_t);
1293__private_extern__ struct mbuf *m_getpackethdrs(int, int);
1294__private_extern__ struct mbuf *m_getpacket_how(int);
1295__private_extern__ struct mbuf *m_getpackets_internal(unsigned int *, int,
1296    int, int, size_t);
1297__private_extern__ struct mbuf *m_allocpacket_internal(unsigned int *, size_t,
1298    unsigned int *, int, int, size_t);
1299
1300__private_extern__ void m_drain(void);
1301
1302/*
1303 * Packets may have annotations attached by affixing a list of "packet
1304 * tags" to the pkthdr structure.  Packet tags are dynamically allocated
1305 * semi-opaque data structures that have a fixed header (struct m_tag)
1306 * that specifies the size of the memory block and an <id,type> pair that
1307 * identifies it. The id identifies the module and the type identifies the
1308 * type of data for that module. The id of zero is reserved for the kernel.
1309 *
1310 * Note that the packet tag returned by m_tag_allocate has the default
1311 * memory alignment implemented by malloc.  To reference private data one
1312 * can use a construct like:
1313 *
1314 *      struct m_tag *mtag = m_tag_allocate(...);
1315 *      struct foo *p = (struct foo *)(mtag+1);
1316 *
1317 * if the alignment of struct m_tag is sufficient for referencing members
1318 * of struct foo.  Otherwise it is necessary to embed struct m_tag within
1319 * the private data structure to insure proper alignment; e.g.
1320 *
1321 *      struct foo {
1322 *              struct m_tag    tag;
1323 *              ...
1324 *      };
1325 *      struct foo *p = (struct foo *) m_tag_allocate(...);
1326 *      struct m_tag *mtag = &p->tag;
1327 */
1328
1329#define	KERNEL_MODULE_TAG_ID	0
1330
1331enum {
1332	KERNEL_TAG_TYPE_NONE			= 0,
1333	KERNEL_TAG_TYPE_DUMMYNET		= 1,
1334	KERNEL_TAG_TYPE_DIVERT			= 2,
1335	KERNEL_TAG_TYPE_IPFORWARD		= 3,
1336	KERNEL_TAG_TYPE_IPFILT			= 4,
1337	KERNEL_TAG_TYPE_MACLABEL		= 5,
1338	KERNEL_TAG_TYPE_MAC_POLICY_LABEL	= 6,
1339	KERNEL_TAG_TYPE_ENCAP			= 8,
1340	KERNEL_TAG_TYPE_INET6			= 9,
1341	KERNEL_TAG_TYPE_IPSEC			= 10,
1342	KERNEL_TAG_TYPE_DRVAUX			= 11,
1343};
1344
1345/* Packet tag routines */
1346__private_extern__ struct  m_tag *m_tag_alloc(u_int32_t, u_int16_t, int, int);
1347__private_extern__ struct  m_tag *m_tag_create(u_int32_t, u_int16_t, int, int,
1348	struct mbuf *);
1349__private_extern__ void m_tag_free(struct m_tag *);
1350__private_extern__ void m_tag_prepend(struct mbuf *, struct m_tag *);
1351__private_extern__ void m_tag_unlink(struct mbuf *, struct m_tag *);
1352__private_extern__ void m_tag_delete(struct mbuf *, struct m_tag *);
1353__private_extern__ void m_tag_delete_chain(struct mbuf *, struct m_tag *);
1354__private_extern__ struct m_tag *m_tag_locate(struct mbuf *, u_int32_t,
1355    u_int16_t, struct m_tag *);
1356__private_extern__ struct m_tag *m_tag_copy(struct m_tag *, int);
1357__private_extern__ int m_tag_copy_chain(struct mbuf *, struct mbuf *, int);
1358__private_extern__ void m_tag_init(struct mbuf *, int);
1359__private_extern__ struct  m_tag *m_tag_first(struct mbuf *);
1360__private_extern__ struct  m_tag *m_tag_next(struct mbuf *, struct m_tag *);
1361
1362__END_DECLS
1363#endif /* XNU_KERNEL_PRIVATE */
1364#ifdef KERNEL
1365#include <sys/kpi_mbuf.h>
1366#ifdef XNU_KERNEL_PRIVATE
1367__BEGIN_DECLS
1368
1369__private_extern__ void m_scratch_init(struct mbuf *);
1370__private_extern__ u_int32_t m_scratch_get(struct mbuf *, u_int8_t **);
1371
1372__private_extern__ void m_classifier_init(struct mbuf *, uint32_t);
1373
1374__private_extern__ int m_set_service_class(struct mbuf *, mbuf_svc_class_t);
1375__private_extern__ mbuf_svc_class_t m_get_service_class(struct mbuf *);
1376__private_extern__ mbuf_svc_class_t m_service_class_from_idx(u_int32_t);
1377__private_extern__ mbuf_svc_class_t m_service_class_from_val(u_int32_t);
1378__private_extern__ int m_set_traffic_class(struct mbuf *, mbuf_traffic_class_t);
1379__private_extern__ mbuf_traffic_class_t m_get_traffic_class(struct mbuf *);
1380
1381#define ADDCARRY(_x)  do {						\
1382	while (((_x) >> 16) != 0)					\
1383		(_x) = ((_x) >> 16) + ((_x) & 0xffff);			\
1384} while (0)
1385
1386__private_extern__ u_int16_t m_adj_sum16(struct mbuf *, u_int32_t,
1387    u_int32_t, u_int32_t);
1388__private_extern__ u_int16_t m_sum16(struct mbuf *, u_int32_t, u_int32_t);
1389
1390__END_DECLS
1391#endif /* XNU_KERNEL_PRIVATE */
1392#endif /* KERNEL */
1393#endif	/* !_SYS_MBUF_H_ */
1394