1/*
2 * Copyright (c) 1999-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */
29/* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
30/*
31 * Mach Operating System
32 * Copyright (c) 1987 Carnegie-Mellon University
33 * All rights reserved. The CMU software License Agreement specifies
34 * the terms and conditions for use and redistribution.
35 */
36/*
37 * Copyright (c) 1994 NeXT Computer, Inc. All rights reserved.
38 *
39 * Copyright (c) 1982, 1986, 1988 Regents of the University of California.
40 * All rights reserved.
41 *
42 * Redistribution and use in source and binary forms, with or without
43 * modification, are permitted provided that the following conditions
44 * are met:
45 * 1. Redistributions of source code must retain the above copyright
46 * notice, this list of conditions and the following disclaimer.
47 * 2. Redistributions in binary form must reproduce the above copyright
48 * notice, this list of conditions and the following disclaimer in the
49 * documentation and/or other materials provided with the distribution.
50 * 3. All advertising materials mentioning features or use of this software
51 * must display the following acknowledgement:
52 * This product includes software developed by the University of
53 * California, Berkeley and its contributors.
54 * 4. Neither the name of the University nor the names of its contributors
55 * may be used to endorse or promote products derived from this software
56 * without specific prior written permission.
57 *
58 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
59 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
60 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
61 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
62 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
63 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
64 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
65 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
66 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
67 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
68 * SUCH DAMAGE.
69 *
70 * @(#)mbuf.h 8.3 (Berkeley) 1/21/94
71 */
72/*
73 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
74 * support for mandatory and extensible security protections. This notice
75 * is included in support of clause 2.2 (b) of the Apple Public License,
76 * Version 2.0.
77 */
78
79#ifndef _SYS_MBUF_H_
80#define _SYS_MBUF_H_
81
82#include <sys/appleapiopts.h>
83#include <sys/cdefs.h>
84#include <sys/_types/_u_int32_t.h> /* u_int32_t */
85#include <sys/_types/_u_int64_t.h> /* u_int64_t */
86#include <sys/_types/_u_short.h> /* u_short */
87
88#ifdef KERNEL
89#include <sys/kpi_mbuf.h>
90#endif
91
92#ifdef XNU_KERNEL_PRIVATE
93#include <sys/lock.h>
94#include <sys/queue.h>
95#include <machine/endian.h>
96/*
97 * Mbufs are of a single size, which includes overhead.
98 * An mbuf may add a single "mbuf cluster" of size
99 * MCLBYTES/MBIGCLBYTES/M16KCLBYTES (also in machine/param.h), which has
100 * no additional overhead and is used instead of the internal data area;
101 * this is done when at least MINCLSIZE of data must be stored.
102 */
103#if CONFIG_MBUF_MCACHE
104#define _MSIZESHIFT 8 /* 256 */
105#define _MSIZE (1 << _MSIZESHIFT) /* size of an mbuf */
106#else /* CONFIG_MBUF_MCACHE */
107#define _MSIZE 512
108#endif /* CONFIG_MBUF_MCACHE */
109
110#define NCLPGSHIFT (PAGE_SHIFT - MCLSHIFT)
111#define NCLPG (1 << NCLPGSHIFT) /* # of cl per page */
112
113#define NBCLPGSHIFT (PAGE_SHIFT - MBIGCLSHIFT)
114#define NBCLPG (1 << NBCLPGSHIFT) /* # of big cl per page */
115
116#define NMBPCL (MCLBYTES / _MSIZE)
117
118#define NCLPJCLSHIFT (M16KCLSHIFT - MCLSHIFT)
119#define NCLPJCL (1 << NCLPJCLSHIFT) /* # of cl per jumbo cl */
120
121#define NCLPBGSHIFT (MBIGCLSHIFT - MCLSHIFT)
122#define NCLPBG (1 << NCLPBGSHIFT) /* # of cl per big cl */
123
124/*
125 * Macros for type conversion
126 * mtod(m,t) - convert mbuf pointer to data pointer of correct type
127 * mtodo(m, o) -- Same as above but with offset 'o' into data.
128 */
129#define mtod(m, t) ((t)(void *)m_mtod_current(m))
130#define mtodo(m, o) ((void *)(mtod(m, uint8_t *) + (o)))
131
132/* header at beginning of each mbuf: */
133struct m_hdr {
134 struct mbuf *mh_next; /* next buffer in chain */
135 struct mbuf *mh_nextpkt; /* next chain in queue/record */
136 uintptr_t mh_data; /* location of data */
137 int32_t mh_len; /* amount of data in this mbuf */
138 u_int16_t mh_type; /* type of data in this mbuf */
139 u_int16_t mh_flags; /* flags; see below */
140#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4)
141/* This is needed because of how _MLEN is defined and used. Ideally, _MLEN
142 * should be defined using the offsetof(struct mbuf, M_dat), since there is
143 * no guarantee that mbuf.M_dat will start where mbuf.m_hdr ends. The compiler
144 * may (and does in the armv7k case) insert padding between m_hdr and M_dat in
145 * mbuf. We cannot easily use offsetof, however, since _MLEN is referenced
146 * in the definition of mbuf.
147 */
148} __attribute__((aligned(8)));
149#else
150};
151#endif
152
153/*
154 * Packet tag structure (see below for details).
155 */
156struct m_tag {
157 uint64_t m_tag_cookie; /* Error checking */
158 SLIST_ENTRY(m_tag) m_tag_link; /* List of packet tags */
159 void *m_tag_data;
160 uint16_t m_tag_type; /* Module specific type */
161 uint16_t m_tag_len; /* Length of data */
162 uint32_t m_tag_id; /* Module ID */
163 void *m_tag_mb_cl; /* pointer to mbuf or cluster container */
164#ifndef __LP64__
165 u_int32_t m_tag_pad;
166#endif /* !__LP64__ */
167};
168
169#define M_TAG_ALIGN(len) \
170 (P2ROUNDUP(len, sizeof (u_int64_t)) + sizeof (struct m_tag))
171
172#define M_TAG_INIT(tag, id, type, len, data, mb_cl) { \
173 VERIFY(IS_P2ALIGNED((tag), sizeof(u_int64_t))); \
174 (tag)->m_tag_type = (type); \
175 (tag)->m_tag_len = (uint16_t)(len); \
176 (tag)->m_tag_id = (id); \
177 (tag)->m_tag_data = (data); \
178 (tag)->m_tag_mb_cl = (mb_cl); \
179 m_tag_create_cookie(tag); \
180}
181
182#define M_TAG_VALID_PATTERN 0xfeedfacefeedfaceULL
183#define M_TAG_FREE_PATTERN 0xdeadbeefdeadbeefULL
184
185/*
186 * Packet tag header structure at the top of mbuf whe mbufs are use for m_tag
187 * Pointers are 32-bit in ILP32; m_tag needs 64-bit alignment, hence padded.
188 */
189struct m_taghdr {
190#ifndef __LP64__
191 u_int32_t pad; /* For structure alignment */
192#endif /* !__LP64__ */
193 u_int64_t mth_refcnt; /* Number of tags in this mbuf */
194};
195
196/*
197 * Driver auxiliary metadata tag (KERNEL_TAG_TYPE_DRVAUX).
198 */
199struct m_drvaux_tag {
200 u_int32_t da_family; /* IFNET_FAMILY values */
201 u_int32_t da_subfamily; /* IFNET_SUBFAMILY values */
202 u_int32_t da_reserved; /* for future */
203 u_int32_t da_length; /* length of following data */
204};
205
206/* Values for pftag_flags (16-bit wide) */
207#define PF_TAG_GENERATED 0x1 /* pkt generated by PF */
208#define PF_TAG_FRAGCACHE 0x2
209#define PF_TAG_TRANSLATE_LOCALHOST 0x4
210#if PF_ECN
211#define PF_TAG_HDR_INET 0x8 /* hdr points to IPv4 */
212#define PF_TAG_HDR_INET6 0x10 /* hdr points to IPv6 */
213#endif /* PF_ECN */
214#define PF_TAG_REASSEMBLED 0x20 /* pkt reassembled by PF */
215#define PF_TAG_REFRAGMENTED 0x40 /* pkt refragmented by PF */
216/*
217 * PF mbuf tag
218 */
219struct pf_mtag {
220 u_int16_t pftag_flags; /* PF_TAG flags */
221 u_int16_t pftag_rtableid; /* alternate routing table id */
222 u_int16_t pftag_tag;
223 u_int16_t pftag_routed;
224#if PF_ECN
225 void *pftag_hdr; /* saved hdr pos in mbuf, for ECN */
226#endif /* PF_ECN */
227};
228
229/* System reserved PF tags */
230#define PF_TAG_ID_SYSTEM_SERVICE 0xff00
231#define PF_TAG_ID_STACK_DROP 0xff01
232
233/*
234 * PF fragment tag
235 */
236struct pf_fragment_tag {
237 uint32_t ft_id; /* fragment id */
238 uint16_t ft_hdrlen; /* header length of reassembled pkt */
239 uint16_t ft_unfragpartlen; /* length of the per-fragment headers */
240 uint16_t ft_extoff; /* last extension header offset or 0 */
241 uint16_t ft_maxlen; /* maximum fragment payload length */
242};
243
244/*
245 * TCP mbuf tag
246 */
247struct tcp_pktinfo {
248 union {
249 struct {
250 uint32_t segsz; /* segment size (actual MSS) */
251 uint32_t start_seq; /* start seq of this packet */
252 pid_t pid;
253 pid_t e_pid;
254 } __tx;
255 struct {
256 uint8_t seg_cnt; /* # of coalesced TCP pkts */
257 } __rx;
258 } __offload;
259#define tso_segsz proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.segsz
260#define tx_start_seq proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.start_seq
261#define tx_tcp_pid proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.pid
262#define tx_tcp_e_pid proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.e_pid
263#define seg_cnt proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.seg_cnt
264};
265
266/*
267 * MPTCP mbuf tag
268 */
269struct mptcp_pktinfo {
270 uint64_t mtpi_dsn; /* MPTCP Data Sequence Number */
271 uint32_t mtpi_rel_seq; /* Relative Seq Number */
272 uint16_t mtpi_length; /* Length of mapping */
273 uint16_t mtpi_csum;
274#define mp_dsn proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_dsn
275#define mp_rseq proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_rel_seq
276#define mp_rlen proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_length
277#define mp_csum proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_csum
278};
279
280/*
281 * TCP specific mbuf tag. Note that the current implementation uses
282 * MPTCP metadata strictly between MPTCP and the TCP subflow layers,
283 * hence tm_tcp and tm_mptcp are mutually exclusive. This also means
284 * that TCP messages functionality is currently incompatible with MPTCP.
285 */
286struct tcp_mtag {
287 union {
288 struct tcp_pktinfo tm_tcp; /* TCP and below */
289 struct mptcp_pktinfo tm_mptcp; /* MPTCP-TCP only */
290 };
291};
292
293struct udp_mtag {
294 pid_t _pid;
295 pid_t _e_pid;
296#define tx_udp_pid proto_mtag.__pr_u.udp._pid
297#define tx_udp_e_pid proto_mtag.__pr_u.udp._e_pid
298};
299
300struct rawip_mtag {
301 pid_t _pid;
302 pid_t _e_pid;
303#define tx_rawip_pid proto_mtag.__pr_u.rawip._pid
304#define tx_rawip_e_pid proto_mtag.__pr_u.rawip._e_pid
305};
306
307struct driver_mtag_ {
308 uintptr_t _drv_tx_compl_arg;
309 uintptr_t _drv_tx_compl_data;
310 kern_return_t _drv_tx_status;
311 uint16_t _drv_flowid;
312#define drv_tx_compl_arg builtin_mtag._drv_mtag._drv_tx_compl_arg
313#define drv_tx_compl_data builtin_mtag._drv_mtag._drv_tx_compl_data
314#define drv_tx_status builtin_mtag._drv_mtag._drv_tx_status
315#define drv_flowid builtin_mtag._drv_mtag._drv_flowid
316};
317
318/*
319 * Protocol specific mbuf tag (at most one protocol metadata per mbuf).
320 *
321 * Care must be taken to ensure that they are mutually exclusive, e.g.
322 * IPsec policy ID implies no TCP segment offload (which is fine given
323 * that the former is used on the virtual ipsec interface that does
324 * not advertise the TSO capability.)
325 */
326struct proto_mtag_ {
327 union {
328 struct tcp_mtag tcp; /* TCP specific */
329 struct udp_mtag udp; /* UDP specific */
330 struct rawip_mtag rawip; /* raw IPv4/IPv6 specific */
331 } __pr_u;
332};
333
334/*
335 * NECP specific mbuf tag.
336 */
337struct necp_mtag_ {
338 u_int32_t necp_policy_id;
339 u_int32_t necp_skip_policy_id;
340 u_int32_t necp_route_rule_id;
341 u_int16_t necp_last_interface_index;
342 u_int16_t necp_app_id;
343};
344
345union builtin_mtag {
346 struct {
347 struct proto_mtag_ _proto_mtag; /* built-in protocol-specific tag */
348 struct pf_mtag _pf_mtag; /* built-in PF tag */
349 struct necp_mtag_ _necp_mtag; /* built-in NECP tag */
350 } _net_mtag;
351 struct driver_mtag_ _drv_mtag;
352#define necp_mtag builtin_mtag._net_mtag._necp_mtag
353#define proto_mtag builtin_mtag._net_mtag._proto_mtag
354#define driver_mtag builtin_mtag._drv_mtag
355};
356
357/*
358 * Record/packet header in first mbuf of chain; valid only if M_PKTHDR set.
359 */
360struct pkthdr {
361 struct ifnet *rcvif; /* rcv interface */
362 /* variables for ip and tcp reassembly */
363 void *pkt_hdr; /* pointer to packet header */
364 int32_t len; /* total packet length */
365 /* variables for hardware checksum */
366 /* Note: csum_flags is used for hardware checksum and VLAN */
367 u_int32_t csum_flags; /* flags regarding checksum */
368 union {
369 struct {
370 u_int16_t val; /* checksum value */
371 u_int16_t start; /* checksum start offset */
372 } _csum_rx;
373#define csum_rx_val _csum_rx.val
374#define csum_rx_start _csum_rx.start
375 struct {
376 u_int16_t start; /* checksum start offset */
377 u_int16_t stuff; /* checksum stuff offset */
378 } _csum_tx;
379#define csum_tx_start _csum_tx.start
380#define csum_tx_stuff _csum_tx.stuff
381 /*
382 * Generic data field used by csum routines.
383 * It gets used differently in different contexts.
384 */
385 u_int32_t csum_data;
386 };
387 u_int16_t vlan_tag; /* VLAN tag, host byte order */
388 /*
389 * Packet classifier info
390 *
391 * PKTF_FLOW_ID set means valid flow ID. A non-zero flow ID value
392 * means the packet has been classified by one of the flow sources.
393 * It is also a prerequisite for flow control advisory, which is
394 * enabled by additionally setting PKTF_FLOW_ADV.
395 *
396 * The protocol value is a best-effort representation of the payload.
397 * It is opportunistically updated and used only for optimization.
398 * It is not a substitute for parsing the protocol header(s); use it
399 * only as a hint.
400 *
401 * If PKTF_IFAINFO is set, pkt_ifainfo contains one or both of the
402 * indices of interfaces which own the source and/or destination
403 * addresses of the packet. For the local/loopback case (PKTF_LOOP),
404 * both should be valid, and thus allows for the receiving end to
405 * quickly determine the actual interfaces used by the the addresses;
406 * they may not necessarily be the same or refer to the loopback
407 * interface. Otherwise, in the non-local/loopback case, the indices
408 * are opportunistically set, and because of that only one may be set
409 * (0 means the index has not been determined.) In addition, the
410 * interface address flags are also recorded. This allows us to avoid
411 * storing the corresponding {in,in6}_ifaddr in an mbuf tag. Ideally
412 * this would be a superset of {ia,ia6}_flags, but the namespaces are
413 * overlapping at present, so we'll need a new set of values in future
414 * to achieve this. For now, we will just rely on the address family
415 * related code paths examining this mbuf to interpret the flags.
416 */
417 u_int8_t pkt_proto; /* IPPROTO value */
418 u_int8_t pkt_flowsrc; /* FLOWSRC values */
419 u_int32_t pkt_flowid; /* flow ID */
420 u_int32_t pkt_flags; /* PKTF flags (see below) */
421 u_int32_t pkt_svc; /* MBUF_SVC value */
422
423 u_int32_t pkt_compl_context; /* Packet completion context */
424
425 union {
426 struct {
427 u_int16_t src; /* ifindex of src addr i/f */
428 u_int16_t src_flags; /* src PKT_IFAIFF flags */
429 u_int16_t dst; /* ifindex of dst addr i/f */
430 u_int16_t dst_flags; /* dst PKT_IFAIFF flags */
431 } _pkt_iaif;
432#define src_ifindex _pkt_iaif.src
433#define src_iff _pkt_iaif.src_flags
434#define dst_ifindex _pkt_iaif.dst
435#define dst_iff _pkt_iaif.dst_flags
436 u_int64_t pkt_ifainfo; /* data field used by ifainfo */
437 struct {
438 u_int32_t if_data; /* bytes in interface queue */
439 u_int32_t sndbuf_data; /* bytes in socket buffer */
440 } _pkt_bsr; /* Buffer status report used by cellular interface */
441#define bufstatus_if _pkt_bsr.if_data
442#define bufstatus_sndbuf _pkt_bsr.sndbuf_data
443 };
444 u_int64_t pkt_timestamp; /* TX: enqueue time, RX: receive timestamp */
445
446 /*
447 * Tags (external and built-in)
448 */
449 SLIST_HEAD(packet_tags, m_tag) tags; /* list of external tags */
450 union builtin_mtag builtin_mtag;
451
452 uint32_t comp_gencnt;
453 uint16_t pkt_ext_flags;
454 uint16_t pkt_crumbs;
455 /*
456 * Module private scratch space (32-bit aligned), currently 16-bytes
457 * large. Anything stored here is not guaranteed to survive across
458 * modules. The AQM layer (outbound) uses all 16-bytes for both
459 * packet scheduling and flow advisory information.
460 */
461 struct {
462 union {
463 u_int8_t __mpriv8[16];
464 u_int16_t __mpriv16[8];
465 struct {
466 union {
467 u_int8_t __val8[4];
468 u_int16_t __val16[2];
469 u_int32_t __val32;
470 } __mpriv32_u;
471 } __mpriv32[4];
472 u_int64_t __mpriv64[2];
473 } __mpriv_u;
474 } pkt_mpriv __attribute__((aligned(4)));
475#define pkt_mpriv_hash pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val32
476#define pkt_mpriv_flags pkt_mpriv.__mpriv_u.__mpriv32[1].__mpriv32_u.__val32
477#define pkt_mpriv_srcid pkt_mpriv.__mpriv_u.__mpriv32[2].__mpriv32_u.__val32
478#define pkt_mpriv_fidx pkt_mpriv.__mpriv_u.__mpriv32[3].__mpriv32_u.__val32
479
480 u_int32_t redzone; /* red zone */
481 u_int32_t pkt_compl_callbacks; /* Packet completion callbacks */
482};
483
484/*
485 * Flow data source type. A data source module is responsible for generating
486 * a unique flow ID and associating it to each data flow as pkt_flowid.
487 * This is required for flow control/advisory, as it allows the output queue
488 * to identify the data source object and inform that it can resume its
489 * transmission (in the event it was flow controlled.)
490 */
491#define FLOWSRC_INPCB 1 /* flow ID generated by INPCB */
492#define FLOWSRC_IFNET 2 /* flow ID generated by interface */
493#define FLOWSRC_PF 3 /* flow ID generated by PF */
494#define FLOWSRC_CHANNEL 4 /* flow ID generated by channel */
495
496/*
497 * FLOWSRC_MPKL_INPUT is not a true flow data source and is used for
498 * multi-layer packet logging. We're usurping the pkt_flowsrc field because
499 * the mbuf packet header ran out of space and pkt_flowsrc is normally
500 * used on output so we assume we can safely overwrite the normal semantic of
501 * the field.
502 * This value is meant to be used on incoming packet from a lower level protocol
503 * to pass information to some upper level protocol. When FLOWSRC_MPKL_INPUT
504 * is set, the following fields are used:
505 * - pkt_proto: the IP protocol ID of the lower level protocol
506 * - pkt_flowid: the identifier of the packet at the lower protocol.
507 * For example ESP would set pkt_proto to IPPROTO_ESP and pkt_flowid to the SPI.
508 */
509
510/*
511 * Packet flags. Unlike m_flags, all packet flags are copied along when
512 * copying m_pkthdr, i.e. no equivalent of M_COPYFLAGS here. These flags
513 * (and other classifier info) will be cleared during DLIL input.
514 *
515 * Some notes about M_LOOP and PKTF_LOOP:
516 *
517 * - M_LOOP flag is overloaded, and its use is discouraged. Historically,
518 * that flag was used by the KAME implementation for allowing certain
519 * certain exceptions to be made in the IP6_EXTHDR_CHECK() logic; this
520 * was originally meant to be set as the packet is looped back to the
521 * system, and in some circumstances temporarily set in ip6_output().
522 * Over time, this flag was used by the pre-output routines to indicate
523 * to the DLIL frameout and output routines, that the packet may be
524 * looped back to the system under the right conditions. In addition,
525 * this is an mbuf flag rather than an mbuf packet header flag.
526 *
527 * - PKTF_LOOP is an mbuf packet header flag, which is set if and only
528 * if the packet was looped back to the system. This flag should be
529 * used instead for newer code.
530 */
531#define PKTF_FLOW_ID 0x1 /* pkt has valid flowid value */
532#define PKTF_FLOW_ADV 0x2 /* pkt triggers local flow advisory */
533#define PKTF_FLOW_LOCALSRC 0x4 /* pkt is locally originated */
534#define PKTF_FLOW_RAWSOCK 0x8 /* pkt locally generated by raw sock */
535#define PKTF_PRIO_PRIVILEGED 0x10 /* packet priority is privileged */
536#define PKTF_PROXY_DST 0x20 /* processed but not locally destined */
537#define PKTF_INET_RESOLVE 0x40 /* IPv4 resolver packet */
538#define PKTF_INET6_RESOLVE 0x80 /* IPv6 resolver packet */
539#define PKTF_RESOLVE_RTR 0x100 /* pkt is for resolving router */
540#define PKTF_SKIP_PKTAP 0x200 /* pkt has already passed through pktap */
541#define PKTF_WAKE_PKT 0x400 /* packet caused system to wake from sleep */
542#define PKTF_MPTCP 0x800 /* TCP with MPTCP metadata */
543#define PKTF_MPSO 0x1000 /* MPTCP socket meta data */
544#define PKTF_LOOP 0x2000 /* loopbacked packet */
545#define PKTF_IFAINFO 0x4000 /* pkt has valid interface addr info */
546#define PKTF_SO_BACKGROUND 0x8000 /* data is from background source */
547#define PKTF_FORWARDED 0x10000 /* pkt was forwarded from another i/f */
548#define PKTF_PRIV_GUARDED 0x20000 /* pkt_mpriv area guard enabled */
549#define PKTF_KEEPALIVE 0x40000 /* pkt is kernel-generated keepalive */
550#define PKTF_SO_REALTIME 0x80000 /* data is realtime traffic */
551#define PKTF_VALID_UNSENT_DATA 0x100000 /* unsent data is valid */
552#define PKTF_TCP_REXMT 0x200000 /* packet is TCP retransmission */
553#define PKTF_REASSEMBLED 0x400000 /* Packet was reassembled */
554#define PKTF_TX_COMPL_TS_REQ 0x800000 /* tx completion timestamp requested */
555#define PKTF_TS_VALID 0x1000000 /* pkt timestamp is valid */
556#define PKTF_DRIVER_MTAG 0x2000000 /* driver mbuf tags fields inited */
557#define PKTF_NEW_FLOW 0x4000000 /* Data from a new flow */
558#define PKTF_START_SEQ 0x8000000 /* valid start sequence */
559#define PKTF_LAST_PKT 0x10000000 /* last packet in the flow */
560#define PKTF_MPTCP_REINJ 0x20000000 /* Packet has been reinjected for MPTCP */
561#define PKTF_MPTCP_DFIN 0x40000000 /* Packet is a data-fin */
562#define PKTF_HBH_CHKED 0x80000000 /* HBH option is checked */
563
564#define PKTF_EXT_OUTPUT_SCOPE 0x1 /* outgoing packet has ipv6 address scope id */
565#define PKTF_EXT_L4S 0x2 /* pkts is from a L4S connection */
566#define PKTF_EXT_QUIC 0x4 /* flag to denote a QUIC packet */
567
568#define PKT_CRUMB_TS_COMP_REQ 0x0001 /* timestamp completion requested */
569#define PKT_CRUMB_TS_COMP_CB 0x0002 /* timestamp callback called */
570#define PKT_CRUMB_DLIL_OUTPUT 0x0004 /* dlil_output called */
571#define PKT_CRUMB_FLOW_TX 0x0008 /* dp_flow_tx_process called */
572#define PKT_CRUMB_FQ_ENQUEUE 0x0010 /* fq_enqueue called */
573#define PKT_CRUMB_FQ_DEQUEUE 0x0020 /* fq_dequeue called */
574#define PKT_CRUMB_SK_PKT_COPY 0x0040 /* copy from mbuf to skywalk packet */
575#define PKT_CRUMB_TCP_OUTPUT 0x0080
576#define PKT_CRUMB_UDP_OUTPUT 0x0100
577#define PKT_CRUMB_SOSEND 0x0200
578#define PKT_CRUMB_DLIL_INPUT 0x0400
579#define PKT_CRUMB_IP_INPUT 0x0800
580#define PKT_CRUMB_TCP_INPUT 0x1000
581#define PKT_CRUMB_UDP_INPUT 0x2000
582
583/* flags related to flow control/advisory and identification */
584#define PKTF_FLOW_MASK \
585 (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC | PKTF_FLOW_RAWSOCK)
586
587/*
588 * Description of external storage mapped into mbuf, valid only if M_EXT set.
589 */
590typedef void (*m_ext_free_func_t)(caddr_t, u_int, caddr_t);
591struct m_ext {
592 caddr_t __counted_by(ext_size) ext_buf; /* start of buffer */
593 m_ext_free_func_t ext_free; /* free routine if not the usual */
594 u_int ext_size; /* size of buffer, for ext_free */
595 caddr_t ext_arg; /* additional ext_free argument */
596 struct ext_ref {
597 struct mbuf *paired;
598 u_int16_t minref;
599 u_int16_t refcnt;
600 u_int16_t prefcnt;
601 u_int16_t flags;
602 u_int32_t priv;
603 uintptr_t ext_token;
604 } *ext_refflags;
605};
606
607/* define m_ext to a type since it gets redefined below */
608typedef struct m_ext _m_ext_t;
609
610#if CONFIG_MBUF_MCACHE
611/*
612 * The following _MLEN and _MHLEN macros are private to xnu. Private code
613 * that are outside of xnu must use the mbuf_get_{mlen,mhlen} routines since
614 * the sizes of the structures are dependent upon specific xnu configs.
615 */
616#define _MLEN (_MSIZE - sizeof(struct m_hdr)) /* normal data len */
617#define _MHLEN (_MLEN - sizeof(struct pkthdr)) /* data len w/pkthdr */
618
619#define NMBPGSHIFT (PAGE_SHIFT - _MSIZESHIFT)
620#define NMBPG (1 << NMBPGSHIFT) /* # of mbufs per page */
621
622#define NMBPCLSHIFT (MCLSHIFT - _MSIZESHIFT)
623
624/*
625 * The mbuf object
626 */
627struct mbuf {
628 struct m_hdr m_hdr;
629 union {
630 struct {
631 struct pkthdr MH_pkthdr; /* M_PKTHDR set */
632 union {
633 struct m_ext MH_ext; /* M_EXT set */
634 char MH_databuf[_MHLEN];
635 } MH_dat;
636 } MH;
637 char M_databuf[_MLEN]; /* !M_PKTHDR, !M_EXT */
638 } M_dat;
639};
640
641#define m_next m_hdr.mh_next
642#define m_len m_hdr.mh_len
643#define m_data m_hdr.mh_data
644#define m_type m_hdr.mh_type
645#define m_flags m_hdr.mh_flags
646#define m_nextpkt m_hdr.mh_nextpkt
647#define m_act m_nextpkt
648
649#define m_ext M_dat.MH.MH_dat.MH_ext
650#define m_pkthdr M_dat.MH.MH_pkthdr
651#define m_pktdat M_dat.MH.MH_dat.MH_databuf
652
653#else /* !CONFIG_MBUF_MCACHE */
654/*
655 * The following _MLEN and _MHLEN macros are private to xnu. Private code
656 * that are outside of xnu must use the mbuf_get_{mlen,mhlen} routines since
657 * the sizes of the structures are dependent upon specific xnu configs.
658 */
659#define _MLEN (_MSIZE - sizeof(struct m_hdr_common)) /* normal data len */
660#define _MHLEN (_MLEN) /* data len w/pkthdr */
661
662struct m_hdr_common {
663 struct m_hdr M_hdr;
664 struct m_ext M_ext __attribute__((aligned(16))); /* M_EXT set */
665 struct pkthdr M_pkthdr __attribute__((aligned(16))); /* M_PKTHDR set */
666};
667
668/*
669 * The mbuf object
670 */
671struct mbuf {
672 struct m_hdr_common M_hdr_common;
673 union {
674 char MH_databuf[_MHLEN];
675 char M_databuf[_MLEN]; /* !M_PKTHDR, !M_EXT */
676 } M_dat __attribute__((aligned(16)));
677};
678
679#define m_next M_hdr_common.M_hdr.mh_next
680#define m_len M_hdr_common.M_hdr.mh_len
681#define m_data M_hdr_common.M_hdr.mh_data
682#define m_type M_hdr_common.M_hdr.mh_type
683#define m_flags M_hdr_common.M_hdr.mh_flags
684#define m_nextpkt M_hdr_common.M_hdr.mh_nextpkt
685
686#define m_ext M_hdr_common.M_ext
687#define m_pkthdr M_hdr_common.M_pkthdr
688#define m_pktdat M_dat.MH_databuf
689
690#endif /* CONFIG_MBUF_MCACHE */
691
692#define m_act m_nextpkt
693#define m_dat M_dat.M_databuf
694#define m_pktlen(_m) ((_m)->m_pkthdr.len)
695#define m_pftag(_m) (&(_m)->m_pkthdr.builtin_mtag._net_mtag._pf_mtag)
696#define m_necptag(_m) (&(_m)->m_pkthdr.builtin_mtag._net_mtag._necp_mtag)
697
698/* mbuf flags (private) */
699#define M_EXT 0x0001 /* has associated external storage */
700#define M_PKTHDR 0x0002 /* start of record */
701#define M_EOR 0x0004 /* end of record */
702#define M_PROTO1 0x0008 /* protocol-specific */
703#define M_PROTO2 0x0010 /* protocol-specific */
704#define M_PROTO3 0x0020 /* protocol-specific */
705#define M_LOOP 0x0040 /* packet is looped back (also see PKTF_LOOP) */
706#define M_PROTO5 0x0080 /* protocol-specific */
707
708/* mbuf pkthdr flags, also in m_flags (private) */
709#define M_BCAST 0x0100 /* send/received as link-level broadcast */
710#define M_MCAST 0x0200 /* send/received as link-level multicast */
711#define M_FRAG 0x0400 /* packet is a fragment of a larger packet */
712#define M_FIRSTFRAG 0x0800 /* packet is first fragment */
713#define M_LASTFRAG 0x1000 /* packet is last fragment */
714#define M_PROMISC 0x2000 /* packet is promiscuous (shouldn't go to stack) */
715#define M_HASFCS 0x4000 /* packet has FCS */
716#define M_TAGHDR 0x8000 /* m_tag hdr structure at top of mbuf data */
717
718/*
719 * Flags to purge when crossing layers.
720 */
721#define M_PROTOFLAGS \
722 (M_PROTO1|M_PROTO2|M_PROTO3|M_PROTO5)
723
724/* flags copied when copying m_pkthdr */
725#define M_COPYFLAGS \
726 (M_PKTHDR|M_EOR|M_PROTO1|M_PROTO2|M_PROTO3 | \
727 M_LOOP|M_PROTO5|M_BCAST|M_MCAST|M_FRAG | \
728 M_FIRSTFRAG|M_LASTFRAG|M_PROMISC|M_HASFCS)
729
730/* flags indicating hw checksum support and sw checksum requirements */
731#define CSUM_IP 0x0001 /* will csum IP */
732#define CSUM_TCP 0x0002 /* will csum TCP */
733#define CSUM_UDP 0x0004 /* will csum UDP */
734#define CSUM_IP_FRAGS 0x0008 /* will csum IP fragments */
735#define CSUM_FRAGMENT 0x0010 /* will do IP fragmentation */
736#define CSUM_TCPIPV6 0x0020 /* will csum TCP for IPv6 */
737#define CSUM_UDPIPV6 0x0040 /* will csum UDP for IPv6 */
738#define CSUM_FRAGMENT_IPV6 0x0080 /* will do IPv6 fragmentation */
739
740#define CSUM_IP_CHECKED 0x0100 /* did csum IP */
741#define CSUM_IP_VALID 0x0200 /* ... the csum is valid */
742#define CSUM_DATA_VALID 0x0400 /* csum_data field is valid */
743#define CSUM_PSEUDO_HDR 0x0800 /* csum_data has pseudo hdr */
744#define CSUM_PARTIAL 0x1000 /* simple Sum16 computation */
745#define CSUM_ZERO_INVERT 0x2000 /* invert 0 to -0 (0xffff) */
746
747#define CSUM_DELAY_DATA (CSUM_TCP | CSUM_UDP)
748#define CSUM_DELAY_IP (CSUM_IP) /* IPv4 only: no IPv6 IP cksum */
749#define CSUM_DELAY_IPV6_DATA (CSUM_TCPIPV6 | CSUM_UDPIPV6)
750#define CSUM_DATA_IPV6_VALID CSUM_DATA_VALID /* csum_data field is valid */
751
752#define CSUM_TX_FLAGS \
753 (CSUM_DELAY_IP | CSUM_DELAY_DATA | CSUM_DELAY_IPV6_DATA | \
754 CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_ZERO_INVERT)
755
756#define CSUM_RX_FULL_FLAGS \
757 (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_PSEUDO_HDR | \
758 CSUM_DATA_VALID)
759
760#define CSUM_RX_FLAGS \
761 (CSUM_RX_FULL_FLAGS | CSUM_PARTIAL)
762
763
764
765/*
766 * Note: see also IF_HWASSIST_CSUM defined in <net/if_var.h>
767 */
768
769/* VLAN tag present */
770#define CSUM_VLAN_TAG_VALID 0x00010000 /* vlan_tag field is valid */
771
772/* checksum start adjustment has been done */
773#define CSUM_ADJUST_DONE 0x00020000
774
775/* VLAN encapsulation present */
776#define CSUM_VLAN_ENCAP_PRESENT 0x00040000 /* mbuf has vlan encapsulation */
777
778/* TCP Segment Offloading requested on this mbuf */
779#define CSUM_TSO_IPV4 0x00100000 /* This mbuf needs to be segmented by the NIC */
780#define CSUM_TSO_IPV6 0x00200000 /* This mbuf needs to be segmented by the NIC */
781
782#define TSO_IPV4_OK(_ifp, _m) \
783 (((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \
784 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \
785
786#define TSO_IPV4_NOTOK(_ifp, _m) \
787 (!((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \
788 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \
789
790#define TSO_IPV6_OK(_ifp, _m) \
791 (((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \
792 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \
793
794#define TSO_IPV6_NOTOK(_ifp, _m) \
795 (!((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \
796 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \
797
798#endif /* XNU_KERNEL_PRIVATE */
799
800/* mbuf types */
801#define MT_FREE 0 /* should be on free list */
802#define MT_DATA 1 /* dynamic (data) allocation */
803#define MT_HEADER 2 /* packet header */
804#define MT_SOCKET 3 /* socket structure */
805#define MT_PCB 4 /* protocol control block */
806#define MT_RTABLE 5 /* routing tables */
807#define MT_HTABLE 6 /* IMP host tables */
808#define MT_ATABLE 7 /* address resolution tables */
809#define MT_SONAME 8 /* socket name */
810#define MT_SOOPTS 10 /* socket options */
811#define MT_FTABLE 11 /* fragment reassembly header */
812#define MT_RIGHTS 12 /* access rights */
813#define MT_IFADDR 13 /* interface address */
814#define MT_CONTROL 14 /* extra-data protocol message */
815#define MT_OOBDATA 15 /* expedited data */
816#define MT_TAG 16 /* volatile metadata associated to pkts */
817#define MT_MAX 32 /* enough? */
818
819enum {
820 MTF_FREE = (1 << MT_FREE),
821 MTF_DATA = (1 << MT_DATA),
822 MTF_HEADER = (1 << MT_HEADER),
823 MTF_SOCKET = (1 << MT_SOCKET),
824 MTF_PCB = (1 << MT_PCB),
825 MTF_RTABLE = (1 << MT_RTABLE),
826 MTF_HTABLE = (1 << MT_HTABLE),
827 MTF_ATABLE = (1 << MT_ATABLE),
828 MTF_SONAME = (1 << MT_SONAME),
829 MTF_SOOPTS = (1 << MT_SOOPTS),
830 MTF_FTABLE = (1 << MT_FTABLE),
831 MTF_RIGHTS = (1 << MT_RIGHTS),
832 MTF_IFADDR = (1 << MT_IFADDR),
833 MTF_CONTROL = (1 << MT_CONTROL),
834 MTF_OOBDATA = (1 << MT_OOBDATA),
835 MTF_TAG = (1 << MT_TAG),
836};
837
838#ifdef XNU_KERNEL_PRIVATE
839/*
840 * mbuf allocation/deallocation macros:
841 *
842 * MGET(struct mbuf *m, int how, int type)
843 * allocates an mbuf and initializes it to contain internal data.
844 *
845 * MGETHDR(struct mbuf *m, int how, int type)
846 * allocates an mbuf and initializes it to contain a packet header
847 * and internal data.
848 */
849
850#if 1
851#define MCHECK(m) m_mcheck(m)
852#else
853#define MCHECK(m)
854#endif
855
856#define MGET(m, how, type) ((m) = m_get((how), (type)))
857
858#define MGETHDR(m, how, type) ((m) = m_gethdr((how), (type)))
859
860/*
861 * Mbuf cluster macros.
862 * MCLALLOC(caddr_t p, int how) allocates an mbuf cluster.
863 * MCLGET adds such clusters to a normal mbuf;
864 * the flag M_EXT is set upon success.
865 * MCLFREE releases a reference to a cluster allocated by MCLALLOC,
866 * freeing the cluster if the reference count has reached 0.
867 *
868 * Normal mbuf clusters are normally treated as character arrays
869 * after allocation, but use the first word of the buffer as a free list
870 * pointer while on the free list.
871 */
872union mcluster {
873 union mcluster *mcl_next;
874 char mcl_buf[MCLBYTES];
875};
876
877#define MCLALLOC(p, how) ((p) = m_mclalloc(how))
878
879#define MCLFREE(p) m_mclfree(p)
880
881#define MCLGET(m, how) ((m) = m_mclget(m, how))
882
883/*
884 * Mbuf big cluster
885 */
886union mbigcluster {
887 union mbigcluster *mbc_next;
888 char mbc_buf[MBIGCLBYTES];
889};
890
891/*
892 * Mbuf jumbo cluster
893 */
894union m16kcluster {
895 union m16kcluster *m16kcl_next;
896 char m16kcl_buf[M16KCLBYTES];
897};
898
899#define MCLHASREFERENCE(m) m_mclhasreference(m)
900
901/*
902 * MFREE(struct mbuf *m, struct mbuf *n)
903 * Free a single mbuf and associated external storage.
904 * Place the successor, if any, in n.
905 */
906
907#define MFREE(m, n) ((n) = m_free(m))
908
909/*
910 * Copy mbuf pkthdr from from to to.
911 * from must have M_PKTHDR set, and to must be empty.
912 * aux pointer will be moved to `to'.
913 */
914#define M_COPY_PKTHDR(to, from) m_copy_pkthdr(to, from)
915
916#define M_COPY_PFTAG(to, from) m_copy_pftag(to, from)
917
918#define M_COPY_NECPTAG(to, from) m_copy_necptag(to, from)
919
920#define M_COPY_CLASSIFIER(to, from) m_copy_classifier(to, from)
921
922/*
923 * Evaluate TRUE if it's safe to write to the mbuf m's data region (this can
924 * be both the local data payload, or an external buffer area, depending on
925 * whether M_EXT is set).
926 */
927#define M_WRITABLE(m) (((m)->m_flags & M_EXT) == 0 || !MCLHASREFERENCE(m))
928
929/*
930 * These macros are mapped to the appropriate KPIs, so that private code
931 * can be simply recompiled in order to be forward-compatible with future
932 * changes toward the struture sizes.
933 */
934#ifdef XNU_KERNEL_PRIVATE
935#define MLEN _MLEN
936#define MHLEN _MHLEN
937#define MINCLSIZE (MLEN + MHLEN)
938#else
939#define MLEN mbuf_get_mlen() /* normal mbuf data len */
940#define MHLEN mbuf_get_mhlen() /* data len in an mbuf w/pkthdr */
941#define MINCLSIZE mbuf_get_minclsize() /* cluster usage threshold */
942#endif
943/*
944 * Return the address of the start of the buffer associated with an mbuf,
945 * handling external storage, packet-header mbufs, and regular data mbufs.
946 */
947#define M_START(m) \
948 (((m)->m_flags & M_EXT) ? (caddr_t)(m)->m_ext.ext_buf : \
949 ((m)->m_flags & M_PKTHDR) ? &(m)->m_pktdat[0] : \
950 &(m)->m_dat[0])
951
952/*
953 * Return the size of the buffer associated with an mbuf, handling external
954 * storage, packet-header mbufs, and regular data mbufs.
955 */
956#define M_SIZE(m) \
957 (((m)->m_flags & M_EXT) ? (m)->m_ext.ext_size : \
958 ((m)->m_flags & M_PKTHDR) ? MHLEN : \
959 MLEN)
960
961#define M_ALIGN(m, len) m_align(m, len)
962#define MH_ALIGN(m, len) m_align(m, len)
963#define MEXT_ALIGN(m, len) m_align(m, len)
964
965/*
966 * Compute the amount of space available before the current start of data in
967 * an mbuf.
968 *
969 * The M_WRITABLE() is a temporary, conservative safety measure: the burden
970 * of checking writability of the mbuf data area rests solely with the caller.
971 */
972#define M_LEADINGSPACE(m) \
973 (M_WRITABLE(m) ? ((m)->m_data - (uintptr_t)M_START(m)) : 0)
974
975/*
976 * Compute the amount of space available after the end of data in an mbuf.
977 *
978 * The M_WRITABLE() is a temporary, conservative safety measure: the burden
979 * of checking writability of the mbuf data area rests solely with the caller.
980 */
981#define M_TRAILINGSPACE(m) \
982 (M_WRITABLE(m) ? \
983 ((M_START(m) + M_SIZE(m)) - (mtod(m, caddr_t) + (m)->m_len)) : 0)
984
985/*
986 * Arrange to prepend space of size plen to mbuf m.
987 * If a new mbuf must be allocated, how specifies whether to wait.
988 * If how is M_DONTWAIT and allocation fails, the original mbuf chain
989 * is freed and m is set to NULL.
990 */
991#define M_PREPEND(m, plen, how, align) \
992 ((m) = m_prepend_2((m), (plen), (how), (align)))
993
994/* change mbuf to new type */
995#define MCHTYPE(m, t) m_mchtype(m, t)
996
997/* compatiblity with 4.3 */
998#define m_copy(m, o, l) m_copym((m), (o), (l), M_DONTWAIT)
999
1000#define MBSHIFT 20 /* 1MB */
1001#define MBSIZE (1 << MBSHIFT)
1002#define GBSHIFT 30 /* 1GB */
1003#define GBSIZE (1 << GBSHIFT)
1004
1005/*
1006 * M_STRUCT_GET ensures that intermediate protocol header (from "off" to
1007 * "off+len") is located in single mbuf, on contiguous memory region.
1008 * The pointer to the region will be returned to pointer variable "val",
1009 * with type "typ".
1010 *
1011 * M_STRUCT_GET0 does the same, except that it aligns the structure at
1012 * very top of mbuf. GET0 is likely to make memory copy than GET.
1013 */
1014#define M_STRUCT_GET(val, typ, m, off, len) \
1015do { \
1016 struct mbuf *t; \
1017 int tmp; \
1018 \
1019 if ((m)->m_len >= (off) + (len)) { \
1020 (val) = (typ)(mtod((m), caddr_t) + (off)); \
1021 } else { \
1022 t = m_pulldown((m), (off), (len), &tmp); \
1023 if (t != NULL) { \
1024 if (t->m_len < tmp + (len)) \
1025 panic("m_pulldown malfunction"); \
1026 (val) = (typ)(mtod(t, caddr_t) + tmp); \
1027 } else { \
1028 (val) = (typ)NULL; \
1029 (m) = NULL; \
1030 } \
1031 } \
1032} while (0)
1033
1034#define M_STRUCT_GET0(val, typ, m, off, len) \
1035do { \
1036 struct mbuf *t; \
1037 \
1038 if ((off) == 0 && ((m)->m_len >= (len))) { \
1039 (val) = (typ)(void *)mtod(m, caddr_t); \
1040 } else { \
1041 t = m_pulldown((m), (off), (len), NULL); \
1042 if (t != NULL) { \
1043 if (t->m_len < (len)) \
1044 panic("m_pulldown malfunction"); \
1045 (val) = (typ)(void *)mtod(t, caddr_t); \
1046 } else { \
1047 (val) = (typ)NULL; \
1048 (m) = NULL; \
1049 } \
1050 } \
1051} while (0)
1052
1053#define MBUF_INPUT_CHECK(m, rcvif) \
1054do { \
1055 if (!(m->m_flags & MBUF_PKTHDR) || \
1056 m->m_len < 0 || \
1057 m->m_len > ((njcl > 0) ? njclbytes : MBIGCLBYTES) || \
1058 m->m_type == MT_FREE || \
1059 ((m->m_flags & M_EXT) != 0 && m->m_ext.ext_buf == NULL)) { \
1060 panic_plain("Failed mbuf validity check: mbuf %p len %d " \
1061 "type %d flags 0x%x data %p rcvif %s ifflags 0x%x", \
1062 m, m->m_len, m->m_type, m->m_flags, \
1063 ((m->m_flags & M_EXT) \
1064 ? m->m_ext.ext_buf \
1065 : (caddr_t __unsafe_indexable)m->m_data), \
1066 if_name(rcvif), \
1067 (rcvif->if_flags & 0xffff)); \
1068 } \
1069} while (0)
1070
1071/*
1072 * Simple mbuf queueing system
1073 *
1074 * This is basically a SIMPLEQ adapted to mbuf use (i.e. using
1075 * m_nextpkt instead of field.sqe_next).
1076 *
1077 * m_next is ignored, so queueing chains of mbufs is possible
1078 */
1079#define MBUFQ_HEAD(name) \
1080struct name { \
1081 struct mbuf *mq_first; /* first packet */ \
1082 struct mbuf **mq_last; /* addr of last next packet */ \
1083}
1084
1085#define MBUFQ_INIT(q) do { \
1086 MBUFQ_FIRST(q) = NULL; \
1087 (q)->mq_last = &MBUFQ_FIRST(q); \
1088} while (0)
1089
1090#define MBUFQ_PREPEND(q, m) do { \
1091 if ((MBUFQ_NEXT(m) = MBUFQ_FIRST(q)) == NULL) \
1092 (q)->mq_last = &MBUFQ_NEXT(m); \
1093 MBUFQ_FIRST(q) = (m); \
1094} while (0)
1095
1096#define MBUFQ_ENQUEUE(q, m) do { \
1097 MBUFQ_NEXT(m) = NULL; \
1098 *(q)->mq_last = (m); \
1099 (q)->mq_last = &MBUFQ_NEXT(m); \
1100} while (0)
1101
1102#define MBUFQ_ENQUEUE_MULTI(q, m, n) do { \
1103 MBUFQ_NEXT(n) = NULL; \
1104 *(q)->mq_last = (m); \
1105 (q)->mq_last = &MBUFQ_NEXT(n); \
1106} while (0)
1107
1108#define MBUFQ_DEQUEUE(q, m) do { \
1109 if (((m) = MBUFQ_FIRST(q)) != NULL) { \
1110 if ((MBUFQ_FIRST(q) = MBUFQ_NEXT(m)) == NULL) \
1111 (q)->mq_last = &MBUFQ_FIRST(q); \
1112 else \
1113 MBUFQ_NEXT(m) = NULL; \
1114 } \
1115} while (0)
1116
1117#define MBUFQ_REMOVE(q, m) do { \
1118 if (MBUFQ_FIRST(q) == (m)) { \
1119 MBUFQ_DEQUEUE(q, m); \
1120 } else { \
1121 struct mbuf *_m = MBUFQ_FIRST(q); \
1122 while (MBUFQ_NEXT(_m) != (m)) \
1123 _m = MBUFQ_NEXT(_m); \
1124 if ((MBUFQ_NEXT(_m) = \
1125 MBUFQ_NEXT(MBUFQ_NEXT(_m))) == NULL) \
1126 (q)->mq_last = &MBUFQ_NEXT(_m); \
1127 } \
1128} while (0)
1129
1130#define MBUFQ_DRAIN(q) do { \
1131 struct mbuf *__m0; \
1132 while ((__m0 = MBUFQ_FIRST(q)) != NULL) { \
1133 MBUFQ_FIRST(q) = MBUFQ_NEXT(__m0); \
1134 MBUFQ_NEXT(__m0) = NULL; \
1135 m_freem(__m0); \
1136 } \
1137 (q)->mq_last = &MBUFQ_FIRST(q); \
1138} while (0)
1139
1140#define MBUFQ_FOREACH(m, q) \
1141 for ((m) = MBUFQ_FIRST(q); \
1142 (m); \
1143 (m) = MBUFQ_NEXT(m))
1144
1145#define MBUFQ_FOREACH_SAFE(m, q, tvar) \
1146 for ((m) = MBUFQ_FIRST(q); \
1147 (m) && ((tvar) = MBUFQ_NEXT(m), 1); \
1148 (m) = (tvar))
1149
1150#define MBUFQ_EMPTY(q) ((q)->mq_first == NULL)
1151#define MBUFQ_FIRST(q) ((q)->mq_first)
1152#define MBUFQ_NEXT(m) ((m)->m_nextpkt)
1153/*
1154 * mq_last is initialized to point to mq_first, so check if they're
1155 * equal and return NULL when the list is empty. Otherwise, we need
1156 * to subtract the offset of MBUQ_NEXT (i.e. m_nextpkt field) to get
1157 * to the base mbuf address to return to caller.
1158 */
1159#define MBUFQ_LAST(head) \
1160 (((head)->mq_last == &MBUFQ_FIRST(head)) ? NULL : \
1161 ((struct mbuf *)(void *)((char *)(head)->mq_last - \
1162 __builtin_offsetof(struct mbuf, m_nextpkt))))
1163
1164#if (DEBUG || DEVELOPMENT)
1165#define MBUFQ_ADD_CRUMB_MULTI(_q, _h, _t, _f) do { \
1166 struct mbuf * _saved = (_t)->m_nextpkt; \
1167 struct mbuf * _m; \
1168 for (_m = (_h); _m != NULL; _m = MBUFQ_NEXT(_m)) { \
1169 m_add_crumb((_m), (_f)); \
1170 } \
1171 (_t)->m_nextpkt = _saved; \
1172} while (0)
1173
1174#define MBUFQ_ADD_CRUMB(_q, _m, _f) do { \
1175 m_add_crumb((_m), (_f)); \
1176} while (0)
1177#else
1178#define MBUFQ_ADD_CRUMB_MULTI(_q, _h, _t, _f)
1179#define MBUFQ_ADD_CRUMB(_q, _m, _f)
1180#endif /* (DEBUG || DEVELOPMENT) */
1181
1182#endif /* XNU_KERNEL_PRIVATE */
1183
1184/*
1185 * Mbuf statistics (legacy).
1186 */
1187struct mbstat {
1188 u_int32_t m_mbufs; /* mbufs obtained from page pool */
1189 u_int32_t m_clusters; /* clusters obtained from page pool */
1190 u_int32_t m_spare; /* spare field */
1191 u_int32_t m_clfree; /* free clusters */
1192 u_int32_t m_drops; /* times failed to find space */
1193 u_int32_t m_wait; /* times waited for space */
1194 u_int32_t m_drain; /* times drained protocols for space */
1195 u_short m_mtypes[256]; /* type specific mbuf allocations */
1196 u_int32_t m_mcfail; /* times m_copym failed */
1197 u_int32_t m_mpfail; /* times m_pullup failed */
1198 u_int32_t m_msize; /* length of an mbuf */
1199 u_int32_t m_mclbytes; /* length of an mbuf cluster */
1200 u_int32_t m_minclsize; /* min length of data to allocate a cluster */
1201 u_int32_t m_mlen; /* length of data in an mbuf */
1202 u_int32_t m_mhlen; /* length of data in a header mbuf */
1203 u_int32_t m_bigclusters; /* clusters obtained from page pool */
1204 u_int32_t m_bigclfree; /* free clusters */
1205 u_int32_t m_bigmclbytes; /* length of an mbuf cluster */
1206 u_int32_t m_forcedefunct; /* times we force defunct'ed an app's sockets */
1207};
1208
1209/* Compatibillity with 10.3 */
1210struct ombstat {
1211 u_int32_t m_mbufs; /* mbufs obtained from page pool */
1212 u_int32_t m_clusters; /* clusters obtained from page pool */
1213 u_int32_t m_spare; /* spare field */
1214 u_int32_t m_clfree; /* free clusters */
1215 u_int32_t m_drops; /* times failed to find space */
1216 u_int32_t m_wait; /* times waited for space */
1217 u_int32_t m_drain; /* times drained protocols for space */
1218 u_short m_mtypes[256]; /* type specific mbuf allocations */
1219 u_int32_t m_mcfail; /* times m_copym failed */
1220 u_int32_t m_mpfail; /* times m_pullup failed */
1221 u_int32_t m_msize; /* length of an mbuf */
1222 u_int32_t m_mclbytes; /* length of an mbuf cluster */
1223 u_int32_t m_minclsize; /* min length of data to allocate a cluster */
1224 u_int32_t m_mlen; /* length of data in an mbuf */
1225 u_int32_t m_mhlen; /* length of data in a header mbuf */
1226};
1227
1228/*
1229 * mbuf class statistics.
1230 */
1231#define MAX_MBUF_CNAME 15
1232
1233#if defined(XNU_KERNEL_PRIVATE)
1234/* For backwards compatibility with 32-bit userland process */
1235struct omb_class_stat {
1236 char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */
1237 u_int32_t mbcl_size; /* buffer size */
1238 u_int32_t mbcl_total; /* # of buffers created */
1239 u_int32_t mbcl_active; /* # of active buffers */
1240 u_int32_t mbcl_infree; /* # of available buffers */
1241 u_int32_t mbcl_slab_cnt; /* # of available slabs */
1242 u_int32_t mbcl_pad; /* padding */
1243 u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */
1244 u_int64_t mbcl_free_cnt; /* # of times free is called */
1245 u_int64_t mbcl_notified; /* # of notified wakeups */
1246 u_int64_t mbcl_purge_cnt; /* # of purges so far */
1247 u_int64_t mbcl_fail_cnt; /* # of allocation failures */
1248 u_int32_t mbcl_ctotal; /* total only for this class */
1249 u_int32_t mbcl_release_cnt; /* amount of memory returned */
1250 /*
1251 * Cache layer statistics
1252 */
1253 u_int32_t mbcl_mc_state; /* cache state (see below) */
1254 u_int32_t mbcl_mc_cached; /* # of cached buffers */
1255 u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */
1256 u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */
1257 u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */
1258 u_int32_t mbcl_reserved[7]; /* for future use */
1259} __attribute__((__packed__));
1260#endif /* XNU_KERNEL_PRIVATE */
1261
1262typedef struct mb_class_stat {
1263 char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */
1264 u_int32_t mbcl_size; /* buffer size */
1265 u_int32_t mbcl_total; /* # of buffers created */
1266 u_int32_t mbcl_active; /* # of active buffers */
1267 u_int32_t mbcl_infree; /* # of available buffers */
1268 u_int32_t mbcl_slab_cnt; /* # of available slabs */
1269#if defined(KERNEL) || defined(__LP64__)
1270 u_int32_t mbcl_pad; /* padding */
1271#endif /* KERNEL || __LP64__ */
1272 u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */
1273 u_int64_t mbcl_free_cnt; /* # of times free is called */
1274 u_int64_t mbcl_notified; /* # of notified wakeups */
1275 u_int64_t mbcl_purge_cnt; /* # of purges so far */
1276 u_int64_t mbcl_fail_cnt; /* # of allocation failures */
1277 u_int32_t mbcl_ctotal; /* total only for this class */
1278 u_int32_t mbcl_release_cnt; /* amount of memory returned */
1279 /*
1280 * Cache layer statistics
1281 */
1282 u_int32_t mbcl_mc_state; /* cache state (see below) */
1283 u_int32_t mbcl_mc_cached; /* # of cached buffers */
1284 u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */
1285 u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */
1286 u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */
1287 u_int32_t mbcl_reserved[7]; /* for future use */
1288} mb_class_stat_t;
1289
1290#define MCS_DISABLED 0 /* cache is permanently disabled */
1291#define MCS_ONLINE 1 /* cache is online */
1292#define MCS_PURGING 2 /* cache is being purged */
1293#define MCS_OFFLINE 3 /* cache is offline (resizing) */
1294
1295#if defined(XNU_KERNEL_PRIVATE)
1296/* For backwards compatibility with 32-bit userland process */
1297struct omb_stat {
1298 u_int32_t mbs_cnt; /* number of classes */
1299 u_int32_t mbs_pad; /* padding */
1300 struct omb_class_stat mbs_class[1]; /* class array */
1301} __attribute__((__packed__));
1302#endif /* XNU_KERNEL_PRIVATE */
1303
1304typedef struct mb_stat {
1305 u_int32_t mbs_cnt; /* number of classes */
1306#if defined(KERNEL) || defined(__LP64__)
1307 u_int32_t mbs_pad; /* padding */
1308#endif /* KERNEL || __LP64__ */
1309 mb_class_stat_t mbs_class[1]; /* class array */
1310} mb_stat_t;
1311
1312#ifdef PRIVATE
1313#define MLEAK_STACK_DEPTH 16 /* Max PC stack depth */
1314
1315typedef struct mleak_trace_stat {
1316 u_int64_t mltr_collisions;
1317 u_int64_t mltr_hitcount;
1318 u_int64_t mltr_allocs;
1319 u_int64_t mltr_depth;
1320 u_int64_t mltr_addr[MLEAK_STACK_DEPTH];
1321} mleak_trace_stat_t;
1322
1323typedef struct mleak_stat {
1324 u_int32_t ml_isaddr64; /* 64-bit KVA? */
1325 u_int32_t ml_cnt; /* number of traces */
1326 mleak_trace_stat_t ml_trace[1]; /* trace array */
1327} mleak_stat_t;
1328
1329struct mleak_table {
1330 u_int32_t mleak_capture; /* sampling capture counter */
1331 u_int32_t mleak_sample_factor; /* sample factor */
1332
1333 /* Times two active records want to occupy the same spot */
1334 u_int64_t alloc_collisions;
1335 u_int64_t trace_collisions;
1336
1337 /* Times new record lands on spot previously occupied by freed alloc */
1338 u_int64_t alloc_overwrites;
1339 u_int64_t trace_overwrites;
1340
1341 /* Times a new alloc or trace is put into the hash table */
1342 u_int64_t alloc_recorded;
1343 u_int64_t trace_recorded;
1344
1345 /* Total number of outstanding allocs */
1346 u_int64_t outstanding_allocs;
1347
1348 /* Times mleak_log returned false because couldn't acquire the lock */
1349 u_int64_t total_conflicts;
1350};
1351
1352#define HAS_M_TAG_STATS 1
1353
1354struct m_tag_stats {
1355 u_int32_t mts_id;
1356 u_int16_t mts_type;
1357 u_int16_t mts_len;
1358 u_int64_t mts_alloc_count;
1359 u_int64_t mts_alloc_failed;
1360 u_int64_t mts_free_count;
1361};
1362
1363
1364#define M_TAG_TYPE_NAMES \
1365 "other,dummynet,ipfilt,encap,inet6,ipsec,cfil_udp,pf_reass,aqm,drvaux"
1366
1367#endif /* PRIVATE */
1368
1369#ifdef KERNEL_PRIVATE
1370__BEGIN_DECLS
1371
1372/*
1373 * Exported (private)
1374 */
1375
1376extern struct mbstat mbstat; /* statistics */
1377
1378__END_DECLS
1379#endif /* KERNEL_PRIVATE */
1380
1381#ifdef XNU_KERNEL_PRIVATE
1382__BEGIN_DECLS
1383
1384/*
1385 * Not exported (xnu private)
1386 */
1387
1388/* flags to m_get/MGET */
1389/* Need to include malloc.h to get right options for malloc */
1390#include <sys/malloc.h>
1391
1392struct mbuf;
1393
1394/* length to m_copy to copy all */
1395#define M_COPYALL 1000000000
1396
1397#define M_DONTWAIT M_NOWAIT
1398#define M_WAIT M_WAITOK
1399
1400/* modes for m_copym and variants */
1401#define M_COPYM_NOOP_HDR 0 /* don't copy/move pkthdr contents */
1402#define M_COPYM_COPY_HDR 1 /* copy pkthdr from old to new */
1403#define M_COPYM_MOVE_HDR 2 /* move pkthdr from old to new */
1404#define M_COPYM_MUST_COPY_HDR 3 /* MUST copy pkthdr from old to new */
1405#define M_COPYM_MUST_MOVE_HDR 4 /* MUST move pkthdr from old to new */
1406
1407extern void m_freem(struct mbuf *) __XNU_INTERNAL(m_freem);
1408extern u_int64_t mcl_to_paddr(char *);
1409extern void m_adj(struct mbuf *, int);
1410extern void m_cat(struct mbuf *, struct mbuf *);
1411extern void m_copydata(struct mbuf *, int, int, void *);
1412extern struct mbuf *m_copym(struct mbuf *, int, int, int);
1413extern struct mbuf *m_copym_mode(struct mbuf *, int, int, int, struct mbuf **, int *, uint32_t);
1414extern struct mbuf *m_get(int, int);
1415extern struct mbuf *m_gethdr(int, int);
1416extern struct mbuf *m_getpacket(void);
1417extern struct mbuf *m_getpackets(int, int, int);
1418extern struct mbuf *m_mclget(struct mbuf *, int);
1419extern void *__unsafe_indexable m_mtod(struct mbuf *);
1420extern struct mbuf *m_prepend_2(struct mbuf *, int, int, int);
1421extern struct mbuf *m_pullup(struct mbuf *, int);
1422extern struct mbuf *m_split(struct mbuf *, int, int);
1423extern void m_mclfree(caddr_t p);
1424extern bool mbuf_class_under_pressure(struct mbuf *m);
1425
1426/*
1427 * Accessors for the mbuf data range.
1428 * The "lower bound" is the start of the memory range that m->m_data is allowed
1429 * to point into. The "start" is where m->m_data points to; equivalent to the
1430 * late m_mtod. The end is where m->m_data + m->m_len points to. The upper bound
1431 * is the end of the memory range that m->m_data + m->m_len is allowed to point
1432 * into.
1433 * In a well-formed range, lower bound <= start <= end <= upper bound. An
1434 * ill-formed range always means a programming error.
1435 */
1436__stateful_pure static inline caddr_t __header_bidi_indexable
1437m_mtod_lower_bound(struct mbuf *m)
1438{
1439 return M_START(m);
1440}
1441
1442__stateful_pure static inline caddr_t __header_bidi_indexable
1443m_mtod_current(struct mbuf *m)
1444{
1445 caddr_t data = m_mtod_lower_bound(m);
1446 return data + (m->m_data - (uintptr_t)data);
1447}
1448
1449__stateful_pure static inline caddr_t __header_bidi_indexable
1450m_mtod_end(struct mbuf *m)
1451{
1452 return m_mtod_current(m) + m->m_len;
1453}
1454
1455__stateful_pure static inline caddr_t __header_bidi_indexable
1456m_mtod_upper_bound(struct mbuf *m)
1457{
1458 return m_mtod_lower_bound(m) + M_SIZE(m);
1459}
1460
1461static inline bool
1462m_has_mtype(const struct mbuf *m, int mtype_flags)
1463{
1464 return (1 << m->m_type) & mtype_flags;
1465}
1466
1467/*
1468 * On platforms which require strict alignment (currently for anything but
1469 * i386 or x86_64 or arm64), this macro checks whether the data pointer of an mbuf
1470 * is 32-bit aligned (this is the expected minimum alignment for protocol
1471 * headers), and assert otherwise.
1472 */
1473#if defined(__i386__) || defined(__x86_64__) || defined(__arm64__)
1474#define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m)
1475#else /* !__i386__ && !__x86_64__ && !__arm64__ */
1476#define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) do { \
1477 if (!IS_P2ALIGNED((_m)->m_data, sizeof (u_int32_t))) { \
1478 if (((_m)->m_flags & M_PKTHDR) && \
1479 (_m)->m_pkthdr.rcvif != NULL) { \
1480 panic_plain("\n%s: mbuf %p data ptr %p is not " \
1481 "32-bit aligned [%s: alignerrs=%lld]\n", \
1482 __func__, (_m), \
1483 (caddr_t __unsafe_indexable)(_m)->m_data, \
1484 if_name((_m)->m_pkthdr.rcvif), \
1485 (_m)->m_pkthdr.rcvif->if_alignerrs); \
1486 } else { \
1487 panic_plain("\n%s: mbuf %p data ptr %p is not " \
1488 "32-bit aligned\n", \
1489 __func__, (_m), \
1490 (caddr_t __unsafe_indexable)(_m)->m_data); \
1491 } \
1492 } \
1493} while (0)
1494#endif /* !__i386__ && !__x86_64__ && !__arm64__ */
1495
1496/* Maximum number of MBUF_SC values (excluding MBUF_SC_UNSPEC) */
1497#define MBUF_SC_MAX_CLASSES 10
1498
1499/*
1500 * These conversion macros rely on the corresponding MBUF_SC and
1501 * MBUF_TC values in order to establish the following mapping:
1502 *
1503 * MBUF_SC_BK_SYS ] ==> MBUF_TC_BK
1504 * MBUF_SC_BK ]
1505 *
1506 * MBUF_SC_BE ] ==> MBUF_TC_BE
1507 * MBUF_SC_RD ]
1508 * MBUF_SC_OAM ]
1509 *
1510 * MBUF_SC_AV ] ==> MBUF_TC_VI
1511 * MBUF_SC_RV ]
1512 * MBUF_SC_VI ]
1513 * MBUF_SC_SIG ]
1514 *
1515 * MBUF_SC_VO ] ==> MBUF_TC_VO
1516 * MBUF_SC_CTL ]
1517 *
1518 * The values assigned to each service class allows for a fast mapping to
1519 * the corresponding MBUF_TC traffic class values, as well as to retrieve the
1520 * assigned index; therefore care must be taken when comparing against these
1521 * values. Use the corresponding class and index macros to retrieve the
1522 * corresponding portion, and never assume that a higher class corresponds
1523 * to a higher index.
1524 */
1525#define MBUF_SCVAL(x) ((x) & 0xffff)
1526#define MBUF_SCIDX(x) ((((x) >> 16) & 0xff) >> 3)
1527#define MBUF_SC2TC(_sc) (MBUF_SCVAL(_sc) >> 7)
1528#define MBUF_TC2SCVAL(_tc) ((_tc) << 7)
1529#define IS_MBUF_SC_BACKGROUND(_sc) (((_sc) == MBUF_SC_BK_SYS) || \
1530 ((_sc) == MBUF_SC_BK))
1531#define IS_MBUF_SC_REALTIME(_sc) ((_sc) >= MBUF_SC_AV && (_sc) <= MBUF_SC_VO)
1532#define IS_MBUF_SC_BESTEFFORT(_sc) ((_sc) == MBUF_SC_BE || \
1533 (_sc) == MBUF_SC_RD || (_sc) == MBUF_SC_OAM)
1534
1535#define SCIDX_BK_SYS MBUF_SCIDX(MBUF_SC_BK_SYS)
1536#define SCIDX_BK MBUF_SCIDX(MBUF_SC_BK)
1537#define SCIDX_BE MBUF_SCIDX(MBUF_SC_BE)
1538#define SCIDX_RD MBUF_SCIDX(MBUF_SC_RD)
1539#define SCIDX_OAM MBUF_SCIDX(MBUF_SC_OAM)
1540#define SCIDX_AV MBUF_SCIDX(MBUF_SC_AV)
1541#define SCIDX_RV MBUF_SCIDX(MBUF_SC_RV)
1542#define SCIDX_VI MBUF_SCIDX(MBUF_SC_VI)
1543#define SCIDX_SIG MBUF_SCIDX(MBUF_SC_SIG)
1544#define SCIDX_VO MBUF_SCIDX(MBUF_SC_VO)
1545#define SCIDX_CTL MBUF_SCIDX(MBUF_SC_CTL)
1546
1547#define SCVAL_BK_SYS MBUF_SCVAL(MBUF_SC_BK_SYS)
1548#define SCVAL_BK MBUF_SCVAL(MBUF_SC_BK)
1549#define SCVAL_BE MBUF_SCVAL(MBUF_SC_BE)
1550#define SCVAL_RD MBUF_SCVAL(MBUF_SC_RD)
1551#define SCVAL_OAM MBUF_SCVAL(MBUF_SC_OAM)
1552#define SCVAL_AV MBUF_SCVAL(MBUF_SC_AV)
1553#define SCVAL_RV MBUF_SCVAL(MBUF_SC_RV)
1554#define SCVAL_VI MBUF_SCVAL(MBUF_SC_VI)
1555#define SCVAL_SIG MBUF_SCVAL(MBUF_SC_SIG)
1556#define SCVAL_VO MBUF_SCVAL(MBUF_SC_VO)
1557#define SCVAL_CTL MBUF_SCVAL(MBUF_SC_CTL)
1558
1559#define MBUF_VALID_SC(c) \
1560 (c == MBUF_SC_BK_SYS || c == MBUF_SC_BK || c == MBUF_SC_BE || \
1561 c == MBUF_SC_RD || c == MBUF_SC_OAM || c == MBUF_SC_AV || \
1562 c == MBUF_SC_RV || c == MBUF_SC_VI || c == MBUF_SC_SIG || \
1563 c == MBUF_SC_VO || c == MBUF_SC_CTL)
1564
1565#define MBUF_VALID_SCIDX(c) \
1566 (c == SCIDX_BK_SYS || c == SCIDX_BK || c == SCIDX_BE || \
1567 c == SCIDX_RD || c == SCIDX_OAM || c == SCIDX_AV || \
1568 c == SCIDX_RV || c == SCIDX_VI || c == SCIDX_SIG || \
1569 c == SCIDX_VO || c == SCIDX_CTL)
1570
1571#define MBUF_VALID_SCVAL(c) \
1572 (c == SCVAL_BK_SYS || c == SCVAL_BK || c == SCVAL_BE || \
1573 c == SCVAL_RD || c == SCVAL_OAM || c == SCVAL_AV || \
1574 c == SCVAL_RV || c == SCVAL_VI || c == SCVAL_SIG || \
1575 c == SCVAL_VO || SCVAL_CTL)
1576
1577extern unsigned char *mbutl; /* start VA of mbuf pool */
1578extern unsigned char *embutl; /* end VA of mbuf pool */
1579extern unsigned int nmbclusters; /* number of mapped clusters */
1580extern int njcl; /* # of jumbo clusters */
1581extern int njclbytes; /* size of a jumbo cluster */
1582extern int max_hdr; /* largest link+protocol header */
1583extern int max_datalen; /* MHLEN - max_hdr */
1584
1585extern int max_linkhdr; /* largest link-level header */
1586
1587/* Use max_protohdr instead of _max_protohdr */
1588extern int max_protohdr; /* largest protocol header */
1589
1590__private_extern__ unsigned int mbuf_default_ncl(uint64_t);
1591__private_extern__ void mbinit(void);
1592__private_extern__ struct mbuf *m_clattach(struct mbuf *, int, caddr_t,
1593 void (*)(caddr_t, u_int, caddr_t), size_t, caddr_t, int, int);
1594__private_extern__ caddr_t m_bigalloc(int);
1595__private_extern__ void m_bigfree(caddr_t, u_int, caddr_t);
1596__private_extern__ struct mbuf *m_mbigget(struct mbuf *, int);
1597__private_extern__ caddr_t m_16kalloc(int);
1598__private_extern__ void m_16kfree(caddr_t, u_int, caddr_t);
1599__private_extern__ struct mbuf *m_m16kget(struct mbuf *, int);
1600__private_extern__ int m_reinit(struct mbuf *, int);
1601__private_extern__ struct mbuf *m_free(struct mbuf *) __XNU_INTERNAL(m_free);
1602__private_extern__ struct mbuf *m_getclr(int, int);
1603__private_extern__ struct mbuf *m_getptr(struct mbuf *, int, int *);
1604__private_extern__ unsigned int m_length(struct mbuf *);
1605__private_extern__ unsigned int m_length2(struct mbuf *, struct mbuf **);
1606__private_extern__ unsigned int m_fixhdr(struct mbuf *);
1607__private_extern__ struct mbuf *m_defrag(struct mbuf *, int);
1608__private_extern__ struct mbuf *m_defrag_offset(struct mbuf *, u_int32_t, int);
1609__private_extern__ struct mbuf *m_prepend(struct mbuf *, int, int);
1610__private_extern__ struct mbuf *m_copyup(struct mbuf *, int, int);
1611__private_extern__ struct mbuf *m_retry(int, int);
1612__private_extern__ struct mbuf *m_retryhdr(int, int);
1613__private_extern__ int m_freem_list(struct mbuf *);
1614__private_extern__ int m_append(struct mbuf *, int, caddr_t);
1615__private_extern__ struct mbuf *m_last(struct mbuf *);
1616__private_extern__ struct mbuf *m_devget(char *, int, int, struct ifnet *,
1617 void (*)(const void *, void *, size_t));
1618__private_extern__ struct mbuf *m_pulldown(struct mbuf *, int, int, int *);
1619
1620__private_extern__ struct mbuf *m_getcl(int, int, int);
1621__private_extern__ caddr_t m_mclalloc(int);
1622__private_extern__ int m_mclhasreference(struct mbuf *);
1623__private_extern__ void m_copy_pkthdr(struct mbuf *, struct mbuf *);
1624__private_extern__ int m_dup_pkthdr(struct mbuf *, struct mbuf *, int);
1625__private_extern__ void m_copy_pftag(struct mbuf *, struct mbuf *);
1626__private_extern__ void m_copy_necptag(struct mbuf *, struct mbuf *);
1627__private_extern__ void m_copy_classifier(struct mbuf *, struct mbuf *);
1628
1629__private_extern__ struct mbuf *m_dtom(void *);
1630__private_extern__ int m_mtocl(void *);
1631__private_extern__ union mcluster *m_cltom(int);
1632
1633__private_extern__ void m_align(struct mbuf *, int);
1634
1635__private_extern__ struct mbuf *m_normalize(struct mbuf *m);
1636__private_extern__ void m_mchtype(struct mbuf *m, int t);
1637__private_extern__ void m_mcheck(struct mbuf *);
1638
1639__private_extern__ void m_copyback(struct mbuf *, int, int, const void *);
1640__private_extern__ struct mbuf *m_copyback_cow(struct mbuf *, int, int,
1641 const void *, int);
1642__private_extern__ int m_makewritable(struct mbuf **, int, int, int);
1643__private_extern__ struct mbuf *m_dup(struct mbuf *m, int how);
1644__private_extern__ struct mbuf *m_copym_with_hdrs(struct mbuf *, int, int, int,
1645 struct mbuf **, int *, uint32_t);
1646__private_extern__ struct mbuf *m_getpackethdrs(int, int);
1647__private_extern__ struct mbuf *m_getpacket_how(int);
1648__private_extern__ struct mbuf *m_getpackets_internal(unsigned int *, int,
1649 int, int, size_t);
1650__private_extern__ struct mbuf *m_allocpacket_internal(unsigned int *, size_t,
1651 unsigned int *, int, int, size_t);
1652
1653__private_extern__ int m_ext_set_prop(struct mbuf *, uint32_t, uint32_t);
1654__private_extern__ uint32_t m_ext_get_prop(struct mbuf *);
1655__private_extern__ int m_ext_paired_is_active(struct mbuf *);
1656__private_extern__ void m_ext_paired_activate(struct mbuf *);
1657
1658__private_extern__ void m_add_crumb(struct mbuf *, uint16_t);
1659
1660__private_extern__ void mbuf_drain(boolean_t);
1661
1662/*
1663 * Packets may have annotations attached by affixing a list of "packet
1664 * tags" to the pkthdr structure. Packet tags are dynamically allocated
1665 * semi-opaque data structures that have a fixed header (struct m_tag)
1666 * that specifies the size of the memory block and an <id,type> pair that
1667 * identifies it. The id identifies the module and the type identifies the
1668 * type of data for that module. The id of zero is reserved for the kernel.
1669 *
1670 * By default packet tags are allocated via kalloc except on Intel that still
1671 * uses the legacy implementation of using mbufs for packet tags.
1672 * This can be overidden via the boot-args 'mb_tag_mbuf'
1673 *
1674 * When kalloc is used for allocation, packet tags returned by m_tag_allocate have
1675 * the default memory alignment implemented by kalloc.
1676 *
1677 * When mbufs are used for allocation packets tag returned by m_tag_allocate has
1678 * the default memory alignment implemented by malloc.
1679 *
1680 * To reference the private data one should use a construct like:
1681 * struct m_tag *mtag = m_tag_allocate(...);
1682 * struct foo *p = (struct foo *)(mtag->m_tag_data);
1683 *
1684 * There should be no assumption on the location of the private data relative to the
1685 * 'struct m_tag'
1686 *
1687 * When kalloc is used, packet tags that are internal to xnu use KERNEL_MODULE_TAG_ID and
1688 * they are allocated with kalloc_type using a single container data structure that has
1689 * the 'struct m_tag' followed by a data structure for the private data
1690 *
1691 * Packet tags that are allocated by KEXTs are external to xnu and type of the private data
1692 * is unknown to xnu, so they are allocated in two chunks:
1693 * - one allocation with kalloc_type for the 'struct m_tag'
1694 * - one allocation using kheap_alloc as for the private data
1695 *
1696 * Note that packet tags of type KERNEL_TAG_TYPE_DRVAUX are allocated by KEXTs with
1697 * a variable length so they are allocated in two chunks
1698 *
1699 * In all cases the 'struct m_tag' is allocated using kalloc_type to avoid type
1700 * confusion.
1701 */
1702
1703#define KERNEL_MODULE_TAG_ID 0
1704
1705enum {
1706 KERNEL_TAG_TYPE_NONE = 0,
1707 KERNEL_TAG_TYPE_DUMMYNET = 1,
1708 KERNEL_TAG_TYPE_IPFILT = 2,
1709 KERNEL_TAG_TYPE_ENCAP = 3,
1710 KERNEL_TAG_TYPE_INET6 = 4,
1711 KERNEL_TAG_TYPE_IPSEC = 5,
1712 KERNEL_TAG_TYPE_CFIL_UDP = 6,
1713 KERNEL_TAG_TYPE_PF_REASS = 7,
1714 KERNEL_TAG_TYPE_AQM = 8,
1715 KERNEL_TAG_TYPE_DRVAUX = 9,
1716 KERNEL_TAG_TYPE_COUNT = 10
1717};
1718
1719/* Packet tag routines */
1720__private_extern__ struct m_tag *m_tag_alloc(u_int32_t, u_int16_t, int, int);
1721__private_extern__ struct m_tag *m_tag_create(u_int32_t, u_int16_t, int, int,
1722 struct mbuf *);
1723__private_extern__ void m_tag_free(struct m_tag *);
1724__private_extern__ void m_tag_prepend(struct mbuf *, struct m_tag *);
1725__private_extern__ void m_tag_unlink(struct mbuf *, struct m_tag *);
1726__private_extern__ void m_tag_delete(struct mbuf *, struct m_tag *);
1727__private_extern__ void m_tag_delete_chain(struct mbuf *);
1728__private_extern__ struct m_tag *m_tag_locate(struct mbuf *, u_int32_t,
1729 u_int16_t);
1730__private_extern__ struct m_tag *m_tag_copy(struct m_tag *, int);
1731__private_extern__ int m_tag_copy_chain(struct mbuf *, struct mbuf *, int);
1732__private_extern__ void m_tag_init(struct mbuf *, int);
1733__private_extern__ struct m_tag *m_tag_first(struct mbuf *);
1734__private_extern__ struct m_tag *m_tag_next(struct mbuf *, struct m_tag *);
1735
1736typedef struct m_tag * (*m_tag_kalloc_func_t)(u_int32_t id, u_int16_t type, uint16_t len, int wait);
1737typedef void (*m_tag_kfree_func_t)(struct m_tag *tag);
1738
1739int m_register_internal_tag_type(uint16_t type, uint16_t len, m_tag_kalloc_func_t alloc_func, m_tag_kfree_func_t free_func);
1740void m_tag_create_cookie(struct m_tag *);
1741
1742void mbuf_tag_init(void);
1743
1744__private_extern__ void m_scratch_init(struct mbuf *);
1745__private_extern__ u_int32_t m_scratch_get(struct mbuf *, u_int8_t **);
1746
1747__private_extern__ void m_classifier_init(struct mbuf *, uint32_t);
1748
1749__private_extern__ int m_set_service_class(struct mbuf *, mbuf_svc_class_t);
1750__private_extern__ mbuf_svc_class_t m_get_service_class(struct mbuf *);
1751__private_extern__ mbuf_svc_class_t m_service_class_from_idx(u_int32_t);
1752__private_extern__ mbuf_svc_class_t m_service_class_from_val(u_int32_t);
1753__private_extern__ int m_set_traffic_class(struct mbuf *, mbuf_traffic_class_t);
1754__private_extern__ mbuf_traffic_class_t m_get_traffic_class(struct mbuf *);
1755
1756__private_extern__ struct m_tag *m_tag_alloc(u_int32_t, u_int16_t, int, int);
1757__private_extern__ void mbuf_tag_init(void);
1758
1759#define ADDCARRY(_x) do { \
1760 while (((_x) >> 16) != 0) \
1761 (_x) = ((_x) >> 16) + ((_x) & 0xffff); \
1762} while (0)
1763
1764__private_extern__ u_int16_t m_adj_sum16(struct mbuf *, u_int32_t,
1765 u_int32_t, u_int32_t, u_int32_t);
1766__private_extern__ u_int16_t m_sum16(struct mbuf *, u_int32_t, u_int32_t);
1767
1768__private_extern__ void m_set_ext(struct mbuf *, struct ext_ref *,
1769 m_ext_free_func_t, caddr_t);
1770__private_extern__ struct ext_ref *m_get_rfa(struct mbuf *);
1771__private_extern__ m_ext_free_func_t m_get_ext_free(struct mbuf *);
1772__private_extern__ caddr_t m_get_ext_arg(struct mbuf *);
1773
1774__private_extern__ void m_do_tx_compl_callback(struct mbuf *, struct ifnet *);
1775__private_extern__ mbuf_tx_compl_func m_get_tx_compl_callback(u_int32_t);
1776
1777
1778
1779__END_DECLS
1780#endif /* XNU_KERNEL_PRIVATE */
1781#endif /* !_SYS_MBUF_H_ */
1782