1/*
2 * Copyright (c) 2016-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#ifndef _SKYWALK_OS_PACKET_H_
30#define _SKYWALK_OS_PACKET_H_
31
32#ifdef PRIVATE
33
34#include <stdint.h>
35#include <sys/types.h>
36#include <sys/cdefs.h>
37#include <uuid/uuid.h>
38#include <mach/boolean.h>
39#include <mach/vm_types.h>
40#include <skywalk/os_nexus.h>
41
42/*
43 * @enum packet_svc_class_t
44 * @abstract Service class of a packet
45 * @discussion Property that represents the category of service
46 * of a packet. This information may be used by the driver
47 * and at the link level.
48 * @constant PKT_SC_BK_SYS "Background System-Initiated", high delay
49 * tolerant, high loss tolerant, elastic flow, variable size &
50 * long-lived.
51 * @constant PKT_SC_BK "Background", user-initiated, high delay tolerant,
52 * high loss tolerant, elastic flow, variable size. This level
53 * corresponds to WMM access class "BG".
54 * @constant PKT_SC_BE "Best Effort", unclassified/standard. This is
55 * the default service class; pretty much a mix of everything.
56 * This level corresponds to WMM access class "BE".
57 * @constant PKT_SC_RD
58 * "Responsive Data", a notch higher than "Best Effort", medium
59 * delay tolerant, medium loss tolerant, elastic flow, bursty,
60 * long-lived.
61 * @constant PKT_SC_OAM "Operations, Administration, and Management",
62 * medium delay tolerant, low-medium loss tolerant, elastic &
63 * inelastic flows, variable size.
64 * @constant PKT_SC_AV "Multimedia Audio/Video Streaming", medium delay
65 * tolerant, low-medium loss tolerant, elastic flow, constant
66 * packet interval, variable rate & size.
67 * @constant PKT_SC_RV "Responsive Multimedia Audio/Video", low delay
68 * tolerant, low-medium loss tolerant, elastic flow, variable
69 * packet interval, rate and size.
70 * @constant PKT_SC_VI "Interactive Video", low delay tolerant, low-
71 * medium loss tolerant, elastic flow, constant packet interval,
72 * variable rate & size. This level corresponds to WMM access
73 * class "VI".
74 * @constant PKT_SC_SIG "Signaling", low delay tolerant, low loss
75 * tolerant, inelastic flow, jitter tolerant, rate is bursty but
76 * short, variable size. e.g. SIP. This level corresponds to WMM
77 * access class "VI".
78 * @constant PKT_SC_VO "Interactive Voice", low delay tolerant, low loss
79 * tolerant, inelastic flow, constant packet rate, somewhat fixed
80 * size. This level corresponds to WMM access class "VO" or
81 * PKT_TC_VO.
82 * @constant PKT_SC_CTL "Network Control", low delay tolerant, low loss
83 * tolerant, inelastic flow, rate is short & burst, variable size.
84 */
85typedef enum {
86 PKT_SC_BK_SYS = 0x00080090, /* lowest class */
87 PKT_SC_BK = 0x00100080,
88
89 PKT_SC_BE = 0x00000000,
90 PKT_SC_RD = 0x00180010,
91 PKT_SC_OAM = 0x00200020,
92
93 PKT_SC_AV = 0x00280120,
94 PKT_SC_RV = 0x00300110,
95 PKT_SC_VI = 0x00380100,
96 PKT_SC_SIG = 0x00380130,
97
98 PKT_SC_VO = 0x00400180,
99 PKT_SC_CTL = 0x00480190, /* highest class */
100} packet_svc_class_t;
101
102/* CSTYLED */
103/*!
104 * @enum packet_traffic_class_t
105 * @abstract Traffic class of a packet
106 * @discussion Property that represent the category of traffic of a packet.
107 * This information may be used by the driver and at the link level.
108 * @constant PKT_TC_BE Best effort, normal class.
109 * @constant PKT_TC_BK Background, low priority or bulk traffic.
110 * @constant PKT_TC_VI Interactive video, constant bit rate, low latency.
111 * @constant PKT_TC_VO Interactive voice, constant bit rate, lowest latency.
112 */
113typedef enum {
114 PKT_TC_BE = 0,
115 PKT_TC_BK = 1,
116 PKT_TC_VI = 2,
117 PKT_TC_VO = 3,
118} packet_traffic_class_t;
119
120/*
121 * These conversion macros rely on the corresponding PKT_SC and
122 * PKT_TC values in order to establish the following mapping:
123 *
124 * PKT_SC_BK_SYS ] ==> PKT_TC_BK
125 * PKT_SC_BK ]
126 *
127 * PKT_SC_BE ] ==> PKT_TC_BE
128 * PKT_SC_RD ]
129 * PKT_SC_OAM ]
130 *
131 * PKT_SC_AV ] ==> PKT_TC_VI
132 * PKT_SC_RV ]
133 * PKT_SC_VI ]
134 * PKT_SC_SIG ]
135 *
136 * PKT_SC_VO ] ==> PKT_TC_VO
137 * PKT_SC_CTL ]
138 *
139 * The values assigned to each service class allows for a fast mapping to
140 * the corresponding PKT_TC traffic class values, as well as to retrieve the
141 * assigned index; therefore care must be taken when comparing against these
142 * values. Use the corresponding class and index macros to retrieve the
143 * corresponding portion, and never assume that a higher class corresponds
144 * to a higher index.
145 */
146#define PKT_SCVAL(x) ((x) & 0xffff)
147#define PKT_SC2TC(_sc) (PKT_SCVAL(_sc) >> 7)
148#define PKT_TC2SCVAL(_tc) ((_tc) << 7)
149
150#define PKT_SCVAL_BK_SYS PKT_SCVAL(PKT_SC_BK_SYS)
151#define PKT_SCVAL_BK PKT_SCVAL(PKT_SC_BK)
152#define PKT_SCVAL_BE PKT_SCVAL(PKT_SC_BE)
153#define PKT_SCVAL_RD PKT_SCVAL(PKT_SC_RD)
154#define PKT_SCVAL_OAM PKT_SCVAL(PKT_SC_OAM)
155#define PKT_SCVAL_AV PKT_SCVAL(PKT_SC_AV)
156#define PKT_SCVAL_RV PKT_SCVAL(PKT_SC_RV)
157#define PKT_SCVAL_VI PKT_SCVAL(PKT_SC_VI)
158#define PKT_SCVAL_SIG PKT_SCVAL(PKT_SC_SIG)
159#define PKT_SCVAL_VO PKT_SCVAL(PKT_SC_VO)
160#define PKT_SCVAL_CTL PKT_SCVAL(PKT_SC_CTL)
161
162/*
163 * Packet checksum offload flags.
164 */
165typedef uint32_t packet_csum_flags_t;
166typedef uint32_t packet_trace_id_t;
167typedef uint32_t packet_flowid_t;
168typedef uint16_t packet_trace_tag_t;
169
170/*
171 * PACKET_CSUM_PARTIAL indicates the following:
172 *
173 * On transmit, the start and stuff offsets are significant, and that the
174 * module setting this information is requesting that the layer below it
175 * compute 16-bit 1's complement sum from the location marked by the start
176 * offset to the end of the packet, and store the resulted sum at the
177 * location marked by the stuff offset. If PACKET_CSUM_ZERO_INVERT is set,
178 * and if the resulted sum is 0, it will be converted to -0 (0xffff).
179 *
180 * On receive, the start offset and checksum value are significant, and
181 * that the module computing the 16-bit 1's complement and setting this
182 * information is requesting that the layer above it perform any necessary
183 * adjustments to exclude/include data span that's not applicable, as well
184 * as to validate the checksum value.
185 */
186#define PACKET_CSUM_PARTIAL 0x01 /* partial one's complement */
187#define PACKET_CSUM_ZERO_INVERT 0x02 /* invert resulted 0 to 0xffff */
188
189#define PACKET_CSUM_IP 0x0004
190#define PACKET_CSUM_TCP 0x0008
191#define PACKET_CSUM_UDP 0x0010
192#define PACKET_CSUM_TCPIPV6 0x0020
193#define PACKET_CSUM_UDPIPV6 0x0040
194
195/*
196 * Below flags are for RX.
197 */
198#define PACKET_CSUM_IP_CHECKED 0x0100 /* did IP checksum */
199#define PACKET_CSUM_IP_VALID 0x0200 /* and the IP checksum is valid */
200#define PACKET_CSUM_DATA_VALID 0x0400 /* csum_rx_val is valid */
201#define PACKET_CSUM_PSEUDO_HDR 0x0800 /* csum_rx_val includes pseudo hdr */
202
203typedef enum : uint32_t {
204 PACKET_TSO_IPV4 = 0x00100000,
205 PACKET_TSO_IPV6 = 0x00200000,
206} packet_tso_flags_t;
207
208#define PACKET_CSUM_TSO_IPV4 0x00100000
209#define PACKET_CSUM_TSO_IPV6 0x00200000
210
211#define PACKET_HAS_VALID_IP_CSUM(_p) \
212 (((_p)->pkt_csum_flags & (PACKET_CSUM_IP_CHECKED | PACKET_CSUM_IP_VALID)) \
213 == (PACKET_CSUM_IP_CHECKED | PACKET_CSUM_IP_VALID))
214
215#define PACKET_HAS_PARTIAL_CHECKSUM(_p) \
216 ((_p)->pkt_csum_flags & (PACKET_CSUM_PARTIAL))
217
218#define PACKET_CSUM_RX_FULL_FLAGS \
219 (PACKET_CSUM_IP_CHECKED | PACKET_CSUM_IP_VALID | \
220 PACKET_CSUM_DATA_VALID | PACKET_CSUM_PSEUDO_HDR)
221
222#define PACKET_CSUM_RX_FLAGS \
223 (PACKET_CSUM_RX_FULL_FLAGS | PACKET_CSUM_PARTIAL)
224
225#define PACKET_CSUM_TSO_FLAGS \
226 (PACKET_CSUM_TSO_IPV4 | PACKET_CSUM_TSO_IPV6)
227
228#define PACKET_HAS_FULL_CHECKSUM_FLAGS(_p) \
229 (((_p)->pkt_csum_flags & PACKET_CSUM_RX_FULL_FLAGS) == PACKET_CSUM_RX_FULL_FLAGS)
230
231#define PACKET_TX_CSUM_OFFLOAD_FLAGS \
232 (PACKET_CSUM_IP | PACKET_CSUM_TCP | PACKET_CSUM_UDP | \
233 PACKET_CSUM_TCPIPV6 | PACKET_CSUM_UDPIPV6 | PACKET_CSUM_ZERO_INVERT)
234
235#define PACKET_CSUM_FLAGS \
236 (PACKET_TX_CSUM_OFFLOAD_FLAGS | PACKET_CSUM_RX_FLAGS | PACKET_CSUM_TSO_FLAGS)
237/*
238 * TODO: adi@apple.com -- these are temporary and should be removed later.
239 */
240#define OS_PACKET_HAS_CHECKSUM_API 1
241#define OS_PACKET_HAS_SEGMENT_COUNT 1
242#define OS_PACKET_HAS_TRACING_API 1
243#define OS_PACKET_HAS_SEGMENT_SIZE 1
244
245/*
246 * Valid values for pkt_aggr_type.
247 */
248#define PKT_AGGR_NONE 0x00 /* no aggregation */
249#define PKT_AGGR_IP_CHAIN 0x01 /* buflet chain of discrete IP packets containing contiguous L4 frames */
250#define PKT_AGGR_SINGLE_IP 0x02 /* buflet chain representing single IP packet */
251#define PKT_AGGR_SINGLE_IP_PACKED 0x03 /* buflet chain representing single IP packet in packed format */
252
253/*
254 * packet_id is a per packet metadata which can be set on a packet by the
255 * application. It can be used to identify either an individual or a group
256 * of packets within the networking stack and driver.
257 */
258typedef struct packet_id {
259 /*
260 * version of this structure.
261 */
262 uint8_t pktid_version;
263#define OS_PACKET_PKTID_VERSION_1 1
264#define OS_PACKET_PKTID_VERSION_CURRENT OS_PACKET_PKTID_VERSION_1
265 /*
266 * payload type of the packet, opaque to the network stack.
267 */
268 uint8_t pktid_payload_type;
269 /*
270 * packet sequence number, monotonically increasing.
271 */
272 uint16_t pktid_sequence_number;
273 /*
274 * packet timestamp, monotonically increasing and opaque to the
275 * network stack. Sample rate of the timestamp clock is determined
276 * by the application.
277 */
278 uint32_t pktid_timestamp;
279 /*
280 * Identifier for streams defined by the application.
281 */
282 uint32_t pktid_stream_identifier;
283 /*
284 * reserved for future use.
285 */
286 uint32_t _reserved;
287} packet_id_t;
288
289/*
290 * Packet Trace code
291 * Used with os_packet_trace_* functions.
292 * Total of 12bit (0xABC) code space available, current sub-code allocation is:
293 * 0x00C code space for FSW Rx path.
294 * 0x01C code space for FSW Tx path.
295 *
296 * More sub-code can be added for other packet data path, e.g. uPipe, BSD, etc.
297 *
298 * Note:
299 * 1. Needs to include <sys/kdebug.h> to use the values.
300 * 2. When making changes to sub-code/value, update static assertions in
301 * pp_init and probe list ariadne-plists/skywalk-tracepoints.plist.
302 *
303 */
304/* Rx Group */
305#define PKT_TRACE_RX_DRV_START (SKYWALKDBG_CODE(DBG_SKYWALK_PACKET, 0x001) | DBG_FUNC_START)
306#define PKT_TRACE_RX_DRV_END (SKYWALKDBG_CODE(DBG_SKYWALK_PACKET, 0x001) | DBG_FUNC_END)
307
308#define PKT_TRACE_RX_FSW_START (SKYWALKDBG_CODE(DBG_SKYWALK_PACKET, 0x002) | DBG_FUNC_START)
309#define PKT_TRACE_RX_FSW_END (SKYWALKDBG_CODE(DBG_SKYWALK_PACKET, 0x002) | DBG_FUNC_END)
310
311#define PKT_TRACE_RX_CHN_START (SKYWALKDBG_CODE(DBG_SKYWALK_PACKET, 0x003) | DBG_FUNC_START)
312#define PKT_TRACE_RX_CHN_END (SKYWALKDBG_CODE(DBG_SKYWALK_PACKET, 0x003) | DBG_FUNC_END)
313
314
315/* Tx Group */
316#define PKT_TRACE_TX_FSW_START (SKYWALKDBG_CODE(DBG_SKYWALK_PACKET, 0x010) | DBG_FUNC_START)
317#define PKT_TRACE_TX_FSW_END (SKYWALKDBG_CODE(DBG_SKYWALK_PACKET, 0x010) | DBG_FUNC_END)
318
319#define PKT_TRACE_TX_AQM_START (SKYWALKDBG_CODE(DBG_SKYWALK_PACKET, 0x011) | DBG_FUNC_START)
320#define PKT_TRACE_TX_AQM_END (SKYWALKDBG_CODE(DBG_SKYWALK_PACKET, 0x011) | DBG_FUNC_END)
321
322#define PKT_TRACE_TX_DRV_START (SKYWALKDBG_CODE(DBG_SKYWALK_PACKET, 0x012) | DBG_FUNC_START)
323#define PKT_TRACE_TX_DRV_END (SKYWALKDBG_CODE(DBG_SKYWALK_PACKET, 0x012) | DBG_FUNC_END)
324
325/*
326 * @enum packet_expiry_action_t
327 * @abstract The desired action(s) to take upon the expiration of the packet
328 * @discussion Bitfield property to express the desired actions to take when the packet expires.
329 * At the moment the actions are only taken for the TX packets.
330 *
331 * @constant PACKET_EXPIRY_ACTION_DROP - drop the packet
332 * @constant PACKET_EXPIRY_ACTION_NOTIFY - notify the upper layers
333 */
334typedef enum {
335 PACKET_EXPIRY_ACTION_NONE = 0x0,
336 PACKET_EXPIRY_ACTION_DROP = 0x1,
337 PACKET_EXPIRY_ACTION_NOTIFY = 0x2,
338} packet_expiry_action_t;
339
340/*
341 * @enum packet_app_metadata_type_t
342 * @abstract Application specific metadata which can be used by the Network
343 * Stack or Network Interface Driver/firmware.
344 * @discussion Supported Application types are:
345 * @constant PACKET_APP_METADATA_TYPE_VOICE - voice application
346 */
347typedef enum : uint8_t {
348#if defined(LIBSYSCALL_INTERFACE) || defined(BSD_KERNEL_PRIVATE)
349 PACKET_APP_METADATA_TYPE_UNSPECIFIED = 0,
350#endif /* LIBSYSCALL_INTERFACE || BSD_KERNEL_PRIVATE */
351 PACKET_APP_METADATA_TYPE_VOICE = 1,
352#if defined(LIBSYSCALL_INTERFACE) || defined(BSD_KERNEL_PRIVATE)
353 PACKET_APP_METADATA_TYPE_MIN = PACKET_APP_METADATA_TYPE_VOICE,
354 PACKET_APP_METADATA_TYPE_MAX = PACKET_APP_METADATA_TYPE_VOICE,
355#endif /* LIBSYSCALL_INTERFACE || BSD_KERNEL_PRIVATE */
356} packet_app_metadata_type_t;
357
358/*
359 * @enum packet_voice_app_metadata_t
360 * @abstract Voice application specific metadata.
361 * @discussion Supported Voice application metadata values are:
362 * @constant PACKET_VOICE_APP_METADATA_UNSPECIFIED - metadata not specified.
363 * @constant PACKET_VOICE_APP_METADATA_SPEECH - speech frame.
364 * @constant PACKET_VOICE_APP_METADATA_SILENCE - silence frame.
365 * @constant PACKET_VOICE_APP_METADATA_OTHER - RTCP XR, RTCP, or DTMF.
366 */
367typedef enum : uint8_t {
368 PACKET_VOICE_APP_METADATA_UNSPECIFIED = 0x0,
369 PACKET_VOICE_APP_METADATA_SPEECH = 0x1,
370 PACKET_VOICE_APP_METADATA_SILENCE = 0x2,
371 PACKET_VOICE_APP_METADATA_OTHER = 0x3,
372} packet_voice_app_metadata_t;
373
374#ifndef KERNEL
375/*
376 * User APIs.
377 */
378
379/*
380 * Opaque handles.
381 */
382struct __user_buflet;
383typedef uint64_t packet_t;
384typedef struct __user_buflet *buflet_t;
385
386#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
387__BEGIN_DECLS
388/*
389 * Packets specific.
390 */
391extern int os_packet_set_headroom(const packet_t, const uint8_t);
392extern uint8_t os_packet_get_headroom(const packet_t);
393extern int os_packet_set_link_header_length(const packet_t, const uint8_t);
394extern uint8_t os_packet_get_link_header_length(const packet_t);
395extern int os_packet_set_link_broadcast(const packet_t);
396extern boolean_t os_packet_get_link_broadcast(const packet_t);
397extern int os_packet_set_link_multicast(const packet_t);
398extern boolean_t os_packet_get_link_multicast(const packet_t);
399extern int os_packet_set_link_ethfcs(const packet_t);
400extern boolean_t os_packet_get_link_ethfcs(const packet_t);
401extern int os_packet_set_transport_traffic_background(const packet_t);
402extern boolean_t os_packet_get_transport_traffic_background(const packet_t);
403extern int os_packet_set_transport_traffic_realtime(const packet_t);
404extern boolean_t os_packet_get_transport_traffic_realtime(const packet_t);
405extern int os_packet_set_transport_retransmit(const packet_t);
406extern boolean_t os_packet_get_transport_retransmit(const packet_t);
407extern int os_packet_set_transport_last_packet(const packet_t);
408extern int os_packet_set_service_class(const packet_t,
409 const packet_svc_class_t);
410extern packet_svc_class_t os_packet_get_service_class(const packet_t);
411extern int os_packet_set_compression_generation_count(const packet_t, const uint32_t);
412extern int os_packet_get_compression_generation_count(const packet_t, uint32_t *);
413extern int os_packet_set_traffic_class(const packet_t, packet_traffic_class_t);
414extern packet_traffic_class_t os_packet_get_traffic_class(const packet_t);
415extern int os_packet_set_inet_checksum(const packet_t,
416 const packet_csum_flags_t, const uint16_t, const uint16_t);
417#define HAS_OS_PACKET_ADD_CSUMF 1
418extern void os_packet_add_inet_csum_flags(const packet_t, const packet_csum_flags_t);
419extern packet_csum_flags_t os_packet_get_inet_checksum(const packet_t,
420 uint16_t *, uint16_t *);
421extern void os_packet_set_group_start(const packet_t);
422extern boolean_t os_packet_get_group_start(const packet_t);
423extern void os_packet_set_group_end(const packet_t);
424extern boolean_t os_packet_get_group_end(const packet_t);
425extern int os_packet_set_expire_time(const packet_t, const uint64_t);
426extern int os_packet_get_expire_time(const packet_t, uint64_t *);
427#define HAS_OS_PACKET_EXPIRY_ACTION 1
428extern int os_packet_set_expiry_action(const packet_t, const packet_expiry_action_t);
429extern int os_packet_get_expiry_action(const packet_t, packet_expiry_action_t *);
430extern int os_packet_set_token(const packet_t, const void *, const uint16_t);
431extern int os_packet_get_packetid(const packet_t, packet_id_t *);
432extern int os_packet_set_packetid(const packet_t, packet_id_t *);
433extern int os_packet_set_vlan_tag(const packet_t, const uint16_t,
434 const boolean_t);
435extern int os_packet_get_vlan_tag(const packet_t, uint16_t *, boolean_t *);
436extern uint16_t os_packet_get_vlan_id(const uint16_t);
437extern uint8_t os_packet_get_vlan_priority(const uint16_t);
438#define HAS_OS_PACKET_GET_WAKE_FLAG 1
439extern boolean_t os_packet_get_wake_flag(const packet_t);
440#define HAS_OS_PACKET_KEEP_ALIVE 1
441extern boolean_t os_packet_get_keep_alive(const packet_t);
442extern void os_packet_set_keep_alive(const packet_t, const boolean_t);
443extern boolean_t os_packet_get_truncated(const packet_t);
444extern uint8_t os_packet_get_aggregation_type(const packet_t);
445extern int os_packet_set_app_metadata(const packet_t,
446 const packet_app_metadata_type_t, const uint8_t);
447extern int os_packet_set_protocol_segment_size(const packet_t, const uint16_t);
448extern void os_packet_set_tso_flags(const packet_t, packet_tso_flags_t);
449#define AQM_SUPPORTS_L4S 1
450extern void os_packet_set_l4s_flag(const packet_t);
451#define AQM_SUPPORTS_PACING 1
452extern void os_packet_set_tx_timestamp(const packet_t ph, const uint64_t ts);
453/*
454 * Quantum & Packets.
455 */
456extern void os_packet_set_flow_uuid(const packet_t, const uuid_t flow_uuid);
457extern void os_packet_get_flow_uuid(const packet_t, uuid_t *flow_uuid);
458extern void os_packet_clear_flow_uuid(const packet_t);
459extern uint32_t os_packet_get_data_length(const packet_t);
460extern uint32_t os_packet_get_buflet_count(const packet_t);
461extern buflet_t os_packet_get_next_buflet(const packet_t, const buflet_t);
462extern uint32_t os_packet_get_segment_count(const packet_t ph);
463extern int os_packet_finalize(const packet_t);
464extern int os_packet_add_buflet(const packet_t ph, const buflet_t bprev,
465 const buflet_t bnew);
466/* increment use count on packet */
467extern int os_packet_increment_use_count(const packet_t ph);
468/* decrement use count on packet and retrieve new value */
469extern int os_packet_decrement_use_count(const packet_t ph, uint16_t *use_cnt);
470
471extern packet_trace_id_t os_packet_get_trace_id(const packet_t ph);
472extern void os_packet_set_trace_id(const packet_t ph, packet_trace_id_t);
473extern void os_packet_trace_event(const packet_t ph, uint32_t);
474
475/*
476 * Misc.
477 */
478extern uint32_t os_inet_checksum(const void *, uint32_t, uint32_t);
479extern uint32_t os_copy_and_inet_checksum(const void *, void *,
480 uint32_t, uint32_t);
481
482/*
483 * Buflets.
484 */
485#define SUPPORT_LARGE_BUFFER 1
486extern int os_buflet_set_data_offset(const buflet_t, const uint32_t);
487extern uint32_t os_buflet_get_data_offset(const buflet_t);
488extern int os_buflet_set_data_length(const buflet_t, const uint32_t);
489extern uint32_t os_buflet_get_data_length(const buflet_t);
490extern void *os_buflet_get_object_address(const buflet_t);
491extern uint32_t os_buflet_get_object_limit(const buflet_t);
492extern void *os_buflet_get_data_address(const buflet_t);
493extern uint32_t os_buflet_get_data_limit(const buflet_t);
494__END_DECLS
495#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
496#else /* KERNEL */
497/*
498 * Kernel APIs.
499 */
500
501/*
502 * Opaque handles.
503 */
504struct __kern_buflet;
505struct kern_pbufpool;
506struct sksegment;
507
508typedef struct kern_pbufpool *kern_pbufpool_t;
509typedef uint64_t kern_packet_t;
510typedef uint32_t kern_packet_idx_t;
511typedef struct __kern_buflet *kern_buflet_t;
512typedef uint32_t kern_obj_idx_seg_t;
513typedef struct sksegment *kern_segment_t;
514typedef uint32_t kern_segment_idx_t;
515
516/*
517 * @typedef pbuf_seg_ctor_fn_t
518 * @abstract Buffer segment constructor callback.
519 * @param buf_seg Buffer segment handle.
520 * @param buf_desc IOSKMemoryDescriptor handle for buffer segment.
521 * @discussion This callback is invoked when a segment has been activated.
522 * If applicable, the owner should wire/prepare the memory and insert
523 * it into the memory mapper (IOMMU/DART) prior to returning.
524 */
525typedef void (*pbuf_seg_ctor_fn_t)(const kern_pbufpool_t,
526 const kern_segment_t buf_seg, const IOSKMemoryDescriptor buf_desc);
527
528/*
529 * @typedef pbuf_seg_dtor_fn_t
530 * @abstract Buffer segment destructor callback.
531 * @param buf_seg Buffer segment handle.
532 * @param buf_desc IOSKMemoryDescriptor handle for buffer segment.
533 * @discussion This callback is invoked when a segment is about to be
534 * freed. The owner should reverse what had been done earlier
535 * at pbuf_seg_ctor_fn_t() constructor time. If applicable,
536 * it should remove the memory from mapper (IOMMU/DART), and
537 * unwire/complete it prior to returning.
538 */
539typedef void (*pbuf_seg_dtor_fn_t)(const kern_pbufpool_t,
540 const kern_segment_t buf_seg, const IOSKMemoryDescriptor buf_desc);
541
542typedef void (*pbuf_ctx_retain_fn_t)(void *const ctx);
543typedef void (*pbuf_ctx_release_fn_t)(void *const ctx);
544
545typedef uint8_t pbufpool_name_t[64];
546
547/*
548 * Kernel packet buffer pool init.
549 */
550struct kern_pbufpool_init {
551 uint32_t kbi_version; /* current version */
552 pbufpool_name_t kbi_name; /* optional */
553 uint32_t kbi_flags; /* see KBI_* */
554 uint32_t kbi_packets; /* required */
555 uint32_t kbi_max_frags; /* max buflets per packet */
556 uint32_t kbi_buflets; /* >= packets (optional) */
557 uint32_t kbi_bufsize; /* required */
558 uint32_t kbi_buf_seg_size; /* optional */
559 pbuf_seg_ctor_fn_t kbi_buf_seg_ctor; /* optional */
560 pbuf_seg_dtor_fn_t kbi_buf_seg_dtor; /* optional */
561 void * kbi_ctx; /* optional */
562 pbuf_ctx_retain_fn_t kbi_ctx_retain; /* optional */
563 pbuf_ctx_release_fn_t kbi_ctx_release; /* optional */
564};
565
566#define KBIF_QUANTUM 0x1 /* simple packet (non-networking) */
567#define KBIF_PERSISTENT 0x2 /* persistent memory (wired) */
568#define KBIF_MONOLITHIC 0x4 /* single segment mode */
569#define KBIF_BUFFER_ON_DEMAND 0x8 /* attach/detach buffers on demand */
570#define KBIF_INHIBIT_CACHE 0x10 /* caching-inhibited */
571#define KBIF_USER_ACCESS 0x20 /* allow userspace access */
572#define KBIF_VIRTUAL_DEVICE 0x40 /* device is virtual (no DMA) */
573#define KBIF_PHYS_CONTIGUOUS 0x80 /* physically-contiguous segment(s) */
574#define KBIF_IODIR_IN 0x100 /* io direction in (device to host) */
575#define KBIF_IODIR_OUT 0x200 /* io direction out (host to device) */
576#define KBIF_KERNEL_READONLY 0x400 /* kernel read-only */
577#define KBIF_NO_MAGAZINES 0x800 /* disable per-CPU magazines layer */
578#define KBIF_THREADSAFE 0x2000 /* thread safe memory descriptor */
579
580#define KERN_PBUFPOOL_VERSION_1 1
581#define KERN_PBUFPOOL_VERSION_2 2 /* added ctx retain/release */
582#define KERN_PBUFPOOL_CURRENT_VERSION KERN_PBUFPOOL_VERSION_2
583
584/*
585 * Kernel packet buffer pool memory info.
586 */
587struct kern_pbufpool_memory_info {
588 uint32_t kpm_flags; /* see KPMF_* */
589 uint32_t kpm_packets; /* number of packets */
590 uint32_t kpm_max_frags; /* max buflets per packet */
591 uint32_t kpm_buflets; /* number of buflets */
592 uint32_t kpm_bufsize; /* size of each buffer */
593 uint32_t kpm_bufsegs; /* number of buffer segments */
594 uint32_t kpm_buf_seg_size; /* size of a buffer segment */
595 uint32_t kpm_buf_obj_size; /* size of the shared buffer object */
596} __attribute__((aligned(64)));
597
598#define KPMF_EXTERNAL 0x1 /* externally configured */
599
600/*
601 * Kernel packet representation of packet_svc_class_t.
602 *
603 * We have a separate enum to separate the namespace just in case we need it.
604 */
605typedef enum {
606#ifdef BSD_KERNEL_PRIVATE
607 KPKT_SC_UNSPEC = -1, /* Internal: not specified */
608#endif /* BSD_KERNEL_PRIVATE */
609 KPKT_SC_BK_SYS = PKT_SC_BK_SYS, /* lowest class */
610 KPKT_SC_BK = PKT_SC_BK,
611
612 KPKT_SC_BE = PKT_SC_BE,
613 KPKT_SC_RD = PKT_SC_RD,
614 KPKT_SC_OAM = PKT_SC_OAM,
615
616 KPKT_SC_AV = PKT_SC_AV,
617 KPKT_SC_RV = PKT_SC_RV,
618 KPKT_SC_VI = PKT_SC_VI,
619 KPKT_SC_SIG = PKT_SC_SIG,
620
621 KPKT_SC_VO = PKT_SC_VO,
622 KPKT_SC_CTL = PKT_SC_CTL, /* highest class */
623} kern_packet_svc_class_t;
624
625/* Maximum number of KPKT_SC values (excluding KPKT_SC_UNSPEC) */
626#define KPKT_SC_MAX_CLASSES 10
627
628#define KPKT_VALID_SVC(c) \
629 (c == KPKT_SC_BK_SYS || c == KPKT_SC_BK || c == KPKT_SC_BE || \
630 c == KPKT_SC_RD || c == KPKT_SC_OAM || c == KPKT_SC_AV || \
631 c == KPKT_SC_RV || c == KPKT_SC_VI || c == KPKT_SC_SIG || \
632 c == KPKT_SC_VO || c == KPKT_SC_CTL)
633
634#define KPKT_SVCIDX(c) ((((c) >> 16) & 0xff) >> 3)
635
636/*
637 * Kernel packet representation of packet_traffic_class_t.
638 *
639 * We have a separate enum to separate the namespace just in case we need it.
640 */
641typedef enum {
642#ifdef BSD_KERNEL_PRIVATE
643 KPKT_TC_UNSPEC = -1, /* Internal: not specified */
644#endif /* BSD_KERNEL_PRIVATE */
645 KPKT_TC_BE = PKT_TC_BE,
646 KPKT_TC_BK = PKT_TC_BK,
647 KPKT_TC_VI = PKT_TC_VI,
648 KPKT_TC_VO = PKT_TC_VO,
649#ifdef BSD_KERNEL_PRIVATE
650 KPKT_TC_MAX = 4, /* Internal: traffic class count */
651#endif /* BSD_KERNEL_PRIVATE */
652} kern_packet_traffic_class_t;
653
654/*
655 * Modes for cloning a kernel packet.
656 *
657 * The "heavy" mode copies most of the metadata (except those pertaining
658 * to linkages to other objects), allocates new buffer(s) for the
659 * cloned packet, and copies old buffer(s) to new one(s).
660 *
661 * The "light" mode is to be used on a packet that's recently allocated,
662 * as the cloning process involves copying minimal metadata information,
663 * as well as adding reference(s) to the buffer(s) rather than copying.
664 */
665typedef enum {
666 KPKT_COPY_HEAVY = 0, /* copy everything including buffers */
667 KPKT_COPY_LIGHT /* minimal copy, adding refs to buffers */
668} kern_packet_copy_mode_t;
669
670
671__BEGIN_DECLS
672/*
673 * Packets specific.
674 */
675extern errno_t kern_packet_set_headroom(const kern_packet_t, const uint8_t);
676extern uint8_t kern_packet_get_headroom(const kern_packet_t);
677/* deprecated -- use kern_packet_set_headroom instead */
678extern errno_t kern_packet_set_link_header_offset(const kern_packet_t,
679 const uint8_t);
680/* deprecated -- use kern_packet_get_headroom instead */
681extern uint16_t kern_packet_get_link_header_offset(const kern_packet_t);
682extern errno_t kern_packet_set_link_header_length(const kern_packet_t,
683 const uint8_t);
684extern uint8_t kern_packet_get_link_header_length(const kern_packet_t);
685extern errno_t kern_packet_set_link_broadcast(const kern_packet_t);
686extern boolean_t kern_packet_get_link_broadcast(const kern_packet_t);
687extern errno_t kern_packet_set_link_multicast(const kern_packet_t);
688extern boolean_t kern_packet_get_link_multicast(const kern_packet_t);
689extern errno_t kern_packet_set_link_ethfcs(const kern_packet_t);
690extern boolean_t kern_packet_get_link_ethfcs(const kern_packet_t);
691/* deprecated -- use kern_packet_set_link_header_length instead */
692extern errno_t kern_packet_set_network_header_offset(const kern_packet_t,
693 const uint16_t);
694/* deprecated -- use kern_packet_get_link_header_length instead */
695extern uint16_t kern_packet_get_network_header_offset(const kern_packet_t);
696extern errno_t kern_packet_set_transport_traffic_background(
697 const kern_packet_t);
698extern boolean_t kern_packet_get_transport_traffic_background(
699 const kern_packet_t);
700extern errno_t kern_packet_set_transport_traffic_realtime(const kern_packet_t);
701extern boolean_t kern_packet_get_transport_traffic_realtime(
702 const kern_packet_t);
703/* deprecated */
704extern errno_t kern_packet_set_transport_header_offset(const kern_packet_t,
705 const uint16_t);
706/* deprecated */
707extern uint16_t kern_packet_get_transport_header_offset(const kern_packet_t);
708extern errno_t kern_packet_set_transport_retransmit(const kern_packet_t);
709extern boolean_t kern_packet_get_transport_retransmit(const kern_packet_t);
710extern boolean_t kern_packet_get_transport_new_flow(const kern_packet_t);
711extern boolean_t kern_packet_get_transport_last_packet(const kern_packet_t);
712extern errno_t kern_packet_set_service_class(const kern_packet_t,
713 const kern_packet_svc_class_t);
714extern kern_packet_svc_class_t kern_packet_get_service_class(
715 const kern_packet_t);
716extern errno_t kern_packet_get_service_class_index(
717 const kern_packet_svc_class_t, uint32_t *);
718#define HAS_KERN_PACKET_COMPRESSION_GENERATION_COUNT 1
719extern errno_t kern_packet_set_compression_generation_count(const kern_packet_t,
720 const uint32_t);
721extern errno_t kern_packet_get_compression_generation_count(const kern_packet_t, uint32_t *);
722extern boolean_t kern_packet_is_high_priority(
723 const kern_packet_t);
724extern errno_t kern_packet_set_traffic_class(const kern_packet_t,
725 kern_packet_traffic_class_t);
726extern kern_packet_traffic_class_t kern_packet_get_traffic_class(
727 const kern_packet_t);
728extern errno_t kern_packet_set_inet_checksum(const kern_packet_t,
729 const packet_csum_flags_t, const uint16_t, const uint16_t, boolean_t);
730extern packet_csum_flags_t kern_packet_get_inet_checksum(const kern_packet_t,
731 uint16_t *, uint16_t *, boolean_t);
732extern errno_t kern_packet_get_timestamp(const kern_packet_t, uint64_t *,
733 boolean_t *);
734extern errno_t kern_packet_set_timestamp(const kern_packet_t, uint64_t,
735 boolean_t);
736extern errno_t kern_packet_get_timestamp_requested(const kern_packet_t,
737 boolean_t *);
738extern errno_t kern_packet_get_tx_completion_status(const kern_packet_t,
739 kern_return_t *);
740extern errno_t kern_packet_set_tx_completion_status(const kern_packet_t,
741 kern_return_t);
742extern void kern_packet_tx_completion(const kern_packet_t, ifnet_t);
743extern void kern_packet_set_group_start(const kern_packet_t);
744extern boolean_t kern_packet_get_group_start(const kern_packet_t);
745extern void kern_packet_set_group_end(const kern_packet_t);
746extern boolean_t kern_packet_get_group_end(const kern_packet_t);
747extern errno_t kern_packet_set_expire_time(const kern_packet_t, const uint64_t);
748extern errno_t kern_packet_get_expire_time(const kern_packet_t, uint64_t *);
749extern errno_t kern_packet_set_token(const kern_packet_t, const void *,
750 const uint16_t);
751extern errno_t kern_packet_get_token(const kern_packet_t, void *, uint16_t *);
752extern errno_t kern_packet_get_packetid(const kern_packet_t, packet_id_t *);
753extern errno_t kern_packet_set_vlan_tag(const kern_packet_t, const uint16_t,
754 const boolean_t);
755extern errno_t kern_packet_get_vlan_tag(const kern_packet_t, uint16_t *,
756 boolean_t *);
757extern uint16_t kern_packet_get_vlan_id(const uint16_t);
758extern uint8_t kern_packet_get_vlan_priority(const uint16_t);
759extern void kern_packet_set_wake_flag(const kern_packet_t);
760extern boolean_t kern_packet_get_wake_flag(const kern_packet_t);
761extern errno_t kern_packet_set_fpd_sequence_number(const kern_packet_t,
762 uint32_t);
763extern errno_t kern_packet_set_fpd_context_id(const kern_packet_t, uint16_t);
764extern errno_t kern_packet_set_fpd_command(const kern_packet_t, uint8_t);
765extern errno_t kern_packet_get_flowid(const kern_packet_t, packet_flowid_t *);
766extern void kern_packet_set_trace_tag(const kern_packet_t ph, packet_trace_tag_t tag);
767extern packet_trace_tag_t kern_packet_get_trace_tag(const kern_packet_t ph);
768extern errno_t kern_packet_get_tx_nexus_port_id(const kern_packet_t,
769 uint32_t *);
770extern errno_t kern_packet_get_app_metadata(const kern_packet_t,
771 packet_app_metadata_type_t *, uint8_t *);
772#define NEW_KERN_PACKET_GET_PROTOCOL_SEGMENT_SIZE 1
773extern uint16_t kern_packet_get_protocol_segment_size(const kern_packet_t);
774extern void * kern_packet_get_priv(const kern_packet_t);
775extern void kern_packet_set_priv(const kern_packet_t, void *);
776extern void kern_packet_get_tso_flags(const kern_packet_t, packet_tso_flags_t *);
777void kern_packet_set_segment_count(const kern_packet_t, uint8_t);
778
779/*
780 * Expiry notification
781 */
782extern errno_t kern_packet_set_expiry_action(const kern_packet_t, const packet_expiry_action_t);
783extern errno_t kern_packet_get_expiry_action(const kern_packet_t, packet_expiry_action_t *);
784extern errno_t kern_packet_check_for_expiry_and_notify(const kern_packet_t ph, ifnet_t ifp,
785 uint16_t origin, uint16_t status);
786
787/*
788 * Quantum & Packets.
789 */
790extern void kern_packet_set_flow_uuid(const kern_packet_t, const uuid_t);
791extern void kern_packet_get_flow_uuid(const kern_packet_t, uuid_t *);
792extern void kern_packet_clear_flow_uuid(const kern_packet_t);
793extern void kern_packet_get_euuid(const kern_packet_t, uuid_t);
794extern void kern_packet_set_policy_id(const kern_packet_t, uint32_t);
795extern uint32_t kern_packet_get_policy_id(const kern_packet_t);
796extern void kern_packet_set_skip_policy_id(const kern_packet_t, uint32_t);
797extern uint32_t kern_packet_get_skip_policy_id(const kern_packet_t);
798extern kern_packet_idx_t kern_packet_get_object_index(const kern_packet_t);
799extern uint32_t kern_packet_get_data_length(const kern_packet_t);
800extern uint32_t kern_packet_get_buflet_count(const kern_packet_t);
801extern errno_t kern_packet_set_buflet_count(const kern_packet_t, uint32_t);
802extern kern_buflet_t kern_packet_get_next_buflet(const kern_packet_t,
803 const kern_buflet_t);
804extern errno_t kern_packet_finalize(const kern_packet_t);
805extern errno_t kern_packet_clone(const kern_packet_t, kern_packet_t *,
806 kern_packet_copy_mode_t);
807extern errno_t kern_packet_clone_nosleep(const kern_packet_t, kern_packet_t *,
808 kern_packet_copy_mode_t);
809extern errno_t kern_packet_add_buflet(const kern_packet_t ph,
810 const kern_buflet_t bprev, const kern_buflet_t bnew);
811extern void kern_packet_append(const kern_packet_t, const kern_packet_t);
812extern kern_packet_t kern_packet_get_next(const kern_packet_t);
813extern void kern_packet_set_next(const kern_packet_t, const kern_packet_t);
814extern void kern_packet_set_chain_counts(const kern_packet_t, uint32_t,
815 uint32_t);
816extern void kern_packet_get_chain_counts(const kern_packet_t, uint32_t *,
817 uint32_t *);
818
819/*
820 * Misc.
821 */
822extern uint32_t kern_inet_checksum(const void *, uint32_t, uint32_t);
823extern uint32_t kern_copy_and_inet_checksum(const void *, void *,
824 uint32_t, uint32_t);
825extern void kern_packet_set_trace_id(const kern_packet_t, packet_trace_id_t);
826extern packet_trace_id_t kern_packet_get_trace_id(const kern_packet_t);
827extern void kern_packet_trace_event(const kern_packet_t, uint32_t);
828extern errno_t kern_packet_copy_bytes(const kern_packet_t, size_t, size_t,
829 void*);
830
831/*
832 * Buflets.
833 */
834extern errno_t kern_buflet_set_data_address(const kern_buflet_t, const void *);
835extern void *kern_buflet_get_data_address(const kern_buflet_t);
836extern errno_t kern_buflet_set_data_offset(const kern_buflet_t, const uint32_t);
837extern uint32_t kern_buflet_get_data_offset(const kern_buflet_t);
838extern errno_t kern_buflet_set_data_length(const kern_buflet_t, const uint32_t);
839extern uint32_t kern_buflet_get_data_length(const kern_buflet_t);
840extern void *kern_buflet_get_object_address(const kern_buflet_t);
841extern uint32_t kern_buflet_get_object_limit(const kern_buflet_t);
842extern kern_segment_t kern_buflet_get_object_segment(const kern_buflet_t,
843 kern_obj_idx_seg_t *);
844extern errno_t kern_buflet_set_data_limit(const kern_buflet_t, const uint32_t);
845extern uint32_t kern_buflet_get_data_limit(const kern_buflet_t);
846extern errno_t kern_buflet_set_buffer_offset(const kern_buflet_t, const uint32_t);
847
848/*
849 * Packet buffer pool.
850 */
851typedef void (*alloc_cb_func_t)(kern_packet_t packet, uint32_t pkt_index,
852 const void *ctx);
853extern errno_t kern_pbufpool_create(const struct kern_pbufpool_init *,
854 kern_pbufpool_t *, struct kern_pbufpool_memory_info *);
855extern void *kern_pbufpool_get_context(const kern_pbufpool_t pbufpool);
856extern errno_t kern_pbufpool_get_memory_info(const kern_pbufpool_t pbufpool,
857 struct kern_pbufpool_memory_info *pbufpool_mem_ref);
858extern errno_t kern_pbufpool_alloc(const kern_pbufpool_t pbufpool,
859 const uint32_t bufcnt, kern_packet_t *packet);
860extern errno_t kern_pbufpool_alloc_batch(const kern_pbufpool_t pbufpool,
861 const uint32_t bufcnt, kern_packet_t *array, uint32_t *size);
862extern errno_t kern_pbufpool_alloc_batch_callback(
863 const kern_pbufpool_t pbufpool, const uint32_t bufcnt, kern_packet_t *array,
864 uint32_t *size, alloc_cb_func_t cb, const void *ctx);
865extern errno_t kern_pbufpool_alloc_nosleep(const kern_pbufpool_t pbufpool,
866 const uint32_t bufcnt, kern_packet_t *packet);
867extern errno_t kern_pbufpool_alloc_batch_nosleep(const kern_pbufpool_t pbufpool,
868 const uint32_t bufcnt, kern_packet_t *array, uint32_t *size);
869extern errno_t kern_pbufpool_alloc_batch_nosleep_callback(
870 const kern_pbufpool_t pbufpool, const uint32_t bufcnt,
871 kern_packet_t *array, uint32_t *size, alloc_cb_func_t cb, const void *ctx);
872extern void kern_pbufpool_free(const kern_pbufpool_t pbufpool, kern_packet_t);
873extern void kern_pbufpool_free_batch(const kern_pbufpool_t pbufpool,
874 kern_packet_t *array, uint32_t size);
875extern void kern_pbufpool_free_chain(const kern_pbufpool_t pbufpool,
876 kern_packet_t chain);
877extern errno_t kern_pbufpool_alloc_buffer(const kern_pbufpool_t pbufpool,
878 mach_vm_address_t *buffer, kern_segment_t *sg, kern_obj_idx_seg_t *sg_idx);
879extern errno_t kern_pbufpool_alloc_buffer_nosleep(const kern_pbufpool_t
880 pbufpool, mach_vm_address_t *buffer, kern_segment_t *sg,
881 kern_obj_idx_seg_t *sg_idx);
882extern void kern_pbufpool_free_buffer(const kern_pbufpool_t pbufpool,
883 mach_vm_address_t baddr);
884extern errno_t kern_pbufpool_alloc_buflet(const kern_pbufpool_t,
885 kern_buflet_t *);
886extern errno_t kern_pbufpool_alloc_buflet_nosleep(const kern_pbufpool_t,
887 kern_buflet_t *);
888extern void kern_pbufpool_destroy(kern_pbufpool_t);
889extern kern_segment_idx_t kern_segment_get_index(const kern_segment_t);
890__END_DECLS
891#endif /* KERNEL */
892#endif /* PRIVATE */
893#endif /* !_SKYWALK_OS_PACKET_H_ */
894