1/*
2 * Copyright (c) 2015-2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#ifndef _SKYWALK_OS_CHANNEL_H_
30#define _SKYWALK_OS_CHANNEL_H_
31
32#ifdef PRIVATE
33
34#include <stdint.h>
35#include <sys/types.h>
36#include <sys/cdefs.h>
37#include <uuid/uuid.h>
38#include <mach/vm_types.h>
39#include <skywalk/os_nexus.h>
40#include <skywalk/os_packet.h>
41#ifndef KERNEL
42#include <skywalk/os_channel_event.h>
43#include <net/if_var.h>
44#endif /* !KERNEL */
45
46/*
47 * Compiler guards used by Libnetcore.
48 */
49#define OS_CHANNEL_HAS_NUM_BUFFERS_ATTR 1 /* CHANNEL_ATTR_NUM_BUFFERS */
50#define OS_CHANNEL_HAS_LARGE_PACKET 1 /* CHANNEL_ATTR_LARGE_BUF_SIZE and */
51 /* os_channel_large_packet_alloc() */
52
53/* Flow advisory table index */
54typedef uint32_t flowadv_idx_t;
55#define FLOWADV_IDX_NONE ((flowadv_idx_t)-1)
56
57/*
58 * Channel ring direction.
59 */
60typedef enum {
61 CHANNEL_DIR_TX_RX, /* default: TX and RX ring(s) */
62 CHANNEL_DIR_TX, /* (monitor) only TX ring(s) */
63 CHANNEL_DIR_RX /* (monitor) only RX ring(s) */
64} ring_dir_t;
65
66/*
67 * Channel ring ID.
68 */
69typedef uint32_t ring_id_t;
70#define CHANNEL_RING_ID_ANY ((ring_id_t)-1)
71
72typedef enum {
73 CHANNEL_FIRST_TX_RING,
74 CHANNEL_LAST_TX_RING,
75 CHANNEL_FIRST_RX_RING,
76 CHANNEL_LAST_RX_RING
77} ring_id_type_t;
78
79/* Sync mode values */
80typedef enum {
81 CHANNEL_SYNC_TX, /* synchronize TX ring(s) */
82 CHANNEL_SYNC_RX, /* synchronize RX ring(s) */
83#if defined(LIBSYSCALL_INTERFACE) || defined(BSD_KERNEL_PRIVATE)
84 CHANNEL_SYNC_UPP /* synchronize packet pool rings only */
85#endif /* LIBSYSCALL_INTERFACE || BSD_KERNEL_PRIVATE */
86} sync_mode_t;
87
88/* Sync flags */
89typedef uint32_t sync_flags_t;
90#if defined(LIBSYSCALL_INTERFACE) || defined(BSD_KERNEL_PRIVATE)
91#define CHANNEL_SYNCF_ALLOC 0x1 /* synchronize alloc ring */
92#define CHANNEL_SYNCF_FREE 0x2 /* synchronize free ring */
93#define CHANNEL_SYNCF_PURGE 0x4 /* purge user packet pool */
94#define CHANNEL_SYNCF_ALLOC_BUF 0x8 /* synchronize buflet alloc ring */
95#define CHANNEL_SYNCF_LARGE_ALLOC 0x10 /* synchronize large alloc ring */
96#endif /* LIBSYSCALL_INTERFACE || BSD_KERNEL_PRIVATE */
97
98/*
99 * Opaque handles.
100 */
101struct channel;
102struct channel_ring_desc;
103struct __slot_desc;
104struct channel_attr;
105
106typedef struct channel *channel_t;
107typedef struct channel_ring_desc *channel_ring_t;
108typedef struct __slot_desc *channel_slot_t;
109typedef struct channel_attr *channel_attr_t;
110
111/*
112 * Channel monitor types.
113 */
114typedef enum {
115 CHANNEL_MONITOR_OFF, /* default */
116 CHANNEL_MONITOR_NO_COPY, /* zero-copy (delayed) mode */
117 CHANNEL_MONITOR_COPY /* copy (immediate) mode */
118} channel_monitor_type_t;
119
120/*
121 * Channel threshold unit types.
122 */
123typedef enum {
124 CHANNEL_THRESHOLD_UNIT_SLOTS, /* unit in slots (default) */
125 CHANNEL_THRESHOLD_UNIT_BYTES, /* unit in bytes */
126} channel_threshold_unit_t;
127
128/*
129 * Channel attribute types gettable/settable via os_channel_attr_{get,set}.
130 *
131 * g: retrievable at any time
132 * s: settable at any time
133 * S: settable once, only at creation time
134 */
135typedef enum {
136 CHANNEL_ATTR_TX_RINGS, /* (g) # of transmit rings */
137 CHANNEL_ATTR_RX_RINGS, /* (g) # of receive rings */
138 CHANNEL_ATTR_TX_SLOTS, /* (g) # of slots per transmit ring */
139 CHANNEL_ATTR_RX_SLOTS, /* (g) # of slots per receive ring */
140 CHANNEL_ATTR_SLOT_BUF_SIZE, /* (g) buffer per slot (bytes) */
141 CHANNEL_ATTR_SLOT_META_SIZE, /* (g) metadata per slot (bytes) */
142 CHANNEL_ATTR_EXCLUSIVE, /* (g/s) bool: exclusive open */
143 CHANNEL_ATTR_NO_AUTO_SYNC, /* (g/s) bool: will do explicit sync */
144 CHANNEL_ATTR_MONITOR, /* (g/s) see channel_monitor_type_t */
145 CHANNEL_ATTR_TX_LOWAT_UNIT, /* (g/s) see channel_threshold_unit_t */
146 CHANNEL_ATTR_TX_LOWAT_VALUE, /* (g/s) transmit low-watermark */
147 CHANNEL_ATTR_RX_LOWAT_UNIT, /* (g/s) see channel_threshold_unit_t */
148 CHANNEL_ATTR_RX_LOWAT_VALUE, /* (g/s) receive low-watermark */
149 CHANNEL_ATTR_NEXUS_TYPE, /* (g) nexus type */
150 CHANNEL_ATTR_NEXUS_EXTENSIONS, /* (g) nexus extension(s) */
151 CHANNEL_ATTR_NEXUS_MHINTS, /* (g) nexus memory usage hints */
152 CHANNEL_ATTR_TX_HOST_RINGS, /* (g) # of transmit host rings */
153 CHANNEL_ATTR_RX_HOST_RINGS, /* (g) # of receive host rings */
154 CHANNEL_ATTR_NEXUS_IFINDEX, /* (g) nexus network interface index */
155 CHANNEL_ATTR_NEXUS_STATS_SIZE, /* (g) nexus statistics region size */
156 CHANNEL_ATTR_NEXUS_FLOWADV_MAX, /* (g) # of flow advisory entries */
157 CHANNEL_ATTR_NEXUS_META_TYPE, /* (g) nexus metadata type */
158 CHANNEL_ATTR_NEXUS_META_SUBTYPE, /* (g) nexus metadata subtype */
159 CHANNEL_ATTR_NEXUS_CHECKSUM_OFFLOAD, /* (g) nexus checksum offload */
160 CHANNEL_ATTR_USER_PACKET_POOL, /* (g/s) bool: use user packet pool */
161 CHANNEL_ATTR_NEXUS_ADV_SIZE, /* (g) nexus advisory region size */
162 CHANNEL_ATTR_NEXUS_DEFUNCT_OK, /* (g/s) bool: allow defunct */
163 CHANNEL_ATTR_FILTER, /* (g/s) bool: filter mode */
164 CHANNEL_ATTR_EVENT_RING, /* (g/s) bool: enable event ring */
165 CHANNEL_ATTR_MAX_FRAGS, /* (g) max length of buflet chain */
166 CHANNEL_ATTR_NUM_BUFFERS, /* (g) # of buffers in user pool */
167 CHANNEL_ATTR_LOW_LATENCY, /* (g/s) bool: low latency channel */
168 CHANNEL_ATTR_LARGE_BUF_SIZE, /* (g) large buffer size (bytes) */
169} channel_attr_type_t;
170
171/*
172 * Channel nexus metadata type.
173 */
174typedef enum {
175 CHANNEL_NEXUS_META_TYPE_INVALID = 0,
176 CHANNEL_NEXUS_META_TYPE_QUANTUM, /* OK for os_packet quantum APIs */
177 CHANNEL_NEXUS_META_TYPE_PACKET, /* OK for all os_packet APIs */
178} channel_nexus_meta_type_t;
179
180/*
181 * Channel nexus metadata subtype.
182 */
183typedef enum {
184 CHANNEL_NEXUS_META_SUBTYPE_INVALID = 0,
185 CHANNEL_NEXUS_META_SUBTYPE_PAYLOAD,
186 CHANNEL_NEXUS_META_SUBTYPE_RAW,
187} channel_nexus_meta_subtype_t;
188
189/*
190 * Valid values for CHANNEL_ATTR_NEXUS_CHECKSUM_OFFLOAD
191 */
192#define CHANNEL_NEXUS_CHECKSUM_PARTIAL 0x1 /* partial checksum */
193
194/*
195 * Channel statistics ID.
196 */
197typedef enum {
198 CHANNEL_STATS_ID_IP = 0, /* struct ip_stats */
199 CHANNEL_STATS_ID_IP6, /* struct ip6_stats */
200 CHANNEL_STATS_ID_TCP, /* struct tcp_stats */
201 CHANNEL_STATS_ID_UDP, /* struct udp_stats */
202 CHANNEL_STATS_ID_QUIC, /* struct quic_stats */
203} channel_stats_id_t;
204
205/*
206 * Slot properties. Structure is aligned to allow for efficient copy.
207 *
208 * Fields except for sp_{flags,len} are immutables (I). The system will
209 * verify for correctness during os_channel_put() across the immutable
210 * fields, and will abort the process if it detects inconsistencies.
211 * This is meant to help with debugging, since it indicates bugs and/or
212 * memory corruption.
213 */
214typedef struct slot_prop {
215 uint16_t sp_flags; /* private flags */
216 uint16_t sp_len; /* length for this slot */
217 uint32_t sp_idx; /* (I) slot index */
218 mach_vm_address_t sp_ext_ptr; /* (I) pointer for indirect buffer */
219 mach_vm_address_t sp_buf_ptr; /* (I) pointer for native buffer */
220 mach_vm_address_t sp_mdata_ptr; /* (I) pointer for metadata */
221 uint32_t _sp_pad[8]; /* reserved */
222} slot_prop_t __attribute__((aligned(sizeof(uint64_t))));
223
224#ifndef KERNEL
225/*
226 * User APIs.
227 */
228#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
229__BEGIN_DECLS
230/*
231 * Creates a Channel attribute object.
232 *
233 * This must be paired with a os_channel_attr_destroy() on the handle.
234 */
235extern channel_attr_t os_channel_attr_create(void);
236
237/*
238 * Clones a Channel attribute object. If source attribute is NULL
239 * it behaves just like os_channel_attr_create();
240 *
241 * This must be paired with a os_channel_attr_destroy() on the handle.
242 */
243extern channel_attr_t os_channel_attr_clone(const channel_attr_t);
244
245/*
246 * Sets a value for a given attribute type on a Channel attribute object.
247 */
248extern int os_channel_attr_set(const channel_attr_t attr,
249 const channel_attr_type_t type, const uint64_t value);
250
251/*
252 * Sets a key blob on a Channel attribute object. Existing key blob
253 * information in the attribute object will be removed, if any, and
254 * replaced with the new key blob. Specifying 0 for key_len will
255 * clear the key stored in the attribute object. The maximum key
256 * length is specified by NEXUS_MAX_KEY_LEN.
257 */
258extern int os_channel_attr_set_key(const channel_attr_t attr,
259 const void *key, const uint32_t key_len);
260
261/*
262 * Gets a value for a given attribute type on a Channel attribute object.
263 */
264extern int os_channel_attr_get(const channel_attr_t attr,
265 const channel_attr_type_t type, uint64_t *value);
266
267/*
268 * Gets a key blob on a Channel attribute object. If key is NULL,
269 * returns the length of the key blob with key_len, so caller knows
270 * how much to allocate space for key blob.
271 */
272extern int os_channel_attr_get_key(const channel_attr_t attr,
273 void *key, uint32_t *key_len);
274
275/*
276 * Destroys a Channel attribute object, along with all resources
277 * associated with it (e.g. key blob).
278 */
279extern void os_channel_attr_destroy(const channel_attr_t attr);
280
281/*
282 * Opens a Channel to a Nexus provider instance. Upon success, maps memory
283 * region and allocates resources.
284 *
285 * This must be paired with a os_channel_destroy() on the handle, in order to
286 * unmap the memory region and free resources.
287 */
288extern channel_t os_channel_create(const uuid_t uuid, const nexus_port_t port);
289
290/*
291 * Extended version of os_channel_create().
292 */
293extern channel_t os_channel_create_extended(const uuid_t uuid,
294 const nexus_port_t port, const ring_dir_t dir, const ring_id_t rid,
295 const channel_attr_t attr);
296
297/*
298 * Retrieves the file descriptor associated with the Channel.
299 */
300extern int os_channel_get_fd(const channel_t channel);
301
302/*
303 * Retrieves current channel attributes into the channel_attr_t handle.
304 */
305extern int os_channel_read_attr(const channel_t channel, channel_attr_t attr);
306
307/*
308 * Updates channel attributes based on those referred to by the channel_attr_t
309 * handle. See comments above on channel_attr_type_t; this routine will only
310 * update attributes that are marked with 's' but not 'S'.
311 */
312extern int os_channel_write_attr(const channel_t channel, channel_attr_t attr);
313
314/*
315 * Retrieves channel's associated nexus type into *nexus_type, and the
316 * provider-specific extension attribute into *ext.
317 */
318extern int os_channel_read_nexus_extension_info(const channel_t channel,
319 nexus_type_t *nexus_type, uint64_t *ext);
320
321/*
322 * Non-blocking synchronization. Channel handle may also be used
323 * with kqueue(2), select(2) or poll(2) through the file descriptor.
324 */
325extern int os_channel_sync(const channel_t channel, const sync_mode_t mode);
326
327/*
328 * Destroys a Channel.
329 */
330extern void os_channel_destroy(const channel_t channel);
331
332/*
333 * Checks if a channel is defunct. Returns non-zero if defunct.
334 */
335extern int os_channel_is_defunct(const channel_t channel);
336
337/*
338 * Data Movement APIs.
339 *
340 * Obtain channel_ring_t handle via os_channel_{tx,rx}_ring(). You will
341 * need to specify the ring_id_t which identifies the ring — this is true
342 * even for a single TX/RX ring case. The Nexus provider can communicate
343 * to the client the ID of the TX and RX ring that should be used to
344 * communicate to it, through a contract between the two. For instance,
345 * it can tell the client to use first TX ring and first RX ring, etc.
346 * through some side-channel. It should not assume 0 or any other number
347 * as ID, however, as the in-kernel Nexus object is the authoritative source
348 * of truth. This is where the os_channel_ring_id() call comes into the
349 * picture, as it will return the first and last usable TX and RX ring IDs
350 * for the Channel opened to that Nexus object.
351 *
352 * Once the TX or RX ring handle is obtained above, the client can ask for
353 * the first usable slot in the ring through os_channel_get_next_slot()
354 * passing NULL for the 'slot' parameter. This returns a channel_slot_t
355 * handle that represents the slot, along with the properties of that slot
356 * described by the slot_prop_t structure. If no slots are available, this
357 * call returns a NULL handle. It’s important to note that this
358 * call does NOT advance the ring’s current slot pointer; calling this
359 * multiple times in succession will yield the same result.
360 *
361 * The client proceeds to use the slot by examining the returned
362 * slot_prop_t fields including the pointer to the internal buffer
363 * associated with that slot. Once the client is finished, it updates
364 * the relevant slot_prop_t fields (e.g. length) and calls
365 * os_channel_set_slot_properties() to apply them to the slot.
366 *
367 * To get the next slot, the client provides the non-NULL slot value obtained
368 * from the previous call to os_channel_get_next_slot() as the 'slot' parameter
369 * in its next invocation of that function.
370 *
371 * To advance the ring’s current pointer, the client invokes
372 * os_channel_advance_slot() specifying the slot to advance past. If the slot
373 * is invalid, this function returns a non-zero value.
374 *
375 * Once the client is ready to commit, call os_channel_sync() in
376 * either/all directions.
377 */
378extern ring_id_t os_channel_ring_id(const channel_t channel,
379 const ring_id_type_t type);
380extern channel_ring_t os_channel_tx_ring(const channel_t channel,
381 const ring_id_t rid);
382extern channel_ring_t os_channel_rx_ring(const channel_t channel,
383 const ring_id_t rid);
384extern int os_channel_pending(const channel_ring_t ring);
385
386/*
387 * This returns a nexus-specific timestamp in nanoseconds taken at the
388 * lasttime os_channel_sync() or its equivalent implicit kevent sync
389 * was called
390 */
391extern uint64_t os_channel_ring_sync_time(const channel_ring_t ring);
392
393/*
394 * This returns a nexus-specific timestamp in nanoseconds to indicate
395 * the time of last activity on the opposite end of the ring.
396 * This is only updated when sync or kevent equivalent is called.
397 */
398extern uint64_t os_channel_ring_notify_time(const channel_ring_t ring);
399
400/*
401 * For TX ring os_channel_available_slot_count() returns the minimum number
402 * of slots available availble for TX, and it is possible that
403 * os_channel_get_next_slot() will return more slots than the what was
404 * returned by an earlier call to os_channel_available_slot_count()
405 */
406extern uint32_t os_channel_available_slot_count(const channel_ring_t ring);
407extern channel_slot_t os_channel_get_next_slot(const channel_ring_t ring,
408 const channel_slot_t slot, slot_prop_t *prop);
409extern int os_channel_advance_slot(channel_ring_t ring,
410 const channel_slot_t slot);
411extern void os_channel_set_slot_properties(const channel_ring_t ring,
412 const channel_slot_t slot, const slot_prop_t *prop);
413
414/*
415 * Return the packet handle associated with a given slot of a ring.
416 */
417extern packet_t os_channel_slot_get_packet(const channel_ring_t ring,
418 const channel_slot_t slot);
419
420/*
421 * Each nexus that the channel is connected to determines whether or
422 * not there is a shareable statistics region identified by one of
423 * the channel_stats_id_t values. This routine returns a pointer to
424 * such a region upon success, or NULL if not supported by the nexus.
425 */
426extern void *os_channel_get_stats_region(const channel_t channel,
427 const channel_stats_id_t id);
428
429/*
430 * Each nexus that the channel is connected to determines whether or
431 * not there is a nexus-wide advisory region. This routine returns
432 * a pointer to such a region upon success, or NULL if not supported
433 * by the nexus.
434 */
435extern void *os_channel_get_advisory_region(const channel_t channel);
436
437/*
438 * Each nexus that supports flow admission control may be queried to
439 * advise whether or not the channel is willing to admit more packets
440 * for a given flow. A return value of 0 indicates that the packets
441 * for the flow are admissible. If ENOBUFS is returned, the flow is
442 * currently suspended, and further attempts to send more packets on
443 * the ring may result in drops. Any other error values indicate
444 * that either the nexus doesn't support admission control, or the
445 * arguments aren't valid.
446 */
447extern int os_channel_flow_admissible(const channel_ring_t ring,
448 uuid_t flow_id, const flowadv_idx_t flow_index);
449
450extern int os_channel_flow_adv_get_ce_count(const channel_ring_t chrd,
451 uuid_t flow_id, const flowadv_idx_t flow_index, uint32_t *ce_cnt,
452 uint32_t *pkt_cnt);
453/*
454 * Allocate a packet from the channel's packet pool.
455 * Returns 0 on success with the packet handle in packet arg.
456 * Note: os_channel_packet_alloc() & os_channel_packet_free() should be
457 * serialized and should not be called from the different thread context.
458 */
459extern int
460os_channel_packet_alloc(const channel_t chd, packet_t *packet);
461
462/*
463 * Allocate a large packet from the channel's packet pool.
464 * Returns 0 on success with the packet handle in packet arg.
465 * Note: os_channel_large_packet_alloc() & os_channel_packet_free() should be
466 * serialized and should not be called from the different thread context.
467 */
468extern int
469os_channel_large_packet_alloc(const channel_t chd, packet_t *packet);
470
471/*
472 * Free a packet allocated from the channel's packet pool.
473 * Returns 0 on success
474 * Note: os_channel_packet_alloc() & os_channel_packet_free() should be
475 * serialized and should not be called from the different thread context.
476 */
477extern int
478os_channel_packet_free(const channel_t chd, packet_t packet);
479
480/*
481 * Attach the given packet to a channel slot
482 */
483extern int
484os_channel_slot_attach_packet(const channel_ring_t chrd,
485 const channel_slot_t slot, packet_t packet);
486
487/*
488 * Detach a given packet from a channel slot
489 */
490extern int
491os_channel_slot_detach_packet(const channel_ring_t chrd,
492 const channel_slot_t slot, packet_t packet);
493
494/*
495 * purge packets from the channel's packet pool.
496 * This API should be called at regular intervals by application to purge
497 * unused packets from the channel's packet pool. Recommended interval is
498 * 11 seconds.
499 * Returns 0 on success.
500 * Note: This call should be serialized with os_channel_packet_alloc() &
501 * os_channel_packet_free() and should not be called from different
502 * thread context.
503 */
504extern int
505os_channel_packet_pool_purge(const channel_t chd);
506
507/*
508 * Retrieve handle to the next available event(s) on the channel.
509 * os_event_get_next_event() can then called on the event handle to
510 * retrieve the individual events from the handle.
511 * Returns 0 on success, ENXIO if the channel is defunct.
512 */
513extern int
514os_channel_get_next_event_handle(const channel_t chd,
515 os_channel_event_handle_t *ehandle, os_channel_event_type_t *etype,
516 uint32_t *nevents);
517
518/*
519 * Free an event retrieved from the channel.
520 * Returns 0 on success, ENXIO if the channel is defunct.
521 */
522extern int
523os_channel_event_free(const channel_t chd, os_channel_event_handle_t ehandle);
524
525/*
526 * API to retrieve the latest interface advisory report on the channel.
527 * Returns 0 on succcess. If the return value is EAGAIN, caller can attempt
528 * to retrieve the information again.
529 */
530extern int
531os_channel_get_interface_advisory(const channel_t chd,
532 struct ifnet_interface_advisory *advisory);
533
534/*
535 * API to configure interface advisory report on the channel.
536 * Returns 0 on succcess.
537 */
538extern int
539os_channel_configure_interface_advisory(const channel_t chd, boolean_t enable);
540
541extern int
542os_channel_buflet_alloc(const channel_t chd, buflet_t *bft);
543
544extern int
545os_channel_buflet_free(const channel_t chd, buflet_t ubft);
546__END_DECLS
547#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
548#else /* KERNEL */
549/*
550 * Kernel APIs.
551 */
552
553/*
554 * Opaque handles.
555 */
556struct kern_channel;
557struct __kern_channel_ring;
558
559typedef struct kern_channel *kern_channel_t;
560typedef struct __kern_channel_ring *kern_channel_ring_t;
561typedef struct __slot_desc *kern_channel_slot_t;
562
563/*
564 * Slot properties (deprecated).
565 */
566struct kern_slot_prop {
567 uint32_t _sp_pad[16]; /* reserved */
568} __attribute__((aligned(sizeof(uint64_t))));
569
570/*
571 * @struct kern_channel_ring_stat_increment
572 * @abstract Structure used to increment the per ring statistic counters.
573 * @field kcrsi_slots_transferred number of slots transferred
574 * @filed kcrsi_bytes_transferred number of bytes transferred
575 */
576struct kern_channel_ring_stat_increment {
577 uint32_t kcrsi_slots_transferred;
578 uint32_t kcrsi_bytes_transferred;
579};
580
581/*
582 * Data Movement APIs.
583 *
584 * See block comment above for userland data movement APIs for general
585 * concepts. The main differences here are the kern_channel_notify()
586 * and kern_channel_reclaim() calls that aren't available for userland.
587 * These calls are typically invoked within the TX and RX sync callbacks
588 * implemented by the nexus provider.
589 *
590 * For TX sync, kern_channel_reclaim() is normally called after the
591 * provider has finished reclaiming slots that have been "transmitted".
592 * In this case, this call is simply a way to indicate to the system
593 * that such condition has happened.
594 *
595 * For RX sync, kern_channel_reclaim() must be called at the beginning
596 * of the callback in order to reclaim user-released slots, and to
597 * ensure that subsequent calls to kern_channel_available_slot_count()
598 * or kern_channel_get_next_slot() operates on the most recent state.
599 *
600 * The kern_channel_notify() is used to post notifications to indicate
601 * slot availability; this may result in the kernel event subsystem
602 * posting readable and writable events.
603 */
604__BEGIN_DECLS
605extern uint32_t kern_channel_notify(const kern_channel_ring_t, uint32_t flags);
606extern uint32_t kern_channel_available_slot_count(
607 const kern_channel_ring_t ring);
608/*
609 * NOTE: kern_channel_set_slot_properties(), kern_channel_get_next_slot(),
610 * kern_channel_reclaim() and kern_channel_advance_slot() require that the
611 * caller invokes them from within the sync callback context; they will
612 * assert otherwise.
613 */
614extern void kern_channel_set_slot_properties(const kern_channel_ring_t,
615 const kern_channel_slot_t slot, const struct kern_slot_prop *prop);
616extern kern_channel_slot_t kern_channel_get_next_slot(
617 const kern_channel_ring_t kring, const kern_channel_slot_t slot,
618 struct kern_slot_prop *slot_prop);
619extern uint32_t kern_channel_reclaim(const kern_channel_ring_t);
620extern void kern_channel_advance_slot(const kern_channel_ring_t kring,
621 kern_channel_slot_t slot);
622
623/*
624 * Packet.
625 */
626extern kern_packet_t kern_channel_slot_get_packet(
627 const kern_channel_ring_t ring, const kern_channel_slot_t slot);
628
629/*
630 * NOTE: kern_channel_slot_attach_packet(), kern_channel_slot_detach_packet()
631 * and kern_channel_ring_get_container() require that the caller invokes them
632 * from within the sync callback context; they will assert otherwise.
633 */
634extern errno_t kern_channel_slot_attach_packet(const kern_channel_ring_t ring,
635 const kern_channel_slot_t slot, kern_packet_t packet);
636extern errno_t kern_channel_slot_detach_packet(const kern_channel_ring_t ring,
637 const kern_channel_slot_t slot, kern_packet_t packet);
638extern errno_t kern_channel_ring_get_container(const kern_channel_ring_t ring,
639 kern_packet_t **array, uint32_t *count);
640extern errno_t kern_channel_tx_refill(const kern_channel_ring_t ring,
641 uint32_t pkt_limit, uint32_t byte_limit, boolean_t tx_doorbell_ctxt,
642 boolean_t *pkts_pending);
643extern errno_t kern_channel_get_service_class(const kern_channel_ring_t ring,
644 kern_packet_svc_class_t *svc);
645extern errno_t kern_netif_queue_get_service_class(kern_netif_queue_t,
646 kern_packet_svc_class_t *);
647
648/*
649 * Misc.
650 */
651extern void *kern_channel_get_context(const kern_channel_t channel);
652extern void *kern_channel_ring_get_context(const kern_channel_ring_t ring);
653extern void *kern_channel_slot_get_context(const kern_channel_ring_t ring,
654 const kern_channel_slot_t slot);
655
656/*
657 * NOTE: kern_channel_increment_ring_{net}_stats() requires
658 * that the caller invokes it from within the sync callback context;
659 * it will assert otherwise.
660 */
661extern void kern_channel_increment_ring_stats(kern_channel_ring_t ring,
662 struct kern_channel_ring_stat_increment *stats);
663extern void kern_channel_increment_ring_net_stats(kern_channel_ring_t ring,
664 ifnet_t, struct kern_channel_ring_stat_increment *stats);
665
666#ifdef BSD_KERNEL_PRIVATE
667/* forward declare */
668struct flowadv_fcentry;
669
670/* Flow advisory token */
671typedef uint32_t flowadv_token_t;
672
673/*
674 * Private, unexported KPIs.
675 */
676__private_extern__ errno_t kern_channel_slot_attach_packet_byidx(
677 const kern_channel_ring_t kring, const uint32_t sidx, kern_packet_t ph);
678__private_extern__ errno_t kern_channel_slot_detach_packet_byidx(
679 const kern_channel_ring_t kring, const uint32_t sidx, kern_packet_t ph);
680__private_extern__ void kern_channel_flowadv_clear(struct flowadv_fcentry *);
681__private_extern__ void kern_channel_flowadv_report_ce_event(
682 struct flowadv_fcentry *, uint32_t, uint32_t);
683__private_extern__ void kern_channel_memstatus(struct proc *, uint32_t,
684 struct kern_channel *);
685__private_extern__ void kern_channel_defunct(struct proc *,
686 struct kern_channel *);
687__private_extern__ errno_t kern_channel_tx_refill_canblock(
688 const kern_channel_ring_t, uint32_t, uint32_t, boolean_t, boolean_t *);
689#endif /* BSD_KERNEL_PRIVATE */
690__END_DECLS
691#endif /* KERNEL */
692#endif /* PRIVATE */
693#endif /* !_SKYWALK_OS_CHANNEL_H_ */
694