1/*
2 * Copyright (c) 2016-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#ifndef _SKYWALK_PACKET_PBUFPOOLVAR_H_
30#define _SKYWALK_PACKET_PBUFPOOLVAR_H_
31
32#ifdef BSD_KERNEL_PRIVATE
33#include <skywalk/core/skywalk_var.h>
34
35struct __kern_quantum;
36struct __kern_packet;
37
38/*
39 * User packet pool hash bucket. Packets allocated by user space are
40 * kept in the hash table. This allows the kernel to validate whether
41 * or not a given packet object is valid or is already-freed, and thus
42 * take the appropriate measure during internalize.
43 */
44struct kern_pbufpool_u_bkt {
45 SLIST_HEAD(, __kern_quantum) upp_head;
46};
47
48struct kern_pbufpool_u_bft_bkt {
49 SLIST_HEAD(, __kern_buflet_ext) upp_head;
50};
51
52#define PBUFPOOL_MAX_BUF_REGIONS 2
53#define PBUFPOOL_BUF_IDX_DEF 0
54#define PBUFPOOL_BUF_IDX_LARGE 1
55
56struct kern_pbufpool {
57 decl_lck_mtx_data(, pp_lock);
58 uint32_t pp_refcnt;
59 uint32_t pp_flags;
60 uint32_t pp_buf_obj_size[PBUFPOOL_MAX_BUF_REGIONS];
61 uint32_t pp_buf_size[PBUFPOOL_MAX_BUF_REGIONS];
62 uint16_t pp_max_frags;
63
64 /*
65 * Caches
66 */
67 struct skmem_cache *pp_buf_cache[PBUFPOOL_MAX_BUF_REGIONS];
68 struct skmem_cache *pp_kmd_cache;
69 struct skmem_cache *pp_kbft_cache[PBUFPOOL_MAX_BUF_REGIONS];
70
71 /*
72 * Regions
73 */
74 struct skmem_region *pp_buf_region[PBUFPOOL_MAX_BUF_REGIONS];
75 struct skmem_region *pp_kmd_region;
76 struct skmem_region *pp_umd_region;
77 struct skmem_region *pp_ubft_region;
78 struct skmem_region *pp_kbft_region;
79
80 /*
81 * User packet pool: packet metadata hash table
82 */
83 struct kern_pbufpool_u_bkt *pp_u_hash_table;
84 uint64_t pp_u_bufinuse;
85
86 /*
87 * User packet pool: buflet hash table
88 */
89 struct kern_pbufpool_u_bft_bkt *pp_u_bft_hash_table;
90 uint64_t pp_u_bftinuse;
91
92 void *pp_ctx;
93 pbuf_ctx_retain_fn_t pp_ctx_retain;
94 pbuf_ctx_release_fn_t pp_ctx_release;
95 nexus_meta_type_t pp_md_type;
96 nexus_meta_subtype_t pp_md_subtype;
97 uint32_t pp_midx_start;
98 uint32_t pp_bidx_start;
99 pbufpool_name_t pp_name;
100 pbuf_seg_ctor_fn_t pp_pbuf_seg_ctor;
101 pbuf_seg_dtor_fn_t pp_pbuf_seg_dtor;
102};
103
104/* valid values for pp_flags */
105#define PPF_EXTERNAL 0x1 /* externally configured */
106#define PPF_CLOSED 0x2 /* closed; awaiting final destruction */
107#define PPF_MONOLITHIC 0x4 /* non slab-based buffer region */
108/* buflet is truncated and may not contain the full payload */
109#define PPF_TRUNCATED_BUF 0x8
110#define PPF_KERNEL 0x10 /* kernel only, no user region(s) */
111#define PPF_BUFFER_ON_DEMAND 0x20 /* attach buffers to packet on demand */
112#define PPF_BATCH 0x40 /* capable of batch alloc/free */
113#define PPF_DYNAMIC 0x80 /* capable of magazine resizing */
114#define PPF_LARGE_BUF 0x100 /* configured with large buffers */
115
116#define PP_KERNEL_ONLY(_pp) \
117 (((_pp)->pp_flags & PPF_KERNEL) != 0)
118
119#define PP_HAS_TRUNCATED_BUF(_pp) \
120 (((_pp)->pp_flags & PPF_TRUNCATED_BUF) != 0)
121
122#define PP_HAS_BUFFER_ON_DEMAND(_pp) \
123 (((_pp)->pp_flags & PPF_BUFFER_ON_DEMAND) != 0)
124
125#define PP_BATCH_CAPABLE(_pp) \
126 (((_pp)->pp_flags & PPF_BATCH) != 0)
127
128#define PP_DYNAMIC(_pp) \
129 (((_pp)->pp_flags & PPF_DYNAMIC) != 0)
130
131#define PP_HAS_LARGE_BUF(_pp) \
132 (((_pp)->pp_flags & PPF_LARGE_BUF) != 0)
133
134#define PP_LOCK(_pp) \
135 lck_mtx_lock(&_pp->pp_lock)
136#define PP_LOCK_ASSERT_HELD(_pp) \
137 LCK_MTX_ASSERT(&_pp->pp_lock, LCK_MTX_ASSERT_OWNED)
138#define PP_LOCK_ASSERT_NOTHELD(_pp) \
139 LCK_MTX_ASSERT(&_pp->pp_lock, LCK_MTX_ASSERT_NOTOWNED)
140#define PP_UNLOCK(_pp) \
141 lck_mtx_unlock(&_pp->pp_lock)
142
143#define PP_BUF_SIZE_DEF(_pp) ((_pp)->pp_buf_size[PBUFPOOL_BUF_IDX_DEF])
144#define PP_BUF_SIZE_LARGE(_pp) ((_pp)->pp_buf_size[PBUFPOOL_BUF_IDX_LARGE])
145
146#define PP_BUF_OBJ_SIZE_DEF(_pp) \
147 ((_pp)->pp_buf_obj_size[PBUFPOOL_BUF_IDX_DEF])
148#define PP_BUF_OBJ_SIZE_LARGE(_pp) \
149 ((_pp)->pp_buf_obj_size[PBUFPOOL_BUF_IDX_LARGE])
150
151#define PP_BUF_REGION_DEF(_pp) ((_pp)->pp_buf_region[PBUFPOOL_BUF_IDX_DEF])
152#define PP_BUF_REGION_LARGE(_pp) ((_pp)->pp_buf_region[PBUFPOOL_BUF_IDX_LARGE])
153
154#define PP_BUF_CACHE_DEF(_pp) ((_pp)->pp_buf_cache[PBUFPOOL_BUF_IDX_DEF])
155#define PP_BUF_CACHE_LARGE(_pp) ((_pp)->pp_buf_cache[PBUFPOOL_BUF_IDX_LARGE])
156
157#define PP_KBFT_CACHE_DEF(_pp) ((_pp)->pp_kbft_cache[PBUFPOOL_BUF_IDX_DEF])
158#define PP_KBFT_CACHE_LARGE(_pp) ((_pp)->pp_kbft_cache[PBUFPOOL_BUF_IDX_LARGE])
159
160__BEGIN_DECLS
161extern int pp_init(void);
162extern void pp_fini(void);
163extern void pp_close(struct kern_pbufpool *);
164
165/* create flags for pp_create() */
166#define PPCREATEF_EXTERNAL 0x1 /* externally requested */
167#define PPCREATEF_KERNEL_ONLY 0x2 /* kernel-only */
168#define PPCREATEF_TRUNCATED_BUF 0x4 /* compat-only (buf is short) */
169#define PPCREATEF_ONDEMAND_BUF 0x8 /* buf alloc/free is decoupled */
170#define PPCREATEF_DYNAMIC 0x10 /* dynamic per-CPU magazines */
171
172extern struct kern_pbufpool *pp_create(const char *name,
173 struct skmem_region_params *srp_array, pbuf_seg_ctor_fn_t buf_seg_ctor,
174 pbuf_seg_dtor_fn_t buf_seg_dtor, const void *ctx,
175 pbuf_ctx_retain_fn_t ctx_retain, pbuf_ctx_release_fn_t ctx_release,
176 uint32_t ppcreatef);
177extern void pp_destroy(struct kern_pbufpool *);
178
179extern int pp_init_upp(struct kern_pbufpool *, boolean_t);
180extern void pp_insert_upp(struct kern_pbufpool *, struct __kern_quantum *,
181 pid_t);
182extern void pp_insert_upp_locked(struct kern_pbufpool *,
183 struct __kern_quantum *, pid_t);
184extern void pp_insert_upp_batch(struct kern_pbufpool *pp, pid_t pid,
185 uint64_t *array, uint32_t num);
186extern struct __kern_quantum *pp_remove_upp(struct kern_pbufpool *, obj_idx_t,
187 int *);
188extern struct __kern_quantum *pp_remove_upp_locked(struct kern_pbufpool *,
189 obj_idx_t, int *);
190extern struct __kern_quantum *pp_find_upp(struct kern_pbufpool *, obj_idx_t);
191extern void pp_purge_upp(struct kern_pbufpool *, pid_t);
192extern struct __kern_buflet *pp_remove_upp_bft(struct kern_pbufpool *,
193 obj_idx_t, int *);
194extern void pp_insert_upp_bft(struct kern_pbufpool *, struct __kern_buflet *,
195 pid_t);
196extern boolean_t pp_isempty_upp(struct kern_pbufpool *);
197
198extern void pp_retain_locked(struct kern_pbufpool *);
199extern void pp_retain(struct kern_pbufpool *);
200extern boolean_t pp_release_locked(struct kern_pbufpool *);
201extern boolean_t pp_release(struct kern_pbufpool *);
202
203/* flags for pp_regions_params_adjust() */
204/* configure packet pool regions for RX only */
205#define PP_REGION_CONFIG_BUF_IODIR_IN 0x00000001
206/* configure packet pool regions for TX only */
207#define PP_REGION_CONFIG_BUF_IODIR_OUT 0x00000002
208/* configure packet pool regions for bidirectional operation */
209#define PP_REGION_CONFIG_BUF_IODIR_BIDIR \
210 (PP_REGION_CONFIG_BUF_IODIR_IN | PP_REGION_CONFIG_BUF_IODIR_OUT)
211/* configure packet pool metadata regions as persistent (wired) */
212#define PP_REGION_CONFIG_MD_PERSISTENT 0x00000004
213/* configure packet pool buffer regions as persistent (wired) */
214#define PP_REGION_CONFIG_BUF_PERSISTENT 0x00000008
215/* Enable magazine layer (per-cpu caches) for packet pool metadata regions */
216#define PP_REGION_CONFIG_MD_MAGAZINE_ENABLE 0x00000010
217/* configure packet pool regions required for kernel-only operations */
218#define PP_REGION_CONFIG_KERNEL_ONLY 0x00000020
219/* configure packet pool buflet regions */
220#define PP_REGION_CONFIG_BUFLET 0x00000040
221/* configure packet pool buffer region as user read-only */
222#define PP_REGION_CONFIG_BUF_UREADONLY 0x00000080
223/* configure packet pool buffer region as kernel read-only */
224#define PP_REGION_CONFIG_BUF_KREADONLY 0x00000100
225/* configure packet pool buffer region as a single segment */
226#define PP_REGION_CONFIG_BUF_MONOLITHIC 0x00000200
227/* configure packet pool buffer region as physically contiguous segment */
228#define PP_REGION_CONFIG_BUF_SEGPHYSCONTIG 0x00000400
229/* configure packet pool buffer region as cache-inhibiting */
230#define PP_REGION_CONFIG_BUF_NOCACHE 0x00000800
231/* configure packet pool buffer region (backing IOMD) as thread safe */
232#define PP_REGION_CONFIG_BUF_THREADSAFE 0x00002000
233
234extern void pp_regions_params_adjust(struct skmem_region_params *,
235 nexus_meta_type_t, nexus_meta_subtype_t, uint32_t, uint16_t, uint32_t,
236 uint32_t, uint32_t, uint32_t, uint32_t);
237
238extern uint64_t pp_alloc_packet(struct kern_pbufpool *, uint16_t, uint32_t);
239extern uint64_t pp_alloc_packet_by_size(struct kern_pbufpool *, uint32_t,
240 uint32_t);
241extern int pp_alloc_packet_batch(struct kern_pbufpool *, uint16_t, uint64_t *,
242 uint32_t *, boolean_t, alloc_cb_func_t, const void *, uint32_t);
243extern int pp_alloc_pktq(struct kern_pbufpool *, uint16_t, struct pktq *,
244 uint32_t, alloc_cb_func_t, const void *, uint32_t);
245extern void pp_free_packet(struct kern_pbufpool *, uint64_t);
246extern void pp_free_packet_batch(struct kern_pbufpool *, uint64_t *, uint32_t);
247extern void pp_free_packet_single(struct __kern_packet *);
248extern void pp_free_packet_chain(struct __kern_packet *, int *);
249extern void pp_free_pktq(struct pktq *);
250extern errno_t pp_alloc_buffer(const kern_pbufpool_t, mach_vm_address_t *,
251 kern_segment_t *, kern_obj_idx_seg_t *, uint32_t);
252extern void pp_free_buffer(const kern_pbufpool_t, mach_vm_address_t);
253extern errno_t pp_alloc_buflet(struct kern_pbufpool *pp, kern_buflet_t *kbft,
254 uint32_t skmflag, bool large);
255extern errno_t pp_alloc_buflet_batch(struct kern_pbufpool *pp, uint64_t *array,
256 uint32_t *size, uint32_t skmflag, bool large);
257extern void pp_free_buflet(const kern_pbufpool_t, kern_buflet_t);
258extern void pp_reap_caches(boolean_t);
259__END_DECLS
260#endif /* BSD_KERNEL_PRIVATE */
261#endif /* !_SKYWALK_PACKET_PBUFPOOLVAR_H_ */
262