1 | /* |
2 | * Copyright (c) 2015-2021 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | #ifndef _SKYWALK_OS_SKYWALK_PRIVATE_H |
29 | #define _SKYWALK_OS_SKYWALK_PRIVATE_H |
30 | |
31 | #if defined(PRIVATE) || defined(XNU_KERNEL_PRIVATE) |
32 | /* |
33 | * This file contains private interfaces for Skywalk, and should not |
34 | * be included by code external to xnu kernel or libsystem_kernel. |
35 | * The only exception to this is skywalk_cmds for the internal tools. |
36 | */ |
37 | |
38 | /* branch prediction helpers */ |
39 | #include <sys/cdefs.h> |
40 | #define SK_ALIGN64_CASSERT(type, field) \ |
41 | _CASSERT((__builtin_offsetof(type, field) % sizeof (uint64_t)) == 0) |
42 | |
43 | #if !defined(KERNEL) || defined(BSD_KERNEL_PRIVATE) |
44 | enum { |
45 | SK_FEATURE_SKYWALK = 1ULL << 0, |
46 | SK_FEATURE_DEVELOPMENT = 1ULL << 1, |
47 | SK_FEATURE_DEBUG = 1ULL << 2, |
48 | SK_FEATURE_NEXUS_FLOWSWITCH = 1ULL << 3, |
49 | SK_FEATURE_NEXUS_MONITOR = 1ULL << 4, |
50 | SK_FEATURE_NEXUS_NETIF = 1ULL << 5, |
51 | SK_FEATURE_NEXUS_USER_PIPE = 1ULL << 6, |
52 | SK_FEATURE_NEXUS_KERNEL_PIPE = 1ULL << 7, |
53 | SK_FEATURE_NEXUS_KERNEL_PIPE_LOOPBACK = 1ULL << 8, |
54 | SK_FEATURE_DEV_OR_DEBUG = 1ULL << 9, |
55 | SK_FEATURE_NETNS = 1ULL << 10, |
56 | SK_FEATURE_PROTONS = 1ULL << 11, |
57 | }; |
58 | #endif /* !KERNEL || BSD_KERNEL_PRIVATE */ |
59 | |
60 | /* valid flags for if_attach_nx */ |
61 | #define IF_ATTACH_NX_NETIF_COMPAT 0x01 /* create compat netif */ |
62 | #define IF_ATTACH_NX_FLOWSWITCH 0x02 /* enable flowswitch */ |
63 | #define IF_ATTACH_NX_FSW_TRANSPORT_NETAGENT 0x04 /* enable fsw TCP/UDP netagent */ |
64 | #define IF_ATTACH_NX_NETIF_NETAGENT 0x08 /* enable netif netagent */ |
65 | #define IF_ATTACH_NX_NETIF_ALL 0x10 /* don't restrict netif */ |
66 | #define IF_ATTACH_NX_FSW_IP_NETAGENT 0x20 /* enable fsw IP netagent */ |
67 | |
68 | /* |
69 | * Enabling Skywalk channel (user) networking stack fundamentally requires |
70 | * the presence of netif nexus attached to the interface. Native Skywalk |
71 | * drivers will come with a netif nexus; non-native drivers will require us |
72 | * to create a netif compat nexus, set through IF_ATTACH_NX_NETIF_COMPAT. |
73 | * |
74 | * The flowswitch nexus creation depends on the presence of netif nexus on |
75 | * the interface. Plumbing the flowswitch atop netif is required to connect |
76 | * the interface to the host (kernel) networking stack; this is the default |
77 | * behavior unless IF_ATTACH_NX_FLOWSWITCH is not set. |
78 | * |
79 | * To further allow for channel (user) networking stack, the netagent gets |
80 | * enabled on the flowswitch by default, unless opted out by clearing the |
81 | * IF_ATTACH_NX_FSW_TRANSPORT_NETAGENT flag. Note that |
82 | * IF_ATTACH_NX_FSW_TRANSPORT_NETAGENT cannot be changed after boot, and so has |
83 | * to be set by the if_attach_nx bootarg. |
84 | */ |
85 | #define SKYWALK_NETWORKING_ENABLED \ |
86 | (IF_ATTACH_NX_NETIF_COMPAT | IF_ATTACH_NX_FLOWSWITCH | \ |
87 | IF_ATTACH_NX_FSW_TRANSPORT_NETAGENT | IF_ATTACH_NX_FSW_IP_NETAGENT) |
88 | |
89 | /* |
90 | * To partially enable Skywalk to only let the host (kernel) stack to work, |
91 | * we set both IF_ATTACH_NX_NETIF_COMPAT and IF_ATTACH_NX_FLOWSWITCH flags, |
92 | * leaving IF_ATTACH_NX_FSW_*_NETAGENT unset. This enables netif and |
93 | * flowswitch flowswitch nexus on all eligible interfaces, except that channel |
94 | * (user) networking stack will be disabled. |
95 | */ |
96 | #define SKYWALK_NETWORKING_BSD_ONLY \ |
97 | (IF_ATTACH_NX_NETIF_COMPAT | IF_ATTACH_NX_FLOWSWITCH) |
98 | |
99 | /* |
100 | * macOS default configuration for enabling support for interpose filters, |
101 | * custom IP, custom ether type providers and user networking stack. |
102 | */ |
103 | #define SKYWALK_NETWORKING_MAC_OS \ |
104 | (SKYWALK_NETWORKING_BSD_ONLY | IF_ATTACH_NX_FSW_IP_NETAGENT | \ |
105 | IF_ATTACH_NX_NETIF_NETAGENT | IF_ATTACH_NX_FSW_TRANSPORT_NETAGENT) |
106 | |
107 | /* |
108 | * Disabling Skywalk networking is done by removing IF_ATTACH_NX_NETIF_COMPAT |
109 | * and IF_ATTACH_NX_FSW_*_NETAGENT flags, and leaving IF_ATTACH_NX_FLOWSWITCH |
110 | * set. This disables the compat netif instances and netagent, while still |
111 | * allowing flowswitch nexus creation for native Skywalk drivers, since |
112 | * otherwise they cease to function due to missing interaction with the host |
113 | * (kernel) stack. |
114 | */ |
115 | #define SKYWALK_NETWORKING_DISABLED IF_ATTACH_NX_FLOWSWITCH |
116 | |
117 | #if !XNU_TARGET_OS_OSX |
118 | #ifdef __LP64__ |
119 | #define IF_ATTACH_NX_DEFAULT SKYWALK_NETWORKING_ENABLED |
120 | #else /* !__LP64__ */ |
121 | #define IF_ATTACH_NX_DEFAULT SKYWALK_NETWORKING_DISABLED |
122 | #endif /* !__LP64__ */ |
123 | #else /* XNU_TARGET_OS_OSX */ |
124 | #define IF_ATTACH_NX_DEFAULT SKYWALK_NETWORKING_MAC_OS |
125 | #endif /* XNU_TARGET_OS_OSX */ |
126 | |
127 | #define SK_VERBOSE_SYSCTL "kern.skywalk.verbose" |
128 | /* |
129 | * Verbose flags. |
130 | * |
131 | * When adding new ones, consider the existing scheme based on the areas; |
132 | * try to "fill in the holes" first before extending the range. |
133 | */ |
134 | #define SK_VERB_FLAGS_TABLE(X) \ |
135 | X(SK_VERB_DEFAULT, 0) /* 0x0000000000000001 */ \ |
136 | X(SK_VERB_DUMP, 1) /* 0x0000000000000002 */ \ |
137 | X(SK_VERB_LOCKS, 2) /* 0x0000000000000004 */ \ |
138 | X(SK_VERB_REFCNT, 3) /* 0x0000000000000008 */ \ |
139 | X(SK_VERB_MEM, 4) /* 0x0000000000000010 */ \ |
140 | X(SK_VERB_MEM_ARENA, 5) /* 0x0000000000000020 */ \ |
141 | X(SK_VERB_MEM_CACHE, 6) /* 0x0000000000000040 */ \ |
142 | X(SK_VERB_MEM_REGION, 7) /* 0x0000000000000080 */ \ |
143 | X(SK_VERB_EVENTS, 8) /* 0x0000000000000100 */ \ |
144 | X(SK_VERB_SYNC, 9) /* 0x0000000000000200 */ \ |
145 | X(SK_VERB_NOTIFY, 10) /* 0x0000000000000400 */ \ |
146 | X(SK_VERB_INTR, 11) /* 0x0000000000000800 */ \ |
147 | X(SK_VERB_MONITOR, 12) /* 0x0000000000001000 */ \ |
148 | X(SK_VERB_DEV, 13) /* 0x0000000000002000 */ \ |
149 | X(SK_VERB_HOST, 14) /* 0x0000000000004000 */ \ |
150 | X(SK_VERB_USER, 15) /* 0x0000000000008000 */ \ |
151 | X(SK_VERB_RX, 16) /* 0x0000000000010000 */ \ |
152 | X(SK_VERB_TX, 17) /* 0x0000000000020000 */ \ |
153 | X(SK_VERB_LOOKUP, 18) /* 0x0000000000040000 */ \ |
154 | X(SK_VERB_RING, 19) /* 0x0000000000080000 */ \ |
155 | X(SK_VERB_NETIF, 20) /* 0x0000000000100000 */ \ |
156 | X(SK_VERB_NETIF_MIT, 21) /* 0x0000000000200000 */ \ |
157 | X(SK_VERB_IOSK, 22) /* 0x0000000000400000 */ \ |
158 | X(SK_VERB_CHANNEL, 23) /* 0x0000000000800000 */ \ |
159 | X(SK_VERB_AQM, 25) /* 0x0000000002000000 */ \ |
160 | X(SK_VERB_FSW, 24) /* 0x0000000001000000 */ \ |
161 | X(SK_VERB_FSW_DP, 26) /* 0x0000000004000000 */ \ |
162 | X(SK_VERB_LLINK, 27) /* 0x0000000008000000 */ \ |
163 | X(SK_VERB_FLOW, 28) /* 0x0000000010000000 */ \ |
164 | X(SK_VERB_FLOW_CLASSIFY, 29) /* 0x0000000020000000 */ \ |
165 | X(SK_VERB_FLOW_TRACK, 30) /* 0x0000000040000000 */ \ |
166 | X(SK_VERB_FLOW_ADVISORY, 31) /* 0x0000000080000000 */ \ |
167 | X(SK_VERB_FLOW_ROUTE, 32) /* 0x0000000100000000 */ \ |
168 | X(SK_VERB_FPD, 33) /* 0x0000000200000000 */ \ |
169 | X(__SK_VERB_34__, 34) /* 0x0000000400000000 */ \ |
170 | X(__SK_VERF_35__, 35) /* 0x0000000800000000 */ \ |
171 | X(SK_VERB_USER_PIPE, 36) /* 0x0000001000000000 */ \ |
172 | X(SK_VERB_NA, 37) /* 0x0000002000000000 */ \ |
173 | X(SK_VERB_KERNEL_PIPE, 38) /* 0x0000004000000000 */ \ |
174 | X(SK_VERB_NS_PROTO, 39) /* 0x0000008000000000 */ \ |
175 | X(SK_VERB_NS_TCP, 40) /* 0x0000010000000000 */ \ |
176 | X(SK_VERB_NS_UDP, 41) /* 0x0000020000000000 */ \ |
177 | X(SK_VERB_NS_IPV4, 42) /* 0x0000040000000000 */ \ |
178 | X(SK_VERB_NS_IPV6, 43) /* 0x0000080000000000 */ \ |
179 | X(SK_VERB_COPY, 44) /* 0x0000100000000000 */ \ |
180 | X(SK_VERB_COPY_MBUF, 45) /* 0x0000200000000000 */ \ |
181 | X(SK_VERB_MOVE, 46) /* 0x0000400000000000 */ \ |
182 | X(SK_VERB_MOVE_MBUF, 47) /* 0x0000800000000000 */ \ |
183 | X(SK_VERB_IP_FRAG, 48) /* 0x0001000000000000 */ \ |
184 | X(SK_VERB_ERROR_INJECT, 49) /* 0x0002000000000000 */ \ |
185 | X(SK_VERB_QOS, 50) /* 0x0004000000000000 */ \ |
186 | X(SK_VERB_NXPORT, 51) /* 0x0008000000000000 */ \ |
187 | X(SK_VERB_FILTER, 52) /* 0x0010000000000000 */ \ |
188 | X(SK_VERB_VP, 53) /* 0x0020000000000000 */ \ |
189 | X(SK_VERB_NETIF_POLL, 54) /* 0x0040000000000000 */ \ |
190 | X(SK_VERB_DROP, 55) /* 0x0080000000000000 */ \ |
191 | X(__SK_VERB_56__, 56) /* 0x0100000000000000 */ \ |
192 | X(__SK_VERB_57__, 57) /* 0x0200000000000000 */ \ |
193 | X(__SK_VERB_58__, 58) /* 0x0400000000000000 */ \ |
194 | X(__SK_VERB_59__, 59) /* 0x0800000000000000 */ \ |
195 | X(__SK_VERB_60__, 60) /* 0x1000000000000000 */ \ |
196 | X(__SK_VERB_61__, 61) /* 0x2000000000000000 */ \ |
197 | X(SK_VERB_PRIV, 62) /* 0x4000000000000000 */ \ |
198 | X(SK_VERB_ERROR, 63) /* 0x8000000000000000 */ |
199 | |
200 | #define EXPAND_TO_STRING(name, bitshift) #name, |
201 | #define EXPAND_TO_ENUMERATION(name, bitshift) name = (1ULL << bitshift), |
202 | |
203 | static const char *sk_verb_flags_string[] = { |
204 | SK_VERB_FLAGS_TABLE(EXPAND_TO_STRING) |
205 | }; |
206 | |
207 | enum SK_VERB_FLAGS { |
208 | SK_VERB_FLAGS_TABLE(EXPAND_TO_ENUMERATION) |
209 | }; |
210 | |
211 | #define SK_VERB_FLAGS_STRINGS_MAX \ |
212 | (sizeof (sk_verb_flags_string) / sizeof (sk_verb_flags_string[0])) |
213 | |
214 | #undef EXPAND_TO_STRING |
215 | #undef EXPAND_TO_ENUMERATION |
216 | |
217 | #ifdef KERNEL |
218 | #include <stdint.h> |
219 | #include <sys/types.h> |
220 | #include <sys/time.h> |
221 | #include <mach/vm_types.h> |
222 | #include <mach/vm_param.h> |
223 | #include <kern/cpu_number.h> |
224 | #include <pexpert/pexpert.h> |
225 | |
226 | #if (DEVELOPMENT || DEBUG) |
227 | #define SK_KVA(p) ((uint64_t)(p)) |
228 | #define SK_LOG 1 |
229 | #else |
230 | #define SK_KVA(p) ((uint64_t)VM_KERNEL_ADDRPERM(p)) |
231 | #define SK_LOG 0 |
232 | #endif /* !DEVELOPMENT && !DEBUG */ |
233 | |
234 | #if SK_LOG |
235 | #define SK_LOG_VAR(x) x |
236 | #else |
237 | #define SK_LOG_VAR(x) |
238 | #endif |
239 | |
240 | #define SK_INLINE_ATTRIBUTE __attribute__((always_inline)) |
241 | #define SK_NO_INLINE_ATTRIBUTE __attribute__((noinline)) |
242 | #define SK_LOG_ATTRIBUTE __attribute__((noinline, cold, not_tail_called)) |
243 | |
244 | #if SK_LOG |
245 | /* |
246 | * Because the compiler doesn't know about the %b format specifier, |
247 | * most warnings for _SK_D are disabled by pragma. |
248 | * |
249 | * XXX adi@apple.com: This means the compiler will not warn us about |
250 | * invalid parameters passed to kprintf(), so make sure to scrutinize |
251 | * any changes made to code using any logging macros defined below. |
252 | */ |
253 | |
254 | extern uint64_t sk_verbose; |
255 | #define _SK_D(_flag, _fmt, ...) do { \ |
256 | if (__improbable(((_flag) && (sk_verbose & (_flag)) == (_flag)) || \ |
257 | (_flag) == SK_VERB_ERROR)) { \ |
258 | _Pragma("clang diagnostic push") \ |
259 | _Pragma("clang diagnostic ignored \"-Wformat-invalid-specifier\"") \ |
260 | _Pragma("clang diagnostic ignored \"-Wformat-extra-args\"") \ |
261 | _Pragma("clang diagnostic ignored \"-Wformat\"") \ |
262 | kprintf("SK[%u]: %-30s " _fmt "\n", \ |
263 | cpu_number(), __FUNCTION__, ##__VA_ARGS__); \ |
264 | _Pragma("clang diagnostic pop") \ |
265 | } \ |
266 | } while (0) |
267 | |
268 | #define SK_DF(_flag, _fmt, ...) _SK_D((uint64_t)_flag, _fmt, ##__VA_ARGS__) |
269 | #define SK_D(_fmt, ...) SK_DF(SK_VERB_DEFAULT, _fmt, ##__VA_ARGS__) |
270 | #define SK_ERR(_fmt, ...) SK_DF(SK_VERB_ERROR, _fmt, ##__VA_ARGS__) |
271 | #define SK_DSC(_p, _fmt, ...) SK_ERR("%s(%d): " _fmt, \ |
272 | sk_proc_name_address(_p), sk_proc_pid(_p), ##__VA_ARGS__) |
273 | |
274 | /* rate limited, lps indicates how many per second */ |
275 | #define _SK_RD(_flag, _lps, _fmt, ...) do { \ |
276 | static int __t0, __now, __cnt; \ |
277 | __now = (int)_net_uptime; \ |
278 | if (__t0 != __now) { \ |
279 | __t0 = __now; \ |
280 | __cnt = 0; \ |
281 | } \ |
282 | if (__cnt++ < (_lps)) \ |
283 | SK_DF(_flag, _fmt, ##__VA_ARGS__); \ |
284 | } while (0) |
285 | |
286 | #define SK_RDF(_flag, _lps, _fmt, ...) \ |
287 | _SK_RD(_flag, _lps, _fmt, ##__VA_ARGS__) |
288 | #define SK_RD(_lps, _fmt, ...) \ |
289 | SK_RDF(SK_VERB_DEFAULT, _lps, _fmt, ##__VA_ARGS__) |
290 | #define SK_RDERR(_lps, _fmt, ...) \ |
291 | SK_RDF(SK_VERB_ERROR, _lps, _fmt, ##__VA_ARGS__) |
292 | #else /* !SK_LOG */ |
293 | #define SK_DF(_flag, _fmt, ...) do { ((void)0); } while (0) |
294 | #define SK_D(_fmt, ...) do { ((void)0); } while (0) |
295 | #define SK_ERR(_fmt, ...) do { ((void)0); } while (0) |
296 | #define SK_DSC(_p, _fmt, ...) do { ((void)0); } while (0) |
297 | #define SK_RDF(_flag, _lps, _fmt, ...) do { ((void)0); } while (0) |
298 | #define SK_RD(_lps, _fmt, ...) do { ((void)0); } while (0) |
299 | #define SK_RDERR(_lps, _fmt, ...) do { ((void)0); } while (0) |
300 | #endif /* ! SK_LOG */ |
301 | |
302 | #ifdef BSD_KERNEL_PRIVATE |
303 | #include <skywalk/core/skywalk_var.h> |
304 | #include <skywalk/lib/cuckoo_hashtable.h> |
305 | #include <skywalk/mem/skmem_var.h> |
306 | #include <skywalk/channel/os_channel_event.h> |
307 | #include <skywalk/channel/channel_var.h> |
308 | #include <skywalk/nexus/nexus_var.h> |
309 | #include <skywalk/packet/pbufpool_var.h> |
310 | #include <skywalk/packet/packet_var.h> |
311 | #endif /* BSD_KERNEL_PRIVATE */ |
312 | #endif /* KERNEL */ |
313 | #if !defined(KERNEL) || defined(BSD_KERNEL_PRIVATE) |
314 | #include <skywalk/skywalk_common.h> |
315 | #include <skywalk/os_nexus_private.h> |
316 | #include <skywalk/os_channel_private.h> |
317 | #include <skywalk/os_packet_private.h> |
318 | #include <skywalk/os_stats_private.h> |
319 | #endif /* !KERNEL || BSD_KERNEL_PRIVATE */ |
320 | #endif /* PRIVATE || XNU_KERNEL_PRIVATE */ |
321 | #endif /* _SKYWALK_OS_SKYWALK_PRIVATE_H */ |
322 | |