1 | /* |
2 | * Copyright (c) 2008-2018 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | |
30 | |
31 | /* ---------------------------------------------------------------------------------- |
32 | Application of kernel control for interface creation |
33 | |
34 | Theory of operation: |
35 | utun (user tunnel) acts as glue between kernel control sockets and network interfaces. |
36 | This kernel control will register an interface for every client that connects. |
37 | ---------------------------------------------------------------------------------- */ |
38 | |
39 | #include <sys/systm.h> |
40 | #include <sys/kern_control.h> |
41 | #include <net/kpi_protocol.h> |
42 | #include <net/kpi_interface.h> |
43 | #include <sys/socket.h> |
44 | #include <net/if.h> |
45 | #include <net/if_types.h> |
46 | #include <net/bpf.h> |
47 | #include <net/if_utun.h> |
48 | #include <sys/mbuf.h> |
49 | #include <sys/sockio.h> |
50 | #include <netinet/in.h> |
51 | #include <netinet/ip.h> |
52 | #include <netinet6/in6_var.h> |
53 | #include <netinet6/in6_var.h> |
54 | #include <sys/kauth.h> |
55 | #include <net/necp.h> |
56 | #include <kern/zalloc.h> |
57 | |
58 | #define UTUN_NEXUS 0 |
59 | |
60 | #if UTUN_NEXUS |
61 | static nexus_controller_t utun_ncd; |
62 | static int utun_ncd_refcount; |
63 | static uuid_t utun_kpipe_uuid; |
64 | static uuid_t utun_nx_dom_prov; |
65 | |
66 | typedef struct utun_nx { |
67 | uuid_t if_provider; |
68 | uuid_t if_instance; |
69 | uuid_t ms_provider; |
70 | uuid_t ms_instance; |
71 | uuid_t ms_device; |
72 | uuid_t ms_host; |
73 | uuid_t ms_agent; |
74 | } *utun_nx_t; |
75 | |
76 | #endif // UTUN_NEXUS |
77 | |
78 | /* Control block allocated for each kernel control connection */ |
79 | struct utun_pcb { |
80 | TAILQ_ENTRY(utun_pcb) utun_chain; |
81 | kern_ctl_ref utun_ctlref; |
82 | ifnet_t utun_ifp; |
83 | u_int32_t utun_unit; |
84 | u_int32_t utun_unique_id; |
85 | u_int32_t utun_flags; |
86 | int utun_ext_ifdata_stats; |
87 | u_int32_t utun_max_pending_packets; |
88 | char utun_if_xname[IFXNAMSIZ]; |
89 | char utun_unique_name[IFXNAMSIZ]; |
90 | // PCB lock protects state fields and rings |
91 | decl_lck_rw_data(, utun_pcb_lock); |
92 | struct mbuf * utun_input_chain; |
93 | struct mbuf * utun_input_chain_last; |
94 | // Input chain lock protects the list of input mbufs |
95 | // The input chain lock must be taken AFTER the PCB lock if both are held |
96 | lck_mtx_t utun_input_chain_lock; |
97 | |
98 | #if UTUN_NEXUS |
99 | struct utun_nx utun_nx; |
100 | int utun_kpipe_enabled; |
101 | uuid_t utun_kpipe_uuid; |
102 | void * utun_kpipe_rxring; |
103 | void * utun_kpipe_txring; |
104 | kern_pbufpool_t utun_kpipe_pp; |
105 | |
106 | kern_nexus_t utun_netif_nexus; |
107 | kern_pbufpool_t utun_netif_pp; |
108 | void * utun_netif_rxring; |
109 | void * utun_netif_txring; |
110 | uint64_t utun_netif_txring_size; |
111 | |
112 | u_int32_t utun_slot_size; |
113 | u_int32_t utun_netif_ring_size; |
114 | u_int32_t utun_tx_fsw_ring_size; |
115 | u_int32_t utun_rx_fsw_ring_size; |
116 | bool utun_use_netif; |
117 | bool utun_needs_netagent; |
118 | #endif // UTUN_NEXUS |
119 | }; |
120 | |
121 | /* Kernel Control functions */ |
122 | static errno_t utun_ctl_bind(kern_ctl_ref kctlref, struct sockaddr_ctl *sac, |
123 | void **unitinfo); |
124 | static errno_t utun_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac, |
125 | void **unitinfo); |
126 | static errno_t utun_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit, |
127 | void *unitinfo); |
128 | static errno_t utun_ctl_send(kern_ctl_ref kctlref, u_int32_t unit, |
129 | void *unitinfo, mbuf_t m, int flags); |
130 | static errno_t utun_ctl_getopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, |
131 | int opt, void *data, size_t *len); |
132 | static errno_t utun_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, |
133 | int opt, void *data, size_t len); |
134 | static void utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, |
135 | int flags); |
136 | |
137 | /* Network Interface functions */ |
138 | static void utun_start(ifnet_t interface); |
139 | static errno_t utun_framer(ifnet_t interface, mbuf_t *packet, |
140 | const struct sockaddr *dest, const char *desk_linkaddr, |
141 | const char *frame_type, u_int32_t *prepend_len, u_int32_t *postpend_len); |
142 | static errno_t utun_output(ifnet_t interface, mbuf_t data); |
143 | static errno_t utun_demux(ifnet_t interface, mbuf_t data, char *, |
144 | protocol_family_t *protocol); |
145 | static errno_t utun_add_proto(ifnet_t interface, protocol_family_t protocol, |
146 | const struct ifnet_demux_desc *demux_array, |
147 | u_int32_t demux_count); |
148 | static errno_t utun_del_proto(ifnet_t interface, protocol_family_t protocol); |
149 | static errno_t utun_ioctl(ifnet_t interface, u_long cmd, void *data); |
150 | static void utun_detached(ifnet_t interface); |
151 | |
152 | /* Protocol handlers */ |
153 | static errno_t utun_attach_proto(ifnet_t interface, protocol_family_t proto); |
154 | static errno_t utun_proto_input(ifnet_t interface, protocol_family_t protocol, |
155 | mbuf_t m, char *); |
156 | static errno_t utun_proto_pre_output(ifnet_t interface, protocol_family_t protocol, |
157 | mbuf_t *packet, const struct sockaddr *dest, void *route, |
158 | char *frame_type, char *link_layer_dest); |
159 | static errno_t utun_pkt_input(struct utun_pcb *pcb, mbuf_t m); |
160 | |
161 | #if UTUN_NEXUS |
162 | |
163 | #define UTUN_IF_DEFAULT_SLOT_SIZE 2048 |
164 | #define UTUN_IF_DEFAULT_RING_SIZE 64 |
165 | #define UTUN_IF_DEFAULT_TX_FSW_RING_SIZE 64 |
166 | #define UTUN_IF_DEFAULT_RX_FSW_RING_SIZE 128 |
167 | #define UTUN_IF_DEFAULT_BUF_SEG_SIZE skmem_usr_buf_seg_size |
168 | #define UTUN_IF_HEADROOM_SIZE 32 |
169 | |
170 | #define UTUN_IF_MIN_RING_SIZE 16 |
171 | #define UTUN_IF_MAX_RING_SIZE 1024 |
172 | |
173 | #define UTUN_IF_MIN_SLOT_SIZE 1024 |
174 | #define UTUN_IF_MAX_SLOT_SIZE 4096 |
175 | |
176 | static int sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS; |
177 | static int sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS; |
178 | static int sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS; |
179 | |
180 | static int if_utun_ring_size = UTUN_IF_DEFAULT_RING_SIZE; |
181 | static int if_utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE; |
182 | static int if_utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE; |
183 | |
184 | SYSCTL_DECL(_net_utun); |
185 | SYSCTL_NODE(_net, OID_AUTO, utun, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "UTun" ); |
186 | |
187 | SYSCTL_PROC(_net_utun, OID_AUTO, ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW, |
188 | &if_utun_ring_size, UTUN_IF_DEFAULT_RING_SIZE, &sysctl_if_utun_ring_size, "I" , "" ); |
189 | SYSCTL_PROC(_net_utun, OID_AUTO, tx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW, |
190 | &if_utun_tx_fsw_ring_size, UTUN_IF_DEFAULT_TX_FSW_RING_SIZE, &sysctl_if_utun_tx_fsw_ring_size, "I" , "" ); |
191 | SYSCTL_PROC(_net_utun, OID_AUTO, rx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW, |
192 | &if_utun_rx_fsw_ring_size, UTUN_IF_DEFAULT_RX_FSW_RING_SIZE, &sysctl_if_utun_rx_fsw_ring_size, "I" , "" ); |
193 | |
194 | static errno_t |
195 | utun_register_nexus(void); |
196 | |
197 | static errno_t |
198 | utun_netif_prepare(__unused kern_nexus_t nexus, ifnet_t ifp); |
199 | static errno_t |
200 | utun_nexus_pre_connect(kern_nexus_provider_t nxprov, |
201 | proc_t p, kern_nexus_t nexus, |
202 | nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx); |
203 | static errno_t |
204 | utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
205 | kern_channel_t channel); |
206 | static void |
207 | utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
208 | kern_channel_t channel); |
209 | static void |
210 | utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
211 | kern_channel_t channel); |
212 | static void |
213 | utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
214 | kern_channel_t channel); |
215 | static errno_t |
216 | utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
217 | kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring, |
218 | void **ring_ctx); |
219 | static void |
220 | utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
221 | kern_channel_ring_t ring); |
222 | static errno_t |
223 | utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
224 | kern_channel_ring_t ring, uint32_t flags); |
225 | static errno_t |
226 | utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
227 | kern_channel_ring_t ring, uint32_t flags); |
228 | #endif // UTUN_NEXUS |
229 | |
230 | #define UTUN_DEFAULT_MTU 1500 |
231 | #define (_pcb) (sizeof(u_int32_t) + (((_pcb)->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) ? sizeof(uuid_t) : 0)) |
232 | |
233 | static kern_ctl_ref utun_kctlref; |
234 | static u_int32_t utun_family; |
235 | static lck_attr_t *utun_lck_attr; |
236 | static lck_grp_attr_t *utun_lck_grp_attr; |
237 | static lck_grp_t *utun_lck_grp; |
238 | static lck_mtx_t utun_lock; |
239 | |
240 | TAILQ_HEAD(utun_list, utun_pcb) utun_head; |
241 | |
242 | #define UTUN_PCB_ZONE_MAX 32 |
243 | #define UTUN_PCB_ZONE_NAME "net.if_utun" |
244 | |
245 | static unsigned int utun_pcb_size; /* size of zone element */ |
246 | static struct zone *utun_pcb_zone; /* zone for utun_pcb */ |
247 | |
248 | #if UTUN_NEXUS |
249 | |
250 | static int |
251 | sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS |
252 | { |
253 | #pragma unused(arg1, arg2) |
254 | int value = if_utun_ring_size; |
255 | |
256 | int error = sysctl_handle_int(oidp, &value, 0, req); |
257 | if (error || !req->newptr) { |
258 | return (error); |
259 | } |
260 | |
261 | if (value < UTUN_IF_MIN_RING_SIZE || |
262 | value > UTUN_IF_MAX_RING_SIZE) { |
263 | return (EINVAL); |
264 | } |
265 | |
266 | if_utun_ring_size = value; |
267 | |
268 | return (0); |
269 | } |
270 | |
271 | static int |
272 | sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS |
273 | { |
274 | #pragma unused(arg1, arg2) |
275 | int value = if_utun_tx_fsw_ring_size; |
276 | |
277 | int error = sysctl_handle_int(oidp, &value, 0, req); |
278 | if (error || !req->newptr) { |
279 | return (error); |
280 | } |
281 | |
282 | if (value < UTUN_IF_MIN_RING_SIZE || |
283 | value > UTUN_IF_MAX_RING_SIZE) { |
284 | return (EINVAL); |
285 | } |
286 | |
287 | if_utun_tx_fsw_ring_size = value; |
288 | |
289 | return (0); |
290 | } |
291 | |
292 | static int |
293 | sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS |
294 | { |
295 | #pragma unused(arg1, arg2) |
296 | int value = if_utun_rx_fsw_ring_size; |
297 | |
298 | int error = sysctl_handle_int(oidp, &value, 0, req); |
299 | if (error || !req->newptr) { |
300 | return (error); |
301 | } |
302 | |
303 | if (value < UTUN_IF_MIN_RING_SIZE || |
304 | value > UTUN_IF_MAX_RING_SIZE) { |
305 | return (EINVAL); |
306 | } |
307 | |
308 | if_utun_rx_fsw_ring_size = value; |
309 | |
310 | return (0); |
311 | } |
312 | |
313 | static errno_t |
314 | utun_netif_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
315 | kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring, |
316 | void **ring_ctx) |
317 | { |
318 | #pragma unused(nxprov) |
319 | #pragma unused(channel) |
320 | #pragma unused(ring_ctx) |
321 | struct utun_pcb *pcb = kern_nexus_get_context(nexus); |
322 | if (!is_tx_ring) { |
323 | VERIFY(pcb->utun_netif_rxring == NULL); |
324 | pcb->utun_netif_rxring = ring; |
325 | } else { |
326 | VERIFY(pcb->utun_netif_txring == NULL); |
327 | pcb->utun_netif_txring = ring; |
328 | } |
329 | return 0; |
330 | } |
331 | |
332 | static void |
333 | utun_netif_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
334 | kern_channel_ring_t ring) |
335 | { |
336 | #pragma unused(nxprov) |
337 | struct utun_pcb *pcb = kern_nexus_get_context(nexus); |
338 | if (pcb->utun_netif_rxring == ring) { |
339 | pcb->utun_netif_rxring = NULL; |
340 | } else if (pcb->utun_netif_txring == ring) { |
341 | pcb->utun_netif_txring = NULL; |
342 | } |
343 | } |
344 | |
345 | static errno_t |
346 | utun_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
347 | kern_channel_ring_t tx_ring, uint32_t flags) |
348 | { |
349 | #pragma unused(nxprov) |
350 | #pragma unused(flags) |
351 | struct utun_pcb *pcb = kern_nexus_get_context(nexus); |
352 | |
353 | struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats; |
354 | |
355 | lck_rw_lock_shared(&pcb->utun_pcb_lock); |
356 | |
357 | struct kern_channel_ring_stat_increment tx_ring_stats; |
358 | bzero(&tx_ring_stats, sizeof(tx_ring_stats)); |
359 | kern_channel_slot_t tx_pslot = NULL; |
360 | kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); |
361 | |
362 | STATS_INC(nifs, NETIF_STATS_TXSYNC); |
363 | |
364 | if (tx_slot == NULL) { |
365 | // Nothing to write, don't bother signalling |
366 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
367 | return 0; |
368 | } |
369 | |
370 | if (pcb->utun_kpipe_enabled) { |
371 | kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring; |
372 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
373 | |
374 | // Signal the kernel pipe ring to read |
375 | if (rx_ring != NULL) { |
376 | kern_channel_notify(rx_ring, 0); |
377 | } |
378 | return 0; |
379 | } |
380 | |
381 | // If we're here, we're injecting into the utun kernel control socket |
382 | while (tx_slot != NULL) { |
383 | size_t length = 0; |
384 | mbuf_t data = NULL; |
385 | |
386 | kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); |
387 | |
388 | if (tx_ph == 0) { |
389 | // Advance TX ring |
390 | tx_pslot = tx_slot; |
391 | tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); |
392 | continue; |
393 | } |
394 | (void) kern_channel_slot_detach_packet(tx_ring, tx_slot, tx_ph); |
395 | |
396 | // Advance TX ring |
397 | tx_pslot = tx_slot; |
398 | tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); |
399 | |
400 | kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); |
401 | VERIFY(tx_buf != NULL); |
402 | |
403 | /* tx_baddr is the absolute buffer address */ |
404 | uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); |
405 | VERIFY(tx_baddr != 0); |
406 | |
407 | bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0); |
408 | |
409 | uint16_t tx_offset = kern_buflet_get_data_offset(tx_buf); |
410 | uint32_t tx_length = kern_buflet_get_data_length(tx_buf); |
411 | |
412 | // The offset must be large enough for the headers |
413 | VERIFY(tx_offset >= UTUN_HEADER_SIZE(pcb)); |
414 | |
415 | // Find family |
416 | uint32_t af = 0; |
417 | uint8_t vhl = *(uint8_t *)(tx_baddr + tx_offset); |
418 | u_int ip_version = (vhl >> 4); |
419 | switch (ip_version) { |
420 | case 4: { |
421 | af = AF_INET; |
422 | break; |
423 | } |
424 | case 6: { |
425 | af = AF_INET6; |
426 | break; |
427 | } |
428 | default: { |
429 | printf("utun_netif_sync_tx %s: unknown ip version %u vhl %u tx_offset %u len %u header_size %zu\n" , |
430 | pcb->utun_ifp->if_xname, ip_version, vhl, tx_offset, tx_length, |
431 | UTUN_HEADER_SIZE(pcb)); |
432 | break; |
433 | } |
434 | } |
435 | |
436 | tx_offset -= UTUN_HEADER_SIZE(pcb); |
437 | tx_length += UTUN_HEADER_SIZE(pcb); |
438 | tx_baddr += tx_offset; |
439 | |
440 | length = MIN(tx_length, pcb->utun_slot_size); |
441 | |
442 | // Copy in family |
443 | memcpy(tx_baddr, &af, sizeof(af)); |
444 | if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) { |
445 | kern_packet_get_euuid(tx_ph, (void *)(tx_baddr + sizeof(af))); |
446 | } |
447 | |
448 | if (length > 0) { |
449 | errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data); |
450 | if (error == 0) { |
451 | error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT); |
452 | if (error == 0) { |
453 | error = utun_output(pcb->utun_ifp, data); |
454 | if (error != 0) { |
455 | printf("utun_netif_sync_tx %s - utun_output error %d\n" , pcb->utun_ifp->if_xname, error); |
456 | } |
457 | } else { |
458 | printf("utun_netif_sync_tx %s - mbuf_copyback(%zu) error %d\n" , pcb->utun_ifp->if_xname, length, error); |
459 | STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); |
460 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
461 | mbuf_freem(data); |
462 | data = NULL; |
463 | } |
464 | } else { |
465 | printf("utun_netif_sync_tx %s - mbuf_gethdr error %d\n" , pcb->utun_ifp->if_xname, error); |
466 | STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); |
467 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
468 | } |
469 | } else { |
470 | printf("utun_netif_sync_tx %s - 0 length packet\n" , pcb->utun_ifp->if_xname); |
471 | STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); |
472 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
473 | } |
474 | |
475 | kern_pbufpool_free(tx_ring->ckr_pp, tx_ph); |
476 | |
477 | if (data == NULL) { |
478 | continue; |
479 | } |
480 | |
481 | STATS_INC(nifs, NETIF_STATS_TXPKTS); |
482 | STATS_INC(nifs, NETIF_STATS_TXCOPY_MBUF); |
483 | |
484 | tx_ring_stats.kcrsi_slots_transferred++; |
485 | tx_ring_stats.kcrsi_bytes_transferred += length; |
486 | } |
487 | |
488 | if (tx_pslot) { |
489 | kern_channel_advance_slot(tx_ring, tx_pslot); |
490 | kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats); |
491 | (void)kern_channel_reclaim(tx_ring); |
492 | } |
493 | |
494 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
495 | |
496 | return 0; |
497 | } |
498 | |
499 | static errno_t |
500 | utun_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
501 | kern_channel_ring_t ring, __unused uint32_t flags) |
502 | { |
503 | #pragma unused(nxprov) |
504 | struct utun_pcb *pcb = kern_nexus_get_context(nexus); |
505 | boolean_t more = false; |
506 | errno_t rc = 0; |
507 | |
508 | /* |
509 | * Refill and sync the ring; we may be racing against another thread doing |
510 | * an RX sync that also wants to do kr_enter(), and so use the blocking |
511 | * variant here. |
512 | */ |
513 | rc = kern_channel_tx_refill_canblock(ring, UINT32_MAX, UINT32_MAX, true, &more); |
514 | if (rc != 0 && rc != EAGAIN && rc != EBUSY) { |
515 | printf("%s, tx refill failed %d\n" , __func__, rc); |
516 | } |
517 | |
518 | (void) kr_enter(ring, TRUE); |
519 | lck_rw_lock_shared(&pcb->utun_pcb_lock); |
520 | |
521 | if (pcb->utun_kpipe_enabled) { |
522 | uint32_t tx_available = kern_channel_available_slot_count(ring); |
523 | if (pcb->utun_netif_txring_size > 0 && |
524 | tx_available >= pcb->utun_netif_txring_size - 1) { |
525 | // No room left in tx ring, disable output for now |
526 | errno_t error = ifnet_disable_output(pcb->utun_ifp); |
527 | if (error != 0) { |
528 | printf("utun_netif_tx_doorbell: ifnet_disable_output returned error %d\n" , error); |
529 | } |
530 | } |
531 | } |
532 | |
533 | if (pcb->utun_kpipe_enabled) { |
534 | kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring; |
535 | |
536 | // Unlock while calling notify |
537 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
538 | // Signal the kernel pipe ring to read |
539 | if (rx_ring != NULL) { |
540 | kern_channel_notify(rx_ring, 0); |
541 | } |
542 | } else { |
543 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
544 | } |
545 | |
546 | kr_exit(ring); |
547 | |
548 | return (0); |
549 | } |
550 | |
551 | static errno_t |
552 | utun_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
553 | kern_channel_ring_t rx_ring, uint32_t flags) |
554 | { |
555 | #pragma unused(nxprov) |
556 | #pragma unused(flags) |
557 | struct utun_pcb *pcb = kern_nexus_get_context(nexus); |
558 | struct kern_channel_ring_stat_increment rx_ring_stats; |
559 | |
560 | struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats; |
561 | |
562 | lck_rw_lock_shared(&pcb->utun_pcb_lock); |
563 | |
564 | // Reclaim user-released slots |
565 | (void) kern_channel_reclaim(rx_ring); |
566 | |
567 | STATS_INC(nifs, NETIF_STATS_RXSYNC); |
568 | |
569 | uint32_t avail = kern_channel_available_slot_count(rx_ring); |
570 | if (avail == 0) { |
571 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
572 | return 0; |
573 | } |
574 | |
575 | struct kern_pbufpool *rx_pp = rx_ring->ckr_pp; |
576 | VERIFY(rx_pp != NULL); |
577 | bzero(&rx_ring_stats, sizeof(rx_ring_stats)); |
578 | kern_channel_slot_t rx_pslot = NULL; |
579 | kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL); |
580 | |
581 | while (rx_slot != NULL) { |
582 | // Check for a waiting packet |
583 | lck_mtx_lock(&pcb->utun_input_chain_lock); |
584 | mbuf_t data = pcb->utun_input_chain; |
585 | if (data == NULL) { |
586 | lck_mtx_unlock(&pcb->utun_input_chain_lock); |
587 | break; |
588 | } |
589 | |
590 | // Allocate rx packet |
591 | kern_packet_t rx_ph = 0; |
592 | errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); |
593 | if (__improbable(error != 0)) { |
594 | STATS_INC(nifs, NETIF_STATS_NOMEM_PKT); |
595 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
596 | lck_mtx_unlock(&pcb->utun_input_chain_lock); |
597 | break; |
598 | } |
599 | |
600 | // Advance waiting packets |
601 | pcb->utun_input_chain = data->m_nextpkt; |
602 | data->m_nextpkt = NULL; |
603 | if (pcb->utun_input_chain == NULL) { |
604 | pcb->utun_input_chain_last = NULL; |
605 | } |
606 | lck_mtx_unlock(&pcb->utun_input_chain_lock); |
607 | |
608 | size_t header_offset = UTUN_HEADER_SIZE(pcb); |
609 | size_t length = mbuf_pkthdr_len(data); |
610 | |
611 | if (length < header_offset) { |
612 | // mbuf is too small |
613 | mbuf_freem(data); |
614 | kern_pbufpool_free(rx_pp, rx_ph); |
615 | STATS_INC(nifs, NETIF_STATS_BADLEN); |
616 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
617 | printf("utun_netif_sync_rx %s: legacy packet length too short for header %zu < %zu\n" , |
618 | pcb->utun_ifp->if_xname, length, header_offset); |
619 | continue; |
620 | } |
621 | |
622 | length -= header_offset; |
623 | if (length > rx_pp->pp_buflet_size) { |
624 | // Flush data |
625 | mbuf_freem(data); |
626 | kern_pbufpool_free(rx_pp, rx_ph); |
627 | STATS_INC(nifs, NETIF_STATS_BADLEN); |
628 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
629 | printf("utun_netif_sync_rx %s: legacy packet length %zu > %u\n" , |
630 | pcb->utun_ifp->if_xname, length, rx_pp->pp_buflet_size); |
631 | continue; |
632 | } |
633 | |
634 | mbuf_pkthdr_setrcvif(data, pcb->utun_ifp); |
635 | |
636 | // Fillout rx packet |
637 | kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); |
638 | VERIFY(rx_buf != NULL); |
639 | void *rx_baddr = kern_buflet_get_object_address(rx_buf); |
640 | VERIFY(rx_baddr != NULL); |
641 | |
642 | // Copy-in data from mbuf to buflet |
643 | mbuf_copydata(data, header_offset, length, (void *)rx_baddr); |
644 | kern_packet_clear_flow_uuid(rx_ph); // Zero flow id |
645 | |
646 | // Finalize and attach the packet |
647 | error = kern_buflet_set_data_offset(rx_buf, 0); |
648 | VERIFY(error == 0); |
649 | error = kern_buflet_set_data_length(rx_buf, length); |
650 | VERIFY(error == 0); |
651 | error = kern_packet_set_link_header_offset(rx_ph, 0); |
652 | VERIFY(error == 0); |
653 | error = kern_packet_set_network_header_offset(rx_ph, 0); |
654 | VERIFY(error == 0); |
655 | error = kern_packet_finalize(rx_ph); |
656 | VERIFY(error == 0); |
657 | error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); |
658 | VERIFY(error == 0); |
659 | |
660 | STATS_INC(nifs, NETIF_STATS_RXPKTS); |
661 | STATS_INC(nifs, NETIF_STATS_RXCOPY_MBUF); |
662 | bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0); |
663 | |
664 | rx_ring_stats.kcrsi_slots_transferred++; |
665 | rx_ring_stats.kcrsi_bytes_transferred += length; |
666 | |
667 | mbuf_freem(data); |
668 | |
669 | // Advance ring |
670 | rx_pslot = rx_slot; |
671 | rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); |
672 | } |
673 | |
674 | struct kern_channel_ring_stat_increment tx_ring_stats; |
675 | bzero(&tx_ring_stats, sizeof(tx_ring_stats)); |
676 | kern_channel_ring_t tx_ring = pcb->utun_kpipe_txring; |
677 | kern_channel_slot_t tx_pslot = NULL; |
678 | kern_channel_slot_t tx_slot = NULL; |
679 | if (tx_ring == NULL) { |
680 | // Net-If TX ring not set up yet, nothing to read |
681 | goto done; |
682 | } |
683 | |
684 | // Unlock utun before entering ring |
685 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
686 | |
687 | (void)kr_enter(tx_ring, TRUE); |
688 | |
689 | // Lock again after entering and validate |
690 | lck_rw_lock_shared(&pcb->utun_pcb_lock); |
691 | if (tx_ring != pcb->utun_kpipe_txring) { |
692 | goto done; |
693 | } |
694 | |
695 | tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); |
696 | if (tx_slot == NULL) { |
697 | // Nothing to read, don't bother signalling |
698 | goto done; |
699 | } |
700 | |
701 | while (rx_slot != NULL && tx_slot != NULL) { |
702 | // Allocate rx packet |
703 | kern_packet_t rx_ph = 0; |
704 | kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); |
705 | |
706 | // Advance TX ring |
707 | tx_pslot = tx_slot; |
708 | tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); |
709 | |
710 | /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */ |
711 | if (tx_ph == 0) { |
712 | continue; |
713 | } |
714 | |
715 | /* XXX We could try this alloc before advancing the slot to avoid |
716 | * dropping the packet on failure to allocate. |
717 | */ |
718 | errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); |
719 | if (__improbable(error != 0)) { |
720 | STATS_INC(nifs, NETIF_STATS_NOMEM_PKT); |
721 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
722 | break; |
723 | } |
724 | |
725 | kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); |
726 | VERIFY(tx_buf != NULL); |
727 | uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); |
728 | VERIFY(tx_baddr != 0); |
729 | tx_baddr += kern_buflet_get_data_offset(tx_buf); |
730 | |
731 | // Check packet length |
732 | size_t header_offset = UTUN_HEADER_SIZE(pcb); |
733 | uint32_t tx_length = kern_packet_get_data_length(tx_ph); |
734 | if (tx_length < header_offset) { |
735 | // Packet is too small |
736 | kern_pbufpool_free(rx_pp, rx_ph); |
737 | STATS_INC(nifs, NETIF_STATS_BADLEN); |
738 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
739 | printf("utun_netif_sync_rx %s: packet length too short for header %u < %zu\n" , |
740 | pcb->utun_ifp->if_xname, tx_length, header_offset); |
741 | continue; |
742 | } |
743 | |
744 | size_t length = MIN(tx_length - header_offset, |
745 | pcb->utun_slot_size); |
746 | |
747 | tx_ring_stats.kcrsi_slots_transferred++; |
748 | tx_ring_stats.kcrsi_bytes_transferred += length; |
749 | |
750 | // Fillout rx packet |
751 | kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); |
752 | VERIFY(rx_buf != NULL); |
753 | void *rx_baddr = kern_buflet_get_object_address(rx_buf); |
754 | VERIFY(rx_baddr != NULL); |
755 | |
756 | // Copy-in data from tx to rx |
757 | memcpy((void *)rx_baddr, (void *)(tx_baddr + header_offset), length); |
758 | kern_packet_clear_flow_uuid(rx_ph); // Zero flow id |
759 | |
760 | // Finalize and attach the packet |
761 | error = kern_buflet_set_data_offset(rx_buf, 0); |
762 | VERIFY(error == 0); |
763 | error = kern_buflet_set_data_length(rx_buf, length); |
764 | VERIFY(error == 0); |
765 | error = kern_packet_set_link_header_offset(rx_ph, 0); |
766 | VERIFY(error == 0); |
767 | error = kern_packet_set_network_header_offset(rx_ph, 0); |
768 | VERIFY(error == 0); |
769 | error = kern_packet_finalize(rx_ph); |
770 | VERIFY(error == 0); |
771 | error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); |
772 | VERIFY(error == 0); |
773 | |
774 | STATS_INC(nifs, NETIF_STATS_RXPKTS); |
775 | STATS_INC(nifs, NETIF_STATS_RXCOPY_DIRECT); |
776 | bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0); |
777 | |
778 | rx_ring_stats.kcrsi_slots_transferred++; |
779 | rx_ring_stats.kcrsi_bytes_transferred += length; |
780 | |
781 | rx_pslot = rx_slot; |
782 | rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); |
783 | } |
784 | |
785 | done: |
786 | if (rx_pslot) { |
787 | kern_channel_advance_slot(rx_ring, rx_pslot); |
788 | kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats); |
789 | } |
790 | |
791 | if (tx_pslot) { |
792 | kern_channel_advance_slot(tx_ring, tx_pslot); |
793 | kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats); |
794 | (void)kern_channel_reclaim(tx_ring); |
795 | } |
796 | |
797 | // Unlock first, then exit ring |
798 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
799 | if (tx_ring != NULL) { |
800 | if (tx_pslot != NULL) { |
801 | kern_channel_notify(tx_ring, 0); |
802 | } |
803 | kr_exit(tx_ring); |
804 | } |
805 | |
806 | return 0; |
807 | } |
808 | |
809 | static errno_t |
810 | utun_nexus_ifattach(struct utun_pcb *pcb, |
811 | struct ifnet_init_eparams *init_params, |
812 | struct ifnet **ifp) |
813 | { |
814 | errno_t err; |
815 | nexus_controller_t controller = kern_nexus_shared_controller(); |
816 | struct kern_nexus_net_init net_init; |
817 | struct kern_pbufpool_init pp_init; |
818 | |
819 | nexus_name_t provider_name; |
820 | snprintf((char *)provider_name, sizeof(provider_name), |
821 | "com.apple.netif.%s" , pcb->utun_if_xname); |
822 | |
823 | struct kern_nexus_provider_init prov_init = { |
824 | .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION, |
825 | .nxpi_flags = NXPIF_VIRTUAL_DEVICE, |
826 | .nxpi_pre_connect = utun_nexus_pre_connect, |
827 | .nxpi_connected = utun_nexus_connected, |
828 | .nxpi_pre_disconnect = utun_netif_pre_disconnect, |
829 | .nxpi_disconnected = utun_nexus_disconnected, |
830 | .nxpi_ring_init = utun_netif_ring_init, |
831 | .nxpi_ring_fini = utun_netif_ring_fini, |
832 | .nxpi_slot_init = NULL, |
833 | .nxpi_slot_fini = NULL, |
834 | .nxpi_sync_tx = utun_netif_sync_tx, |
835 | .nxpi_sync_rx = utun_netif_sync_rx, |
836 | .nxpi_tx_doorbell = utun_netif_tx_doorbell, |
837 | }; |
838 | |
839 | nexus_attr_t nxa = NULL; |
840 | err = kern_nexus_attr_create(&nxa); |
841 | if (err != 0) { |
842 | printf("%s: kern_nexus_attr_create failed: %d\n" , |
843 | __func__, err); |
844 | goto failed; |
845 | } |
846 | |
847 | uint64_t slot_buffer_size = pcb->utun_slot_size; |
848 | err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); |
849 | VERIFY(err == 0); |
850 | |
851 | // Reset ring size for netif nexus to limit memory usage |
852 | uint64_t ring_size = pcb->utun_netif_ring_size; |
853 | err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size); |
854 | VERIFY(err == 0); |
855 | err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size); |
856 | VERIFY(err == 0); |
857 | |
858 | pcb->utun_netif_txring_size = ring_size; |
859 | |
860 | bzero(&pp_init, sizeof (pp_init)); |
861 | pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION; |
862 | pp_init.kbi_packets = pcb->utun_netif_ring_size * 2; |
863 | pp_init.kbi_bufsize = pcb->utun_slot_size; |
864 | pp_init.kbi_buf_seg_size = UTUN_IF_DEFAULT_BUF_SEG_SIZE; |
865 | pp_init.kbi_max_frags = 1; |
866 | (void) snprintf((char *)pp_init.kbi_name, sizeof (pp_init.kbi_name), |
867 | "%s" , provider_name); |
868 | |
869 | err = kern_pbufpool_create(&pp_init, &pp_init, &pcb->utun_netif_pp, NULL); |
870 | if (err != 0) { |
871 | printf("%s pbufbool create failed, error %d\n" , __func__, err); |
872 | goto failed; |
873 | } |
874 | |
875 | err = kern_nexus_controller_register_provider(controller, |
876 | utun_nx_dom_prov, |
877 | provider_name, |
878 | &prov_init, |
879 | sizeof(prov_init), |
880 | nxa, |
881 | &pcb->utun_nx.if_provider); |
882 | if (err != 0) { |
883 | printf("%s register provider failed, error %d\n" , |
884 | __func__, err); |
885 | goto failed; |
886 | } |
887 | |
888 | bzero(&net_init, sizeof(net_init)); |
889 | net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION; |
890 | net_init.nxneti_flags = 0; |
891 | net_init.nxneti_eparams = init_params; |
892 | net_init.nxneti_lladdr = NULL; |
893 | net_init.nxneti_prepare = utun_netif_prepare; |
894 | net_init.nxneti_tx_pbufpool = pcb->utun_netif_pp; |
895 | err = kern_nexus_controller_alloc_net_provider_instance(controller, |
896 | pcb->utun_nx.if_provider, |
897 | pcb, |
898 | &pcb->utun_nx.if_instance, |
899 | &net_init, |
900 | ifp); |
901 | if (err != 0) { |
902 | printf("%s alloc_net_provider_instance failed, %d\n" , |
903 | __func__, err); |
904 | kern_nexus_controller_deregister_provider(controller, |
905 | pcb->utun_nx.if_provider); |
906 | uuid_clear(pcb->utun_nx.if_provider); |
907 | goto failed; |
908 | } |
909 | |
910 | failed: |
911 | if (nxa) { |
912 | kern_nexus_attr_destroy(nxa); |
913 | } |
914 | if (err && pcb->utun_netif_pp != NULL) { |
915 | kern_pbufpool_destroy(pcb->utun_netif_pp); |
916 | pcb->utun_netif_pp = NULL; |
917 | } |
918 | return (err); |
919 | } |
920 | |
921 | static void |
922 | utun_detach_provider_and_instance(uuid_t provider, uuid_t instance) |
923 | { |
924 | nexus_controller_t controller = kern_nexus_shared_controller(); |
925 | errno_t err; |
926 | |
927 | if (!uuid_is_null(instance)) { |
928 | err = kern_nexus_controller_free_provider_instance(controller, |
929 | instance); |
930 | if (err != 0) { |
931 | printf("%s free_provider_instance failed %d\n" , |
932 | __func__, err); |
933 | } |
934 | uuid_clear(instance); |
935 | } |
936 | if (!uuid_is_null(provider)) { |
937 | err = kern_nexus_controller_deregister_provider(controller, |
938 | provider); |
939 | if (err != 0) { |
940 | printf("%s deregister_provider %d\n" , __func__, err); |
941 | } |
942 | uuid_clear(provider); |
943 | } |
944 | return; |
945 | } |
946 | |
947 | static void |
948 | utun_nexus_detach(struct utun_pcb *pcb) |
949 | { |
950 | utun_nx_t nx = &pcb->utun_nx; |
951 | nexus_controller_t controller = kern_nexus_shared_controller(); |
952 | errno_t err; |
953 | |
954 | if (!uuid_is_null(nx->ms_host)) { |
955 | err = kern_nexus_ifdetach(controller, |
956 | nx->ms_instance, |
957 | nx->ms_host); |
958 | if (err != 0) { |
959 | printf("%s: kern_nexus_ifdetach ms host failed %d\n" , |
960 | __func__, err); |
961 | } |
962 | } |
963 | |
964 | if (!uuid_is_null(nx->ms_device)) { |
965 | err = kern_nexus_ifdetach(controller, |
966 | nx->ms_instance, |
967 | nx->ms_device); |
968 | if (err != 0) { |
969 | printf("%s: kern_nexus_ifdetach ms device failed %d\n" , |
970 | __func__, err); |
971 | } |
972 | } |
973 | |
974 | utun_detach_provider_and_instance(nx->if_provider, |
975 | nx->if_instance); |
976 | utun_detach_provider_and_instance(nx->ms_provider, |
977 | nx->ms_instance); |
978 | |
979 | if (pcb->utun_netif_pp != NULL) { |
980 | kern_pbufpool_destroy(pcb->utun_netif_pp); |
981 | pcb->utun_netif_pp = NULL; |
982 | |
983 | } |
984 | memset(nx, 0, sizeof(*nx)); |
985 | } |
986 | |
987 | static errno_t |
988 | utun_create_fs_provider_and_instance(struct utun_pcb *pcb, |
989 | uint32_t subtype, const char *type_name, |
990 | const char *ifname, |
991 | uuid_t *provider, uuid_t *instance) |
992 | { |
993 | nexus_attr_t attr = NULL; |
994 | nexus_controller_t controller = kern_nexus_shared_controller(); |
995 | uuid_t dom_prov; |
996 | errno_t err; |
997 | struct kern_nexus_init init; |
998 | nexus_name_t provider_name; |
999 | |
1000 | err = kern_nexus_get_builtin_domain_provider(NEXUS_TYPE_FLOW_SWITCH, |
1001 | &dom_prov); |
1002 | if (err != 0) { |
1003 | printf("%s can't get %s provider, error %d\n" , |
1004 | __func__, type_name, err); |
1005 | goto failed; |
1006 | } |
1007 | |
1008 | err = kern_nexus_attr_create(&attr); |
1009 | if (err != 0) { |
1010 | printf("%s: kern_nexus_attr_create failed: %d\n" , |
1011 | __func__, err); |
1012 | goto failed; |
1013 | } |
1014 | |
1015 | err = kern_nexus_attr_set(attr, NEXUS_ATTR_EXTENSIONS, subtype); |
1016 | VERIFY(err == 0); |
1017 | |
1018 | uint64_t slot_buffer_size = pcb->utun_slot_size; |
1019 | err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); |
1020 | VERIFY(err == 0); |
1021 | |
1022 | // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif. |
1023 | uint64_t tx_ring_size = pcb->utun_tx_fsw_ring_size; |
1024 | err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size); |
1025 | VERIFY(err == 0); |
1026 | uint64_t rx_ring_size = pcb->utun_rx_fsw_ring_size; |
1027 | err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size); |
1028 | VERIFY(err == 0); |
1029 | |
1030 | snprintf((char *)provider_name, sizeof(provider_name), |
1031 | "com.apple.%s.%s" , type_name, ifname); |
1032 | err = kern_nexus_controller_register_provider(controller, |
1033 | dom_prov, |
1034 | provider_name, |
1035 | NULL, |
1036 | 0, |
1037 | attr, |
1038 | provider); |
1039 | kern_nexus_attr_destroy(attr); |
1040 | attr = NULL; |
1041 | if (err != 0) { |
1042 | printf("%s register %s provider failed, error %d\n" , |
1043 | __func__, type_name, err); |
1044 | goto failed; |
1045 | } |
1046 | bzero(&init, sizeof (init)); |
1047 | init.nxi_version = KERN_NEXUS_CURRENT_VERSION; |
1048 | err = kern_nexus_controller_alloc_provider_instance(controller, |
1049 | *provider, |
1050 | NULL, |
1051 | instance, &init); |
1052 | if (err != 0) { |
1053 | printf("%s alloc_provider_instance %s failed, %d\n" , |
1054 | __func__, type_name, err); |
1055 | kern_nexus_controller_deregister_provider(controller, |
1056 | *provider); |
1057 | uuid_clear(*provider); |
1058 | } |
1059 | failed: |
1060 | return (err); |
1061 | } |
1062 | |
1063 | static errno_t |
1064 | utun_multistack_attach(struct utun_pcb *pcb) |
1065 | { |
1066 | nexus_controller_t controller = kern_nexus_shared_controller(); |
1067 | errno_t err = 0; |
1068 | utun_nx_t nx = &pcb->utun_nx; |
1069 | |
1070 | // Allocate multistack flowswitch |
1071 | err = utun_create_fs_provider_and_instance(pcb, |
1072 | NEXUS_EXTENSION_FSW_TYPE_MULTISTACK, |
1073 | "multistack" , |
1074 | pcb->utun_ifp->if_xname, |
1075 | &nx->ms_provider, |
1076 | &nx->ms_instance); |
1077 | if (err != 0) { |
1078 | printf("%s: failed to create bridge provider and instance\n" , |
1079 | __func__); |
1080 | goto failed; |
1081 | } |
1082 | |
1083 | // Attach multistack to device port |
1084 | err = kern_nexus_ifattach(controller, nx->ms_instance, |
1085 | NULL, nx->if_instance, |
1086 | FALSE, &nx->ms_device); |
1087 | if (err != 0) { |
1088 | printf("%s kern_nexus_ifattach ms device %d\n" , __func__, err); |
1089 | goto failed; |
1090 | } |
1091 | |
1092 | // Attach multistack to host port |
1093 | err = kern_nexus_ifattach(controller, nx->ms_instance, |
1094 | NULL, nx->if_instance, |
1095 | TRUE, &nx->ms_host); |
1096 | if (err != 0) { |
1097 | printf("%s kern_nexus_ifattach ms host %d\n" , __func__, err); |
1098 | goto failed; |
1099 | } |
1100 | |
1101 | // Extract the agent UUID and save for later |
1102 | struct kern_nexus *multistack_nx = nx_find(nx->ms_instance, false); |
1103 | if (multistack_nx != NULL) { |
1104 | struct nx_flowswitch *flowswitch = NX_FSW_PRIVATE(multistack_nx); |
1105 | if (flowswitch != NULL) { |
1106 | FSW_RLOCK(flowswitch); |
1107 | struct fsw_ms_context *ms_context = (struct fsw_ms_context *)flowswitch->fsw_ops_private; |
1108 | if (ms_context != NULL) { |
1109 | uuid_copy(nx->ms_agent, ms_context->mc_agent_uuid); |
1110 | } else { |
1111 | printf("utun_multistack_attach - fsw_ms_context is NULL\n" ); |
1112 | } |
1113 | FSW_UNLOCK(flowswitch); |
1114 | } else { |
1115 | printf("utun_multistack_attach - flowswitch is NULL\n" ); |
1116 | } |
1117 | nx_release(multistack_nx); |
1118 | } else { |
1119 | printf("utun_multistack_attach - unable to find multistack nexus\n" ); |
1120 | } |
1121 | |
1122 | return (0); |
1123 | |
1124 | failed: |
1125 | utun_nexus_detach(pcb); |
1126 | |
1127 | errno_t detach_error = 0; |
1128 | if ((detach_error = ifnet_detach(pcb->utun_ifp)) != 0) { |
1129 | panic("utun_multistack_attach - ifnet_detach failed: %d\n" , detach_error); |
1130 | /* NOT REACHED */ |
1131 | } |
1132 | |
1133 | return (err); |
1134 | } |
1135 | |
1136 | static errno_t |
1137 | utun_register_kernel_pipe_nexus(void) |
1138 | { |
1139 | nexus_attr_t nxa = NULL; |
1140 | errno_t result; |
1141 | |
1142 | lck_mtx_lock(&utun_lock); |
1143 | if (utun_ncd_refcount++) { |
1144 | lck_mtx_unlock(&utun_lock); |
1145 | return 0; |
1146 | } |
1147 | |
1148 | result = kern_nexus_controller_create(&utun_ncd); |
1149 | if (result) { |
1150 | printf("%s: kern_nexus_controller_create failed: %d\n" , |
1151 | __FUNCTION__, result); |
1152 | goto done; |
1153 | } |
1154 | |
1155 | uuid_t dom_prov; |
1156 | result = kern_nexus_get_builtin_domain_provider( |
1157 | NEXUS_TYPE_KERNEL_PIPE, &dom_prov); |
1158 | if (result) { |
1159 | printf("%s: kern_nexus_get_builtin_domain_provider failed: %d\n" , |
1160 | __FUNCTION__, result); |
1161 | goto done; |
1162 | } |
1163 | |
1164 | struct kern_nexus_provider_init prov_init = { |
1165 | .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION, |
1166 | .nxpi_flags = NXPIF_VIRTUAL_DEVICE, |
1167 | .nxpi_pre_connect = utun_nexus_pre_connect, |
1168 | .nxpi_connected = utun_nexus_connected, |
1169 | .nxpi_pre_disconnect = utun_nexus_pre_disconnect, |
1170 | .nxpi_disconnected = utun_nexus_disconnected, |
1171 | .nxpi_ring_init = utun_kpipe_ring_init, |
1172 | .nxpi_ring_fini = utun_kpipe_ring_fini, |
1173 | .nxpi_slot_init = NULL, |
1174 | .nxpi_slot_fini = NULL, |
1175 | .nxpi_sync_tx = utun_kpipe_sync_tx, |
1176 | .nxpi_sync_rx = utun_kpipe_sync_rx, |
1177 | .nxpi_tx_doorbell = NULL, |
1178 | }; |
1179 | |
1180 | result = kern_nexus_attr_create(&nxa); |
1181 | if (result) { |
1182 | printf("%s: kern_nexus_attr_create failed: %d\n" , |
1183 | __FUNCTION__, result); |
1184 | goto done; |
1185 | } |
1186 | |
1187 | uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE; |
1188 | result = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); |
1189 | VERIFY(result == 0); |
1190 | |
1191 | // Reset ring size for kernel pipe nexus to limit memory usage |
1192 | uint64_t ring_size = if_utun_ring_size; |
1193 | result = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size); |
1194 | VERIFY(result == 0); |
1195 | result = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size); |
1196 | VERIFY(result == 0); |
1197 | |
1198 | result = kern_nexus_controller_register_provider(utun_ncd, |
1199 | dom_prov, |
1200 | (const uint8_t *)"com.apple.nexus.utun.kpipe" , |
1201 | &prov_init, |
1202 | sizeof(prov_init), |
1203 | nxa, |
1204 | &utun_kpipe_uuid); |
1205 | if (result) { |
1206 | printf("%s: kern_nexus_controller_register_provider failed: %d\n" , |
1207 | __FUNCTION__, result); |
1208 | goto done; |
1209 | } |
1210 | |
1211 | done: |
1212 | if (nxa) { |
1213 | kern_nexus_attr_destroy(nxa); |
1214 | } |
1215 | |
1216 | if (result) { |
1217 | if (utun_ncd) { |
1218 | kern_nexus_controller_destroy(utun_ncd); |
1219 | utun_ncd = NULL; |
1220 | } |
1221 | utun_ncd_refcount = 0; |
1222 | } |
1223 | |
1224 | lck_mtx_unlock(&utun_lock); |
1225 | |
1226 | return result; |
1227 | } |
1228 | |
1229 | static void |
1230 | utun_unregister_kernel_pipe_nexus(void) |
1231 | { |
1232 | lck_mtx_lock(&utun_lock); |
1233 | |
1234 | VERIFY(utun_ncd_refcount > 0); |
1235 | |
1236 | if (--utun_ncd_refcount == 0) { |
1237 | kern_nexus_controller_destroy(utun_ncd); |
1238 | utun_ncd = NULL; |
1239 | } |
1240 | |
1241 | lck_mtx_unlock(&utun_lock); |
1242 | } |
1243 | |
1244 | // For use by socket option, not internally |
1245 | static errno_t |
1246 | utun_disable_channel(struct utun_pcb *pcb) |
1247 | { |
1248 | errno_t result; |
1249 | int enabled; |
1250 | uuid_t uuid; |
1251 | |
1252 | lck_rw_lock_exclusive(&pcb->utun_pcb_lock); |
1253 | |
1254 | enabled = pcb->utun_kpipe_enabled; |
1255 | uuid_copy(uuid, pcb->utun_kpipe_uuid); |
1256 | |
1257 | VERIFY(uuid_is_null(pcb->utun_kpipe_uuid) == !enabled); |
1258 | |
1259 | pcb->utun_kpipe_enabled = 0; |
1260 | uuid_clear(pcb->utun_kpipe_uuid); |
1261 | |
1262 | lck_rw_unlock_exclusive(&pcb->utun_pcb_lock); |
1263 | |
1264 | if (enabled) { |
1265 | result = kern_nexus_controller_free_provider_instance(utun_ncd, uuid); |
1266 | } else { |
1267 | result = ENXIO; |
1268 | } |
1269 | |
1270 | if (!result) { |
1271 | if (pcb->utun_kpipe_pp != NULL) { |
1272 | kern_pbufpool_destroy(pcb->utun_kpipe_pp); |
1273 | pcb->utun_kpipe_pp = NULL; |
1274 | } |
1275 | utun_unregister_kernel_pipe_nexus(); |
1276 | } |
1277 | |
1278 | return result; |
1279 | } |
1280 | |
1281 | static errno_t |
1282 | utun_enable_channel(struct utun_pcb *pcb, struct proc *proc) |
1283 | { |
1284 | struct kern_nexus_init init; |
1285 | struct kern_pbufpool_init pp_init; |
1286 | errno_t result; |
1287 | |
1288 | kauth_cred_t cred = kauth_cred_get(); |
1289 | result = priv_check_cred(cred, PRIV_SKYWALK_REGISTER_KERNEL_PIPE, 0); |
1290 | if (result) { |
1291 | return result; |
1292 | } |
1293 | |
1294 | result = utun_register_kernel_pipe_nexus(); |
1295 | if (result) { |
1296 | return result; |
1297 | } |
1298 | |
1299 | VERIFY(utun_ncd); |
1300 | |
1301 | lck_rw_lock_exclusive(&pcb->utun_pcb_lock); |
1302 | |
1303 | if (pcb->utun_kpipe_enabled) { |
1304 | result = EEXIST; // return success instead? |
1305 | goto done; |
1306 | } |
1307 | |
1308 | /* |
1309 | * Make sure we can fit packets in the channel buffers and |
1310 | * Allow an extra 4 bytes for the protocol number header in the channel |
1311 | */ |
1312 | if (pcb->utun_ifp->if_mtu + UTUN_HEADER_SIZE(pcb) > pcb->utun_slot_size) { |
1313 | result = EOPNOTSUPP; |
1314 | goto done; |
1315 | } |
1316 | |
1317 | bzero(&pp_init, sizeof (pp_init)); |
1318 | pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION; |
1319 | pp_init.kbi_packets = pcb->utun_netif_ring_size * 2; |
1320 | pp_init.kbi_bufsize = pcb->utun_slot_size; |
1321 | pp_init.kbi_buf_seg_size = UTUN_IF_DEFAULT_BUF_SEG_SIZE; |
1322 | pp_init.kbi_max_frags = 1; |
1323 | pp_init.kbi_flags |= KBIF_QUANTUM; |
1324 | (void) snprintf((char *)pp_init.kbi_name, sizeof (pp_init.kbi_name), |
1325 | "com.apple.kpipe.%s" , pcb->utun_if_xname); |
1326 | |
1327 | result = kern_pbufpool_create(&pp_init, &pp_init, &pcb->utun_kpipe_pp, |
1328 | NULL); |
1329 | if (result != 0) { |
1330 | printf("%s pbufbool create failed, error %d\n" , __func__, result); |
1331 | goto done; |
1332 | } |
1333 | |
1334 | VERIFY(uuid_is_null(pcb->utun_kpipe_uuid)); |
1335 | bzero(&init, sizeof (init)); |
1336 | init.nxi_version = KERN_NEXUS_CURRENT_VERSION; |
1337 | init.nxi_tx_pbufpool = pcb->utun_kpipe_pp; |
1338 | result = kern_nexus_controller_alloc_provider_instance(utun_ncd, |
1339 | utun_kpipe_uuid, pcb, &pcb->utun_kpipe_uuid, &init); |
1340 | if (result) { |
1341 | goto done; |
1342 | } |
1343 | |
1344 | nexus_port_t port = NEXUS_PORT_KERNEL_PIPE_CLIENT; |
1345 | result = kern_nexus_controller_bind_provider_instance(utun_ncd, |
1346 | pcb->utun_kpipe_uuid, &port, |
1347 | proc_pid(proc), NULL, NULL, 0, NEXUS_BIND_PID); |
1348 | if (result) { |
1349 | kern_nexus_controller_free_provider_instance(utun_ncd, |
1350 | pcb->utun_kpipe_uuid); |
1351 | uuid_clear(pcb->utun_kpipe_uuid); |
1352 | goto done; |
1353 | } |
1354 | |
1355 | pcb->utun_kpipe_enabled = 1; |
1356 | |
1357 | done: |
1358 | lck_rw_unlock_exclusive(&pcb->utun_pcb_lock); |
1359 | |
1360 | if (result) { |
1361 | if (pcb->utun_kpipe_pp != NULL) { |
1362 | kern_pbufpool_destroy(pcb->utun_kpipe_pp); |
1363 | pcb->utun_kpipe_pp = NULL; |
1364 | } |
1365 | utun_unregister_kernel_pipe_nexus(); |
1366 | } |
1367 | |
1368 | return result; |
1369 | } |
1370 | |
1371 | #endif // UTUN_NEXUS |
1372 | |
1373 | errno_t |
1374 | utun_register_control(void) |
1375 | { |
1376 | struct kern_ctl_reg kern_ctl; |
1377 | errno_t result = 0; |
1378 | |
1379 | /* Find a unique value for our interface family */ |
1380 | result = mbuf_tag_id_find(UTUN_CONTROL_NAME, &utun_family); |
1381 | if (result != 0) { |
1382 | printf("utun_register_control - mbuf_tag_id_find_internal failed: %d\n" , result); |
1383 | return result; |
1384 | } |
1385 | |
1386 | utun_pcb_size = sizeof(struct utun_pcb); |
1387 | utun_pcb_zone = zinit(utun_pcb_size, |
1388 | UTUN_PCB_ZONE_MAX * utun_pcb_size, |
1389 | 0, UTUN_PCB_ZONE_NAME); |
1390 | if (utun_pcb_zone == NULL) { |
1391 | printf("utun_register_control - zinit(utun_pcb) failed" ); |
1392 | return ENOMEM; |
1393 | } |
1394 | |
1395 | #if UTUN_NEXUS |
1396 | utun_register_nexus(); |
1397 | #endif // UTUN_NEXUS |
1398 | |
1399 | TAILQ_INIT(&utun_head); |
1400 | |
1401 | bzero(&kern_ctl, sizeof(kern_ctl)); |
1402 | strlcpy(kern_ctl.ctl_name, UTUN_CONTROL_NAME, sizeof(kern_ctl.ctl_name)); |
1403 | kern_ctl.ctl_name[sizeof(kern_ctl.ctl_name) - 1] = 0; |
1404 | kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED; /* Require root */ |
1405 | kern_ctl.ctl_sendsize = 512 * 1024; |
1406 | kern_ctl.ctl_recvsize = 512 * 1024; |
1407 | kern_ctl.ctl_bind = utun_ctl_bind; |
1408 | kern_ctl.ctl_connect = utun_ctl_connect; |
1409 | kern_ctl.ctl_disconnect = utun_ctl_disconnect; |
1410 | kern_ctl.ctl_send = utun_ctl_send; |
1411 | kern_ctl.ctl_setopt = utun_ctl_setopt; |
1412 | kern_ctl.ctl_getopt = utun_ctl_getopt; |
1413 | kern_ctl.ctl_rcvd = utun_ctl_rcvd; |
1414 | |
1415 | result = ctl_register(&kern_ctl, &utun_kctlref); |
1416 | if (result != 0) { |
1417 | printf("utun_register_control - ctl_register failed: %d\n" , result); |
1418 | return result; |
1419 | } |
1420 | |
1421 | /* Register the protocol plumbers */ |
1422 | if ((result = proto_register_plumber(PF_INET, utun_family, |
1423 | utun_attach_proto, NULL)) != 0) { |
1424 | printf("utun_register_control - proto_register_plumber(PF_INET, %d) failed: %d\n" , |
1425 | utun_family, result); |
1426 | ctl_deregister(utun_kctlref); |
1427 | return result; |
1428 | } |
1429 | |
1430 | /* Register the protocol plumbers */ |
1431 | if ((result = proto_register_plumber(PF_INET6, utun_family, |
1432 | utun_attach_proto, NULL)) != 0) { |
1433 | proto_unregister_plumber(PF_INET, utun_family); |
1434 | ctl_deregister(utun_kctlref); |
1435 | printf("utun_register_control - proto_register_plumber(PF_INET6, %d) failed: %d\n" , |
1436 | utun_family, result); |
1437 | return result; |
1438 | } |
1439 | |
1440 | utun_lck_attr = lck_attr_alloc_init(); |
1441 | utun_lck_grp_attr = lck_grp_attr_alloc_init(); |
1442 | utun_lck_grp = lck_grp_alloc_init("utun" , utun_lck_grp_attr); |
1443 | |
1444 | lck_mtx_init(&utun_lock, utun_lck_grp, utun_lck_attr); |
1445 | |
1446 | return 0; |
1447 | } |
1448 | |
1449 | /* Kernel control functions */ |
1450 | |
1451 | static inline void |
1452 | utun_free_pcb(struct utun_pcb *pcb, bool in_list) |
1453 | { |
1454 | #ifdef UTUN_NEXUS |
1455 | mbuf_freem_list(pcb->utun_input_chain); |
1456 | lck_mtx_destroy(&pcb->utun_input_chain_lock, utun_lck_grp); |
1457 | #endif // UTUN_NEXUS |
1458 | lck_rw_destroy(&pcb->utun_pcb_lock, utun_lck_grp); |
1459 | if (in_list) { |
1460 | lck_mtx_lock(&utun_lock); |
1461 | TAILQ_REMOVE(&utun_head, pcb, utun_chain); |
1462 | lck_mtx_unlock(&utun_lock); |
1463 | } |
1464 | zfree(utun_pcb_zone, pcb); |
1465 | } |
1466 | |
1467 | static errno_t |
1468 | utun_ctl_bind(kern_ctl_ref kctlref, |
1469 | struct sockaddr_ctl *sac, |
1470 | void **unitinfo) |
1471 | { |
1472 | struct utun_pcb *pcb = zalloc(utun_pcb_zone); |
1473 | memset(pcb, 0, sizeof(*pcb)); |
1474 | |
1475 | *unitinfo = pcb; |
1476 | pcb->utun_ctlref = kctlref; |
1477 | pcb->utun_unit = sac->sc_unit; |
1478 | pcb->utun_max_pending_packets = 1; |
1479 | |
1480 | #if UTUN_NEXUS |
1481 | pcb->utun_use_netif = false; |
1482 | pcb->utun_slot_size = UTUN_IF_DEFAULT_SLOT_SIZE; |
1483 | pcb->utun_netif_ring_size = UTUN_IF_DEFAULT_RING_SIZE; |
1484 | pcb->utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE; |
1485 | pcb->utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE; |
1486 | #endif // UTUN_NEXUS |
1487 | |
1488 | lck_mtx_init(&pcb->utun_input_chain_lock, utun_lck_grp, utun_lck_attr); |
1489 | lck_rw_init(&pcb->utun_pcb_lock, utun_lck_grp, utun_lck_attr); |
1490 | |
1491 | return (0); |
1492 | } |
1493 | |
1494 | static errno_t |
1495 | utun_ctl_connect(kern_ctl_ref kctlref, |
1496 | struct sockaddr_ctl *sac, |
1497 | void **unitinfo) |
1498 | { |
1499 | struct ifnet_init_eparams utun_init = {}; |
1500 | errno_t result = 0; |
1501 | |
1502 | if (*unitinfo == NULL) { |
1503 | (void)utun_ctl_bind(kctlref, sac, unitinfo); |
1504 | } |
1505 | |
1506 | struct utun_pcb *pcb = *unitinfo; |
1507 | |
1508 | lck_mtx_lock(&utun_lock); |
1509 | |
1510 | /* Find some open interface id */ |
1511 | u_int32_t chosen_unique_id = 1; |
1512 | struct utun_pcb *next_pcb = TAILQ_LAST(&utun_head, utun_list); |
1513 | if (next_pcb != NULL) { |
1514 | /* List was not empty, add one to the last item */ |
1515 | chosen_unique_id = next_pcb->utun_unique_id + 1; |
1516 | next_pcb = NULL; |
1517 | |
1518 | /* |
1519 | * If this wrapped the id number, start looking at |
1520 | * the front of the list for an unused id. |
1521 | */ |
1522 | if (chosen_unique_id == 0) { |
1523 | /* Find the next unused ID */ |
1524 | chosen_unique_id = 1; |
1525 | TAILQ_FOREACH(next_pcb, &utun_head, utun_chain) { |
1526 | if (next_pcb->utun_unique_id > chosen_unique_id) { |
1527 | /* We found a gap */ |
1528 | break; |
1529 | } |
1530 | |
1531 | chosen_unique_id = next_pcb->utun_unique_id + 1; |
1532 | } |
1533 | } |
1534 | } |
1535 | |
1536 | pcb->utun_unique_id = chosen_unique_id; |
1537 | |
1538 | if (next_pcb != NULL) { |
1539 | TAILQ_INSERT_BEFORE(next_pcb, pcb, utun_chain); |
1540 | } else { |
1541 | TAILQ_INSERT_TAIL(&utun_head, pcb, utun_chain); |
1542 | } |
1543 | lck_mtx_unlock(&utun_lock); |
1544 | |
1545 | snprintf(pcb->utun_if_xname, sizeof(pcb->utun_if_xname), "utun%d" , pcb->utun_unit - 1); |
1546 | snprintf(pcb->utun_unique_name, sizeof(pcb->utun_unique_name), "utunid%d" , pcb->utun_unique_id - 1); |
1547 | printf("utun_ctl_connect: creating interface %s (id %s)\n" , pcb->utun_if_xname, pcb->utun_unique_name); |
1548 | |
1549 | /* Create the interface */ |
1550 | bzero(&utun_init, sizeof(utun_init)); |
1551 | utun_init.ver = IFNET_INIT_CURRENT_VERSION; |
1552 | utun_init.len = sizeof (utun_init); |
1553 | |
1554 | #if UTUN_NEXUS |
1555 | if (pcb->utun_use_netif) { |
1556 | utun_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO); |
1557 | utun_init.tx_headroom = UTUN_IF_HEADROOM_SIZE; |
1558 | } else |
1559 | #endif // UTUN_NEXUS |
1560 | { |
1561 | utun_init.flags = IFNET_INIT_NX_NOAUTO; |
1562 | utun_init.start = utun_start; |
1563 | utun_init.framer_extended = utun_framer; |
1564 | } |
1565 | utun_init.name = "utun" ; |
1566 | utun_init.unit = pcb->utun_unit - 1; |
1567 | utun_init.uniqueid = pcb->utun_unique_name; |
1568 | utun_init.uniqueid_len = strlen(pcb->utun_unique_name); |
1569 | utun_init.family = utun_family; |
1570 | utun_init.subfamily = IFNET_SUBFAMILY_UTUN; |
1571 | utun_init.type = IFT_OTHER; |
1572 | utun_init.demux = utun_demux; |
1573 | utun_init.add_proto = utun_add_proto; |
1574 | utun_init.del_proto = utun_del_proto; |
1575 | utun_init.softc = pcb; |
1576 | utun_init.ioctl = utun_ioctl; |
1577 | utun_init.detach = utun_detached; |
1578 | |
1579 | #if UTUN_NEXUS |
1580 | if (pcb->utun_use_netif) { |
1581 | result = utun_nexus_ifattach(pcb, &utun_init, &pcb->utun_ifp); |
1582 | if (result != 0) { |
1583 | printf("utun_ctl_connect - utun_nexus_ifattach failed: %d\n" , result); |
1584 | utun_free_pcb(pcb, true); |
1585 | *unitinfo = NULL; |
1586 | return result; |
1587 | } |
1588 | |
1589 | result = utun_multistack_attach(pcb); |
1590 | if (result != 0) { |
1591 | printf("utun_ctl_connect - utun_multistack_attach failed: %d\n" , result); |
1592 | *unitinfo = NULL; |
1593 | return result; |
1594 | } |
1595 | |
1596 | /* Attach to bpf */ |
1597 | bpfattach(pcb->utun_ifp, DLT_RAW, 0); |
1598 | } else |
1599 | #endif // UTUN_NEXUS |
1600 | { |
1601 | /* |
1602 | * Upon success, this holds an ifnet reference which we will |
1603 | * release via ifnet_release() at final detach time. |
1604 | */ |
1605 | result = ifnet_allocate_extended(&utun_init, &pcb->utun_ifp); |
1606 | if (result != 0) { |
1607 | printf("utun_ctl_connect - ifnet_allocate failed: %d\n" , result); |
1608 | utun_free_pcb(pcb, true); |
1609 | *unitinfo = NULL; |
1610 | return result; |
1611 | } |
1612 | |
1613 | /* Set flags and additional information. */ |
1614 | ifnet_set_mtu(pcb->utun_ifp, UTUN_DEFAULT_MTU); |
1615 | ifnet_set_flags(pcb->utun_ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff); |
1616 | |
1617 | /* The interface must generate its own IPv6 LinkLocal address, |
1618 | * if possible following the recommendation of RFC2472 to the 64bit interface ID |
1619 | */ |
1620 | ifnet_set_eflags(pcb->utun_ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL); |
1621 | |
1622 | /* Reset the stats in case as the interface may have been recycled */ |
1623 | struct ifnet_stats_param stats; |
1624 | bzero(&stats, sizeof(struct ifnet_stats_param)); |
1625 | ifnet_set_stat(pcb->utun_ifp, &stats); |
1626 | |
1627 | /* Attach the interface */ |
1628 | result = ifnet_attach(pcb->utun_ifp, NULL); |
1629 | if (result != 0) { |
1630 | printf("utun_ctl_connect - ifnet_attach failed: %d\n" , result); |
1631 | /* Release reference now since attach failed */ |
1632 | ifnet_release(pcb->utun_ifp); |
1633 | utun_free_pcb(pcb, true); |
1634 | *unitinfo = NULL; |
1635 | return (result); |
1636 | } |
1637 | |
1638 | /* Attach to bpf */ |
1639 | bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb)); |
1640 | } |
1641 | |
1642 | /* The interfaces resoures allocated, mark it as running */ |
1643 | ifnet_set_flags(pcb->utun_ifp, IFF_RUNNING, IFF_RUNNING); |
1644 | |
1645 | return result; |
1646 | } |
1647 | |
1648 | static errno_t |
1649 | utun_detach_ip(ifnet_t interface, |
1650 | protocol_family_t protocol, |
1651 | socket_t pf_socket) |
1652 | { |
1653 | errno_t result = EPROTONOSUPPORT; |
1654 | |
1655 | /* Attempt a detach */ |
1656 | if (protocol == PF_INET) { |
1657 | struct ifreq ifr; |
1658 | |
1659 | bzero(&ifr, sizeof(ifr)); |
1660 | snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d" , |
1661 | ifnet_name(interface), ifnet_unit(interface)); |
1662 | |
1663 | result = sock_ioctl(pf_socket, SIOCPROTODETACH, &ifr); |
1664 | } else if (protocol == PF_INET6) { |
1665 | struct in6_ifreq ifr6; |
1666 | |
1667 | bzero(&ifr6, sizeof(ifr6)); |
1668 | snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d" , |
1669 | ifnet_name(interface), ifnet_unit(interface)); |
1670 | |
1671 | result = sock_ioctl(pf_socket, SIOCPROTODETACH_IN6, &ifr6); |
1672 | } |
1673 | |
1674 | return result; |
1675 | } |
1676 | |
1677 | static void |
1678 | utun_remove_address(ifnet_t interface, |
1679 | protocol_family_t protocol, |
1680 | ifaddr_t address, |
1681 | socket_t pf_socket) |
1682 | { |
1683 | errno_t result = 0; |
1684 | |
1685 | /* Attempt a detach */ |
1686 | if (protocol == PF_INET) { |
1687 | struct ifreq ifr; |
1688 | |
1689 | bzero(&ifr, sizeof(ifr)); |
1690 | snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d" , |
1691 | ifnet_name(interface), ifnet_unit(interface)); |
1692 | result = ifaddr_address(address, &ifr.ifr_addr, sizeof(ifr.ifr_addr)); |
1693 | if (result != 0) { |
1694 | printf("utun_remove_address - ifaddr_address failed: %d" , result); |
1695 | } else { |
1696 | result = sock_ioctl(pf_socket, SIOCDIFADDR, &ifr); |
1697 | if (result != 0) { |
1698 | printf("utun_remove_address - SIOCDIFADDR failed: %d" , result); |
1699 | } |
1700 | } |
1701 | } else if (protocol == PF_INET6) { |
1702 | struct in6_ifreq ifr6; |
1703 | |
1704 | bzero(&ifr6, sizeof(ifr6)); |
1705 | snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d" , |
1706 | ifnet_name(interface), ifnet_unit(interface)); |
1707 | result = ifaddr_address(address, (struct sockaddr*)&ifr6.ifr_addr, |
1708 | sizeof(ifr6.ifr_addr)); |
1709 | if (result != 0) { |
1710 | printf("utun_remove_address - ifaddr_address failed (v6): %d" , |
1711 | result); |
1712 | } else { |
1713 | result = sock_ioctl(pf_socket, SIOCDIFADDR_IN6, &ifr6); |
1714 | if (result != 0) { |
1715 | printf("utun_remove_address - SIOCDIFADDR_IN6 failed: %d" , |
1716 | result); |
1717 | } |
1718 | } |
1719 | } |
1720 | } |
1721 | |
1722 | static void |
1723 | utun_cleanup_family(ifnet_t interface, |
1724 | protocol_family_t protocol) |
1725 | { |
1726 | errno_t result = 0; |
1727 | socket_t pf_socket = NULL; |
1728 | ifaddr_t *addresses = NULL; |
1729 | int i; |
1730 | |
1731 | if (protocol != PF_INET && protocol != PF_INET6) { |
1732 | printf("utun_cleanup_family - invalid protocol family %d\n" , protocol); |
1733 | return; |
1734 | } |
1735 | |
1736 | /* Create a socket for removing addresses and detaching the protocol */ |
1737 | result = sock_socket(protocol, SOCK_DGRAM, 0, NULL, NULL, &pf_socket); |
1738 | if (result != 0) { |
1739 | if (result != EAFNOSUPPORT) |
1740 | printf("utun_cleanup_family - failed to create %s socket: %d\n" , |
1741 | protocol == PF_INET ? "IP" : "IPv6" , result); |
1742 | goto cleanup; |
1743 | } |
1744 | |
1745 | /* always set SS_PRIV, we want to close and detach regardless */ |
1746 | sock_setpriv(pf_socket, 1); |
1747 | |
1748 | result = utun_detach_ip(interface, protocol, pf_socket); |
1749 | if (result == 0 || result == ENXIO) { |
1750 | /* We are done! We either detached or weren't attached. */ |
1751 | goto cleanup; |
1752 | } else if (result != EBUSY) { |
1753 | /* Uh, not really sure what happened here... */ |
1754 | printf("utun_cleanup_family - utun_detach_ip failed: %d\n" , result); |
1755 | goto cleanup; |
1756 | } |
1757 | |
1758 | /* |
1759 | * At this point, we received an EBUSY error. This means there are |
1760 | * addresses attached. We should detach them and then try again. |
1761 | */ |
1762 | result = ifnet_get_address_list_family(interface, &addresses, protocol); |
1763 | if (result != 0) { |
1764 | printf("fnet_get_address_list_family(%s%d, 0xblah, %s) - failed: %d\n" , |
1765 | ifnet_name(interface), ifnet_unit(interface), |
1766 | protocol == PF_INET ? "PF_INET" : "PF_INET6" , result); |
1767 | goto cleanup; |
1768 | } |
1769 | |
1770 | for (i = 0; addresses[i] != 0; i++) { |
1771 | utun_remove_address(interface, protocol, addresses[i], pf_socket); |
1772 | } |
1773 | ifnet_free_address_list(addresses); |
1774 | addresses = NULL; |
1775 | |
1776 | /* |
1777 | * The addresses should be gone, we should try the remove again. |
1778 | */ |
1779 | result = utun_detach_ip(interface, protocol, pf_socket); |
1780 | if (result != 0 && result != ENXIO) { |
1781 | printf("utun_cleanup_family - utun_detach_ip failed: %d\n" , result); |
1782 | } |
1783 | |
1784 | cleanup: |
1785 | if (pf_socket != NULL) { |
1786 | sock_close(pf_socket); |
1787 | } |
1788 | |
1789 | if (addresses != NULL) { |
1790 | ifnet_free_address_list(addresses); |
1791 | } |
1792 | } |
1793 | |
1794 | static errno_t |
1795 | utun_ctl_disconnect(__unused kern_ctl_ref kctlref, |
1796 | __unused u_int32_t unit, |
1797 | void *unitinfo) |
1798 | { |
1799 | struct utun_pcb *pcb = unitinfo; |
1800 | ifnet_t ifp = NULL; |
1801 | errno_t result = 0; |
1802 | |
1803 | if (pcb == NULL) { |
1804 | return EINVAL; |
1805 | } |
1806 | |
1807 | #if UTUN_NEXUS |
1808 | // Tell the nexus to stop all rings |
1809 | if (pcb->utun_netif_nexus != NULL) { |
1810 | kern_nexus_stop(pcb->utun_netif_nexus); |
1811 | } |
1812 | #endif // UTUN_NEXUS |
1813 | |
1814 | lck_rw_lock_exclusive(&pcb->utun_pcb_lock); |
1815 | |
1816 | #if UTUN_NEXUS |
1817 | uuid_t kpipe_uuid; |
1818 | uuid_copy(kpipe_uuid, pcb->utun_kpipe_uuid); |
1819 | uuid_clear(pcb->utun_kpipe_uuid); |
1820 | pcb->utun_kpipe_enabled = FALSE; |
1821 | #endif // UTUN_NEXUS |
1822 | |
1823 | pcb->utun_ctlref = NULL; |
1824 | |
1825 | ifp = pcb->utun_ifp; |
1826 | if (ifp != NULL) { |
1827 | #if UTUN_NEXUS |
1828 | // Tell the nexus to stop all rings |
1829 | if (pcb->utun_netif_nexus != NULL) { |
1830 | /* |
1831 | * Quiesce the interface and flush any pending outbound packets. |
1832 | */ |
1833 | if_down(ifp); |
1834 | |
1835 | /* Increment refcnt, but detach interface */ |
1836 | ifnet_incr_iorefcnt(ifp); |
1837 | if ((result = ifnet_detach(ifp)) != 0) { |
1838 | panic("utun_ctl_disconnect - ifnet_detach failed: %d\n" , result); |
1839 | } |
1840 | |
1841 | /* |
1842 | * We want to do everything in our power to ensure that the interface |
1843 | * really goes away when the socket is closed. We must remove IP/IPv6 |
1844 | * addresses and detach the protocols. Finally, we can remove and |
1845 | * release the interface. |
1846 | */ |
1847 | utun_cleanup_family(ifp, AF_INET); |
1848 | utun_cleanup_family(ifp, AF_INET6); |
1849 | |
1850 | lck_rw_unlock_exclusive(&pcb->utun_pcb_lock); |
1851 | |
1852 | if (!uuid_is_null(kpipe_uuid)) { |
1853 | if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) { |
1854 | if (pcb->utun_kpipe_pp != NULL) { |
1855 | kern_pbufpool_destroy(pcb->utun_kpipe_pp); |
1856 | pcb->utun_kpipe_pp = NULL; |
1857 | } |
1858 | utun_unregister_kernel_pipe_nexus(); |
1859 | } |
1860 | } |
1861 | utun_nexus_detach(pcb); |
1862 | |
1863 | /* Decrement refcnt to finish detaching and freeing */ |
1864 | ifnet_decr_iorefcnt(ifp); |
1865 | } else |
1866 | #endif // UTUN_NEXUS |
1867 | { |
1868 | lck_rw_unlock_exclusive(&pcb->utun_pcb_lock); |
1869 | |
1870 | #if UTUN_NEXUS |
1871 | if (!uuid_is_null(kpipe_uuid)) { |
1872 | if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) { |
1873 | if (pcb->utun_kpipe_pp != NULL) { |
1874 | kern_pbufpool_destroy(pcb->utun_kpipe_pp); |
1875 | pcb->utun_kpipe_pp = NULL; |
1876 | } |
1877 | utun_unregister_kernel_pipe_nexus(); |
1878 | } |
1879 | } |
1880 | #endif // UTUN_NEXUS |
1881 | |
1882 | /* |
1883 | * We want to do everything in our power to ensure that the interface |
1884 | * really goes away when the socket is closed. We must remove IP/IPv6 |
1885 | * addresses and detach the protocols. Finally, we can remove and |
1886 | * release the interface. |
1887 | */ |
1888 | utun_cleanup_family(ifp, AF_INET); |
1889 | utun_cleanup_family(ifp, AF_INET6); |
1890 | |
1891 | /* |
1892 | * Detach now; utun_detach() will be called asynchronously once |
1893 | * the I/O reference count drops to 0. There we will invoke |
1894 | * ifnet_release(). |
1895 | */ |
1896 | if ((result = ifnet_detach(ifp)) != 0) { |
1897 | printf("utun_ctl_disconnect - ifnet_detach failed: %d\n" , result); |
1898 | } |
1899 | } |
1900 | } else { |
1901 | // Bound, but not connected |
1902 | lck_rw_unlock_exclusive(&pcb->utun_pcb_lock); |
1903 | utun_free_pcb(pcb, false); |
1904 | } |
1905 | |
1906 | return 0; |
1907 | } |
1908 | |
1909 | static errno_t |
1910 | utun_ctl_send(__unused kern_ctl_ref kctlref, |
1911 | __unused u_int32_t unit, |
1912 | void *unitinfo, |
1913 | mbuf_t m, |
1914 | __unused int flags) |
1915 | { |
1916 | /* |
1917 | * The userland ABI requires the first four bytes have the protocol family |
1918 | * in network byte order: swap them |
1919 | */ |
1920 | if (m_pktlen(m) >= (int32_t)UTUN_HEADER_SIZE((struct utun_pcb *)unitinfo)) { |
1921 | *(protocol_family_t *)mbuf_data(m) = ntohl(*(protocol_family_t *)mbuf_data(m)); |
1922 | } else { |
1923 | printf("%s - unexpected short mbuf pkt len %d\n" , __func__, m_pktlen(m) ); |
1924 | } |
1925 | |
1926 | return utun_pkt_input((struct utun_pcb *)unitinfo, m); |
1927 | } |
1928 | |
1929 | static errno_t |
1930 | utun_ctl_setopt(__unused kern_ctl_ref kctlref, |
1931 | __unused u_int32_t unit, |
1932 | void *unitinfo, |
1933 | int opt, |
1934 | void *data, |
1935 | size_t len) |
1936 | { |
1937 | struct utun_pcb *pcb = unitinfo; |
1938 | errno_t result = 0; |
1939 | /* check for privileges for privileged options */ |
1940 | switch (opt) { |
1941 | case UTUN_OPT_FLAGS: |
1942 | case UTUN_OPT_EXT_IFDATA_STATS: |
1943 | case UTUN_OPT_SET_DELEGATE_INTERFACE: |
1944 | if (kauth_cred_issuser(kauth_cred_get()) == 0) { |
1945 | return EPERM; |
1946 | } |
1947 | break; |
1948 | } |
1949 | |
1950 | switch (opt) { |
1951 | case UTUN_OPT_FLAGS: |
1952 | if (len != sizeof(u_int32_t)) { |
1953 | result = EMSGSIZE; |
1954 | } else { |
1955 | if (pcb->utun_ifp == NULL) { |
1956 | // Only can set after connecting |
1957 | result = EINVAL; |
1958 | break; |
1959 | } |
1960 | #if UTUN_NEXUS |
1961 | if (pcb->utun_use_netif) { |
1962 | pcb->utun_flags = *(u_int32_t *)data; |
1963 | } else |
1964 | #endif // UTUN_NEXUS |
1965 | { |
1966 | u_int32_t old_flags = pcb->utun_flags; |
1967 | pcb->utun_flags = *(u_int32_t *)data; |
1968 | if (((old_flags ^ pcb->utun_flags) & UTUN_FLAGS_ENABLE_PROC_UUID)) { |
1969 | // If UTUN_FLAGS_ENABLE_PROC_UUID flag changed, update bpf |
1970 | bpfdetach(pcb->utun_ifp); |
1971 | bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb)); |
1972 | } |
1973 | } |
1974 | } |
1975 | break; |
1976 | |
1977 | case UTUN_OPT_EXT_IFDATA_STATS: |
1978 | if (len != sizeof(int)) { |
1979 | result = EMSGSIZE; |
1980 | break; |
1981 | } |
1982 | if (pcb->utun_ifp == NULL) { |
1983 | // Only can set after connecting |
1984 | result = EINVAL; |
1985 | break; |
1986 | } |
1987 | pcb->utun_ext_ifdata_stats = (*(int *)data) ? 1 : 0; |
1988 | break; |
1989 | |
1990 | case UTUN_OPT_INC_IFDATA_STATS_IN: |
1991 | case UTUN_OPT_INC_IFDATA_STATS_OUT: { |
1992 | struct utun_stats_param *utsp = (struct utun_stats_param *)data; |
1993 | |
1994 | if (utsp == NULL || len < sizeof(struct utun_stats_param)) { |
1995 | result = EINVAL; |
1996 | break; |
1997 | } |
1998 | if (pcb->utun_ifp == NULL) { |
1999 | // Only can set after connecting |
2000 | result = EINVAL; |
2001 | break; |
2002 | } |
2003 | if (!pcb->utun_ext_ifdata_stats) { |
2004 | result = EINVAL; |
2005 | break; |
2006 | } |
2007 | if (opt == UTUN_OPT_INC_IFDATA_STATS_IN) |
2008 | ifnet_stat_increment_in(pcb->utun_ifp, utsp->utsp_packets, |
2009 | utsp->utsp_bytes, utsp->utsp_errors); |
2010 | else |
2011 | ifnet_stat_increment_out(pcb->utun_ifp, utsp->utsp_packets, |
2012 | utsp->utsp_bytes, utsp->utsp_errors); |
2013 | break; |
2014 | } |
2015 | case UTUN_OPT_SET_DELEGATE_INTERFACE: { |
2016 | ifnet_t del_ifp = NULL; |
2017 | char name[IFNAMSIZ]; |
2018 | |
2019 | if (len > IFNAMSIZ - 1) { |
2020 | result = EMSGSIZE; |
2021 | break; |
2022 | } |
2023 | if (pcb->utun_ifp == NULL) { |
2024 | // Only can set after connecting |
2025 | result = EINVAL; |
2026 | break; |
2027 | } |
2028 | if (len != 0) { /* if len==0, del_ifp will be NULL causing the delegate to be removed */ |
2029 | bcopy(data, name, len); |
2030 | name[len] = 0; |
2031 | result = ifnet_find_by_name(name, &del_ifp); |
2032 | } |
2033 | if (result == 0) { |
2034 | result = ifnet_set_delegate(pcb->utun_ifp, del_ifp); |
2035 | if (del_ifp) |
2036 | ifnet_release(del_ifp); |
2037 | } |
2038 | break; |
2039 | } |
2040 | case UTUN_OPT_MAX_PENDING_PACKETS: { |
2041 | u_int32_t max_pending_packets = 0; |
2042 | if (len != sizeof(u_int32_t)) { |
2043 | result = EMSGSIZE; |
2044 | break; |
2045 | } |
2046 | max_pending_packets = *(u_int32_t *)data; |
2047 | if (max_pending_packets == 0) { |
2048 | result = EINVAL; |
2049 | break; |
2050 | } |
2051 | pcb->utun_max_pending_packets = max_pending_packets; |
2052 | break; |
2053 | } |
2054 | #if UTUN_NEXUS |
2055 | case UTUN_OPT_ENABLE_CHANNEL: { |
2056 | if (len != sizeof(int)) { |
2057 | result = EMSGSIZE; |
2058 | break; |
2059 | } |
2060 | if (pcb->utun_ifp == NULL) { |
2061 | // Only can set after connecting |
2062 | result = EINVAL; |
2063 | break; |
2064 | } |
2065 | if (*(int *)data) { |
2066 | result = utun_enable_channel(pcb, current_proc()); |
2067 | } else { |
2068 | result = utun_disable_channel(pcb); |
2069 | } |
2070 | break; |
2071 | } |
2072 | case UTUN_OPT_ENABLE_FLOWSWITCH: { |
2073 | if (len != sizeof(int)) { |
2074 | result = EMSGSIZE; |
2075 | break; |
2076 | } |
2077 | if (pcb->utun_ifp == NULL) { |
2078 | // Only can set after connecting |
2079 | result = EINVAL; |
2080 | break; |
2081 | } |
2082 | if (!if_is_netagent_enabled()) { |
2083 | result = ENOTSUP; |
2084 | break; |
2085 | } |
2086 | if (uuid_is_null(pcb->utun_nx.ms_agent)) { |
2087 | result = ENOENT; |
2088 | break; |
2089 | } |
2090 | |
2091 | if (*(int *)data) { |
2092 | if_add_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent); |
2093 | pcb->utun_needs_netagent = true; |
2094 | } else { |
2095 | pcb->utun_needs_netagent = false; |
2096 | if_delete_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent); |
2097 | } |
2098 | break; |
2099 | } |
2100 | case UTUN_OPT_ENABLE_NETIF: { |
2101 | if (len != sizeof(int)) { |
2102 | result = EMSGSIZE; |
2103 | break; |
2104 | } |
2105 | if (pcb->utun_ifp != NULL) { |
2106 | // Only can set before connecting |
2107 | result = EINVAL; |
2108 | break; |
2109 | } |
2110 | lck_rw_lock_exclusive(&pcb->utun_pcb_lock); |
2111 | pcb->utun_use_netif = !!(*(int *)data); |
2112 | lck_rw_unlock_exclusive(&pcb->utun_pcb_lock); |
2113 | break; |
2114 | } |
2115 | case UTUN_OPT_SLOT_SIZE: { |
2116 | if (len != sizeof(u_int32_t)) { |
2117 | result = EMSGSIZE; |
2118 | break; |
2119 | } |
2120 | if (pcb->utun_ifp != NULL) { |
2121 | // Only can set before connecting |
2122 | result = EINVAL; |
2123 | break; |
2124 | } |
2125 | u_int32_t slot_size = *(u_int32_t *)data; |
2126 | if (slot_size < UTUN_IF_MIN_SLOT_SIZE || |
2127 | slot_size > UTUN_IF_MAX_SLOT_SIZE) { |
2128 | return (EINVAL); |
2129 | } |
2130 | pcb->utun_slot_size = slot_size; |
2131 | break; |
2132 | } |
2133 | case UTUN_OPT_NETIF_RING_SIZE: { |
2134 | if (len != sizeof(u_int32_t)) { |
2135 | result = EMSGSIZE; |
2136 | break; |
2137 | } |
2138 | if (pcb->utun_ifp != NULL) { |
2139 | // Only can set before connecting |
2140 | result = EINVAL; |
2141 | break; |
2142 | } |
2143 | u_int32_t ring_size = *(u_int32_t *)data; |
2144 | if (ring_size < UTUN_IF_MIN_RING_SIZE || |
2145 | ring_size > UTUN_IF_MAX_RING_SIZE) { |
2146 | return (EINVAL); |
2147 | } |
2148 | pcb->utun_netif_ring_size = ring_size; |
2149 | break; |
2150 | } |
2151 | case UTUN_OPT_TX_FSW_RING_SIZE: { |
2152 | if (len != sizeof(u_int32_t)) { |
2153 | result = EMSGSIZE; |
2154 | break; |
2155 | } |
2156 | if (pcb->utun_ifp != NULL) { |
2157 | // Only can set before connecting |
2158 | result = EINVAL; |
2159 | break; |
2160 | } |
2161 | u_int32_t ring_size = *(u_int32_t *)data; |
2162 | if (ring_size < UTUN_IF_MIN_RING_SIZE || |
2163 | ring_size > UTUN_IF_MAX_RING_SIZE) { |
2164 | return (EINVAL); |
2165 | } |
2166 | pcb->utun_tx_fsw_ring_size = ring_size; |
2167 | break; |
2168 | } |
2169 | case UTUN_OPT_RX_FSW_RING_SIZE: { |
2170 | if (len != sizeof(u_int32_t)) { |
2171 | result = EMSGSIZE; |
2172 | break; |
2173 | } |
2174 | if (pcb->utun_ifp != NULL) { |
2175 | // Only can set before connecting |
2176 | result = EINVAL; |
2177 | break; |
2178 | } |
2179 | u_int32_t ring_size = *(u_int32_t *)data; |
2180 | if (ring_size < UTUN_IF_MIN_RING_SIZE || |
2181 | ring_size > UTUN_IF_MAX_RING_SIZE) { |
2182 | return (EINVAL); |
2183 | } |
2184 | pcb->utun_rx_fsw_ring_size = ring_size; |
2185 | break; |
2186 | } |
2187 | #endif // UTUN_NEXUS |
2188 | default: { |
2189 | result = ENOPROTOOPT; |
2190 | break; |
2191 | } |
2192 | } |
2193 | |
2194 | return result; |
2195 | } |
2196 | |
2197 | static errno_t |
2198 | utun_ctl_getopt(__unused kern_ctl_ref kctlref, |
2199 | __unused u_int32_t unit, |
2200 | void *unitinfo, |
2201 | int opt, |
2202 | void *data, |
2203 | size_t *len) |
2204 | { |
2205 | struct utun_pcb *pcb = unitinfo; |
2206 | errno_t result = 0; |
2207 | |
2208 | switch (opt) { |
2209 | case UTUN_OPT_FLAGS: |
2210 | if (*len != sizeof(u_int32_t)) { |
2211 | result = EMSGSIZE; |
2212 | } else { |
2213 | *(u_int32_t *)data = pcb->utun_flags; |
2214 | } |
2215 | break; |
2216 | |
2217 | case UTUN_OPT_EXT_IFDATA_STATS: |
2218 | if (*len != sizeof(int)) { |
2219 | result = EMSGSIZE; |
2220 | } else { |
2221 | *(int *)data = (pcb->utun_ext_ifdata_stats) ? 1 : 0; |
2222 | } |
2223 | break; |
2224 | |
2225 | case UTUN_OPT_IFNAME: |
2226 | if (*len < MIN(strlen(pcb->utun_if_xname) + 1, sizeof(pcb->utun_if_xname))) { |
2227 | result = EMSGSIZE; |
2228 | } else { |
2229 | if (pcb->utun_ifp == NULL) { |
2230 | // Only can get after connecting |
2231 | result = EINVAL; |
2232 | break; |
2233 | } |
2234 | *len = snprintf(data, *len, "%s" , pcb->utun_if_xname) + 1; |
2235 | } |
2236 | break; |
2237 | |
2238 | case UTUN_OPT_MAX_PENDING_PACKETS: { |
2239 | if (*len != sizeof(u_int32_t)) { |
2240 | result = EMSGSIZE; |
2241 | } else { |
2242 | *((u_int32_t *)data) = pcb->utun_max_pending_packets; |
2243 | } |
2244 | break; |
2245 | } |
2246 | |
2247 | #if UTUN_NEXUS |
2248 | case UTUN_OPT_ENABLE_CHANNEL: { |
2249 | if (*len != sizeof(int)) { |
2250 | result = EMSGSIZE; |
2251 | } else { |
2252 | lck_rw_lock_shared(&pcb->utun_pcb_lock); |
2253 | *(int *)data = pcb->utun_kpipe_enabled; |
2254 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
2255 | } |
2256 | break; |
2257 | } |
2258 | |
2259 | case UTUN_OPT_ENABLE_FLOWSWITCH: { |
2260 | if (*len != sizeof(int)) { |
2261 | result = EMSGSIZE; |
2262 | } else { |
2263 | *(int *)data = if_check_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent); |
2264 | } |
2265 | break; |
2266 | } |
2267 | |
2268 | case UTUN_OPT_ENABLE_NETIF: { |
2269 | if (*len != sizeof(int)) { |
2270 | result = EMSGSIZE; |
2271 | } else { |
2272 | lck_rw_lock_shared(&pcb->utun_pcb_lock); |
2273 | *(int *)data = !!pcb->utun_use_netif; |
2274 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
2275 | } |
2276 | break; |
2277 | } |
2278 | |
2279 | case UTUN_OPT_GET_CHANNEL_UUID: { |
2280 | lck_rw_lock_shared(&pcb->utun_pcb_lock); |
2281 | if (uuid_is_null(pcb->utun_kpipe_uuid)) { |
2282 | result = ENXIO; |
2283 | } else if (*len != sizeof(uuid_t)) { |
2284 | result = EMSGSIZE; |
2285 | } else { |
2286 | uuid_copy(data, pcb->utun_kpipe_uuid); |
2287 | } |
2288 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
2289 | break; |
2290 | } |
2291 | case UTUN_OPT_SLOT_SIZE: { |
2292 | if (*len != sizeof(u_int32_t)) { |
2293 | result = EMSGSIZE; |
2294 | } else { |
2295 | *(u_int32_t *)data = pcb->utun_slot_size; |
2296 | } |
2297 | break; |
2298 | } |
2299 | case UTUN_OPT_NETIF_RING_SIZE: { |
2300 | if (*len != sizeof(u_int32_t)) { |
2301 | result = EMSGSIZE; |
2302 | } else { |
2303 | *(u_int32_t *)data = pcb->utun_netif_ring_size; |
2304 | } |
2305 | break; |
2306 | } |
2307 | case UTUN_OPT_TX_FSW_RING_SIZE: { |
2308 | if (*len != sizeof(u_int32_t)) { |
2309 | result = EMSGSIZE; |
2310 | } else { |
2311 | *(u_int32_t *)data = pcb->utun_tx_fsw_ring_size; |
2312 | } |
2313 | break; |
2314 | } |
2315 | case UTUN_OPT_RX_FSW_RING_SIZE: { |
2316 | if (*len != sizeof(u_int32_t)) { |
2317 | result = EMSGSIZE; |
2318 | } else { |
2319 | *(u_int32_t *)data = pcb->utun_rx_fsw_ring_size; |
2320 | } |
2321 | break; |
2322 | } |
2323 | #endif // UTUN_NEXUS |
2324 | |
2325 | default: |
2326 | result = ENOPROTOOPT; |
2327 | break; |
2328 | } |
2329 | |
2330 | return result; |
2331 | } |
2332 | |
2333 | static void |
2334 | utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int flags) |
2335 | { |
2336 | #pragma unused(flags) |
2337 | bool reenable_output = false; |
2338 | struct utun_pcb *pcb = unitinfo; |
2339 | if (pcb == NULL) { |
2340 | return; |
2341 | } |
2342 | ifnet_lock_exclusive(pcb->utun_ifp); |
2343 | |
2344 | u_int32_t utun_packet_cnt; |
2345 | errno_t error_pc = ctl_getenqueuepacketcount(kctlref, unit, &utun_packet_cnt); |
2346 | if (error_pc != 0) { |
2347 | printf("utun_ctl_rcvd: ctl_getenqueuepacketcount returned error %d\n" , error_pc); |
2348 | utun_packet_cnt = 0; |
2349 | } |
2350 | |
2351 | if (utun_packet_cnt < pcb->utun_max_pending_packets) { |
2352 | reenable_output = true; |
2353 | } |
2354 | |
2355 | if (reenable_output) { |
2356 | errno_t error = ifnet_enable_output(pcb->utun_ifp); |
2357 | if (error != 0) { |
2358 | printf("utun_ctl_rcvd: ifnet_enable_output returned error %d\n" , error); |
2359 | } |
2360 | } |
2361 | ifnet_lock_done(pcb->utun_ifp); |
2362 | } |
2363 | |
2364 | /* Network Interface functions */ |
2365 | static void |
2366 | utun_start(ifnet_t interface) |
2367 | { |
2368 | mbuf_t data; |
2369 | struct utun_pcb *pcb = ifnet_softc(interface); |
2370 | |
2371 | VERIFY(pcb != NULL); |
2372 | |
2373 | #if UTUN_NEXUS |
2374 | lck_rw_lock_shared(&pcb->utun_pcb_lock); |
2375 | if (pcb->utun_kpipe_enabled) { |
2376 | /* It's possible to have channels enabled, but not yet have the channel opened, |
2377 | * in which case the rxring will not be set |
2378 | */ |
2379 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
2380 | if (pcb->utun_kpipe_rxring != NULL) { |
2381 | kern_channel_notify(pcb->utun_kpipe_rxring, 0); |
2382 | } |
2383 | return; |
2384 | } |
2385 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
2386 | #endif // UTUN_NEXUS |
2387 | |
2388 | for (;;) { |
2389 | bool can_accept_packets = true; |
2390 | ifnet_lock_shared(pcb->utun_ifp); |
2391 | |
2392 | u_int32_t utun_packet_cnt; |
2393 | errno_t error_pc = ctl_getenqueuepacketcount(pcb->utun_ctlref, pcb->utun_unit, &utun_packet_cnt); |
2394 | if (error_pc != 0) { |
2395 | printf("utun_start: ctl_getenqueuepacketcount returned error %d\n" , error_pc); |
2396 | utun_packet_cnt = 0; |
2397 | } |
2398 | |
2399 | can_accept_packets = (utun_packet_cnt < pcb->utun_max_pending_packets); |
2400 | if (!can_accept_packets && pcb->utun_ctlref) { |
2401 | u_int32_t difference = 0; |
2402 | if (ctl_getenqueuereadable(pcb->utun_ctlref, pcb->utun_unit, &difference) == 0) { |
2403 | if (difference > 0) { |
2404 | // If the low-water mark has not yet been reached, we still need to enqueue data |
2405 | // into the buffer |
2406 | can_accept_packets = true; |
2407 | } |
2408 | } |
2409 | } |
2410 | if (!can_accept_packets) { |
2411 | errno_t error = ifnet_disable_output(interface); |
2412 | if (error != 0) { |
2413 | printf("utun_start: ifnet_disable_output returned error %d\n" , error); |
2414 | } |
2415 | ifnet_lock_done(pcb->utun_ifp); |
2416 | break; |
2417 | } |
2418 | ifnet_lock_done(pcb->utun_ifp); |
2419 | if (ifnet_dequeue(interface, &data) != 0) { |
2420 | break; |
2421 | } |
2422 | if (utun_output(interface, data) != 0) { |
2423 | break; |
2424 | } |
2425 | } |
2426 | } |
2427 | |
2428 | static errno_t |
2429 | utun_output(ifnet_t interface, |
2430 | mbuf_t data) |
2431 | { |
2432 | struct utun_pcb *pcb = ifnet_softc(interface); |
2433 | errno_t result; |
2434 | |
2435 | VERIFY(interface == pcb->utun_ifp); |
2436 | |
2437 | #if UTUN_NEXUS |
2438 | if (!pcb->utun_use_netif) |
2439 | #endif // UTUN_NEXUS |
2440 | { |
2441 | if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) { |
2442 | bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0); |
2443 | } |
2444 | } |
2445 | |
2446 | if (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT) { |
2447 | /* flush data */ |
2448 | mbuf_freem(data); |
2449 | return 0; |
2450 | } |
2451 | |
2452 | // otherwise, fall thru to ctl_enqueumbuf |
2453 | if (pcb->utun_ctlref) { |
2454 | int length; |
2455 | |
2456 | /* |
2457 | * The ABI requires the protocol in network byte order |
2458 | */ |
2459 | if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) { |
2460 | *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data)); |
2461 | } |
2462 | |
2463 | length = mbuf_pkthdr_len(data); |
2464 | result = ctl_enqueuembuf(pcb->utun_ctlref, pcb->utun_unit, data, CTL_DATA_EOR); |
2465 | if (result != 0) { |
2466 | mbuf_freem(data); |
2467 | printf("utun_output - ctl_enqueuembuf failed: %d\n" , result); |
2468 | #if UTUN_NEXUS |
2469 | if (!pcb->utun_use_netif) |
2470 | #endif // UTUN_NEXUS |
2471 | { |
2472 | ifnet_stat_increment_out(interface, 0, 0, 1); |
2473 | } |
2474 | } else { |
2475 | #if UTUN_NEXUS |
2476 | if (!pcb->utun_use_netif) |
2477 | #endif // UTUN_NEXUS |
2478 | { |
2479 | if (!pcb->utun_ext_ifdata_stats) { |
2480 | ifnet_stat_increment_out(interface, 1, length, 0); |
2481 | } |
2482 | } |
2483 | } |
2484 | } else { |
2485 | mbuf_freem(data); |
2486 | } |
2487 | |
2488 | return 0; |
2489 | } |
2490 | |
2491 | static errno_t |
2492 | utun_demux(__unused ifnet_t interface, |
2493 | mbuf_t data, |
2494 | __unused char *, |
2495 | protocol_family_t *protocol) |
2496 | { |
2497 | #if UTUN_NEXUS |
2498 | struct utun_pcb *pcb = ifnet_softc(interface); |
2499 | struct ip *ip; |
2500 | u_int ip_version; |
2501 | #endif |
2502 | |
2503 | while (data != NULL && mbuf_len(data) < 1) { |
2504 | data = mbuf_next(data); |
2505 | } |
2506 | |
2507 | if (data == NULL) { |
2508 | return ENOENT; |
2509 | } |
2510 | |
2511 | #if UTUN_NEXUS |
2512 | if (pcb->utun_use_netif) { |
2513 | ip = mtod(data, struct ip *); |
2514 | ip_version = ip->ip_v; |
2515 | |
2516 | switch(ip_version) { |
2517 | case 4: |
2518 | *protocol = PF_INET; |
2519 | return 0; |
2520 | case 6: |
2521 | *protocol = PF_INET6; |
2522 | return 0; |
2523 | default: |
2524 | *protocol = 0; |
2525 | break; |
2526 | } |
2527 | } else |
2528 | #endif // UTUN_NEXUS |
2529 | { |
2530 | *protocol = *(u_int32_t *)mbuf_data(data); |
2531 | } |
2532 | |
2533 | return 0; |
2534 | } |
2535 | |
2536 | static errno_t |
2537 | utun_framer(ifnet_t interface, |
2538 | mbuf_t *packet, |
2539 | __unused const struct sockaddr *dest, |
2540 | __unused const char *desk_linkaddr, |
2541 | const char *frame_type, |
2542 | u_int32_t *prepend_len, |
2543 | u_int32_t *postpend_len) |
2544 | { |
2545 | struct utun_pcb *pcb = ifnet_softc(interface); |
2546 | VERIFY(interface == pcb->utun_ifp); |
2547 | |
2548 | u_int32_t = UTUN_HEADER_SIZE(pcb); |
2549 | if (mbuf_prepend(packet, header_length, MBUF_DONTWAIT) != 0) { |
2550 | printf("utun_framer - ifnet_output prepend failed\n" ); |
2551 | |
2552 | ifnet_stat_increment_out(interface, 0, 0, 1); |
2553 | |
2554 | // just return, because the buffer was freed in mbuf_prepend |
2555 | return EJUSTRETURN; |
2556 | } |
2557 | if (prepend_len != NULL) { |
2558 | *prepend_len = header_length; |
2559 | } |
2560 | if (postpend_len != NULL) { |
2561 | *postpend_len = 0; |
2562 | } |
2563 | |
2564 | // place protocol number at the beginning of the mbuf |
2565 | *(protocol_family_t *)mbuf_data(*packet) = *(protocol_family_t *)(uintptr_t)(size_t)frame_type; |
2566 | |
2567 | |
2568 | return 0; |
2569 | } |
2570 | |
2571 | static errno_t |
2572 | utun_add_proto(__unused ifnet_t interface, |
2573 | protocol_family_t protocol, |
2574 | __unused const struct ifnet_demux_desc *demux_array, |
2575 | __unused u_int32_t demux_count) |
2576 | { |
2577 | switch(protocol) { |
2578 | case PF_INET: |
2579 | return 0; |
2580 | case PF_INET6: |
2581 | return 0; |
2582 | default: |
2583 | break; |
2584 | } |
2585 | |
2586 | return ENOPROTOOPT; |
2587 | } |
2588 | |
2589 | static errno_t |
2590 | utun_del_proto(__unused ifnet_t interface, |
2591 | __unused protocol_family_t protocol) |
2592 | { |
2593 | return 0; |
2594 | } |
2595 | |
2596 | static errno_t |
2597 | utun_ioctl(ifnet_t interface, |
2598 | u_long command, |
2599 | void *data) |
2600 | { |
2601 | #if UTUN_NEXUS |
2602 | struct utun_pcb *pcb = ifnet_softc(interface); |
2603 | #endif |
2604 | errno_t result = 0; |
2605 | |
2606 | switch(command) { |
2607 | case SIOCSIFMTU: { |
2608 | #if UTUN_NEXUS |
2609 | if (pcb->utun_use_netif) { |
2610 | // Make sure we can fit packets in the channel buffers |
2611 | // Allow for the headroom in the slot |
2612 | if (((uint64_t)((struct ifreq*)data)->ifr_mtu) + UTUN_IF_HEADROOM_SIZE > pcb->utun_slot_size) { |
2613 | result = EINVAL; |
2614 | } else { |
2615 | ifnet_set_mtu(interface, (uint32_t)((struct ifreq*)data)->ifr_mtu); |
2616 | } |
2617 | } else |
2618 | #endif // UTUN_NEXUS |
2619 | { |
2620 | ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu); |
2621 | } |
2622 | break; |
2623 | } |
2624 | |
2625 | case SIOCSIFFLAGS: |
2626 | /* ifioctl() takes care of it */ |
2627 | break; |
2628 | |
2629 | default: |
2630 | result = EOPNOTSUPP; |
2631 | } |
2632 | |
2633 | return result; |
2634 | } |
2635 | |
2636 | static void |
2637 | utun_detached(ifnet_t interface) |
2638 | { |
2639 | struct utun_pcb *pcb = ifnet_softc(interface); |
2640 | (void)ifnet_release(interface); |
2641 | utun_free_pcb(pcb, true); |
2642 | } |
2643 | |
2644 | /* Protocol Handlers */ |
2645 | |
2646 | static errno_t |
2647 | utun_proto_input(__unused ifnet_t interface, |
2648 | protocol_family_t protocol, |
2649 | mbuf_t m, |
2650 | __unused char *) |
2651 | { |
2652 | struct utun_pcb *pcb = ifnet_softc(interface); |
2653 | #if UTUN_NEXUS |
2654 | if (!pcb->utun_use_netif) |
2655 | #endif // UTUN_NEXUS |
2656 | { |
2657 | mbuf_adj(m, UTUN_HEADER_SIZE(pcb)); |
2658 | } |
2659 | int32_t pktlen = m->m_pkthdr.len; |
2660 | if (proto_input(protocol, m) != 0) { |
2661 | m_freem(m); |
2662 | #if UTUN_NEXUS |
2663 | if (!pcb->utun_use_netif) |
2664 | #endif // UTUN_NEXUS |
2665 | { |
2666 | ifnet_stat_increment_in(interface, 0, 0, 1); |
2667 | } |
2668 | } else { |
2669 | #if UTUN_NEXUS |
2670 | if (!pcb->utun_use_netif) |
2671 | #endif // UTUN_NEXUS |
2672 | { |
2673 | ifnet_stat_increment_in(interface, 1, pktlen, 0); |
2674 | } |
2675 | } |
2676 | |
2677 | return 0; |
2678 | } |
2679 | |
2680 | static errno_t |
2681 | utun_proto_pre_output(__unused ifnet_t interface, |
2682 | protocol_family_t protocol, |
2683 | __unused mbuf_t *packet, |
2684 | __unused const struct sockaddr *dest, |
2685 | __unused void *route, |
2686 | char *frame_type, |
2687 | __unused char *link_layer_dest) |
2688 | { |
2689 | *(protocol_family_t *)(void *)frame_type = protocol; |
2690 | return 0; |
2691 | } |
2692 | |
2693 | static errno_t |
2694 | utun_attach_proto(ifnet_t interface, |
2695 | protocol_family_t protocol) |
2696 | { |
2697 | struct ifnet_attach_proto_param proto; |
2698 | |
2699 | bzero(&proto, sizeof(proto)); |
2700 | proto.input = utun_proto_input; |
2701 | proto.pre_output = utun_proto_pre_output; |
2702 | |
2703 | errno_t result = ifnet_attach_protocol(interface, protocol, &proto); |
2704 | if (result != 0 && result != EEXIST) { |
2705 | printf("utun_attach_inet - ifnet_attach_protocol %d failed: %d\n" , |
2706 | protocol, result); |
2707 | } |
2708 | |
2709 | return result; |
2710 | } |
2711 | |
2712 | static errno_t |
2713 | utun_pkt_input(struct utun_pcb *pcb, mbuf_t packet) |
2714 | { |
2715 | #if UTUN_NEXUS |
2716 | if (pcb->utun_use_netif) { |
2717 | lck_rw_lock_shared(&pcb->utun_pcb_lock); |
2718 | |
2719 | lck_mtx_lock(&pcb->utun_input_chain_lock); |
2720 | if (pcb->utun_input_chain != NULL) { |
2721 | pcb->utun_input_chain_last->m_nextpkt = packet; |
2722 | } else { |
2723 | pcb->utun_input_chain = packet; |
2724 | } |
2725 | while (packet->m_nextpkt) { |
2726 | VERIFY(packet != packet->m_nextpkt); |
2727 | packet = packet->m_nextpkt; |
2728 | } |
2729 | pcb->utun_input_chain_last = packet; |
2730 | lck_mtx_unlock(&pcb->utun_input_chain_lock); |
2731 | |
2732 | kern_channel_ring_t rx_ring = pcb->utun_netif_rxring; |
2733 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
2734 | |
2735 | if (rx_ring != NULL) { |
2736 | kern_channel_notify(rx_ring, 0); |
2737 | } |
2738 | |
2739 | return (0); |
2740 | } else |
2741 | #endif // IPSEC_NEXUS |
2742 | { |
2743 | mbuf_pkthdr_setrcvif(packet, pcb->utun_ifp); |
2744 | |
2745 | if (m_pktlen(packet) >= (int32_t)UTUN_HEADER_SIZE(pcb)) { |
2746 | bpf_tap_in(pcb->utun_ifp, DLT_NULL, packet, 0, 0); |
2747 | } |
2748 | if (pcb->utun_flags & UTUN_FLAGS_NO_INPUT) { |
2749 | /* flush data */ |
2750 | mbuf_freem(packet); |
2751 | return 0; |
2752 | } |
2753 | |
2754 | errno_t result = 0; |
2755 | if (!pcb->utun_ext_ifdata_stats) { |
2756 | struct ifnet_stat_increment_param incs = {}; |
2757 | incs.packets_in = 1; |
2758 | incs.bytes_in = mbuf_pkthdr_len(packet); |
2759 | result = ifnet_input(pcb->utun_ifp, packet, &incs); |
2760 | } else { |
2761 | result = ifnet_input(pcb->utun_ifp, packet, NULL); |
2762 | } |
2763 | if (result != 0) { |
2764 | ifnet_stat_increment_in(pcb->utun_ifp, 0, 0, 1); |
2765 | |
2766 | printf("%s - ifnet_input failed: %d\n" , __FUNCTION__, result); |
2767 | mbuf_freem(packet); |
2768 | } |
2769 | |
2770 | return (0); |
2771 | } |
2772 | } |
2773 | |
2774 | #if UTUN_NEXUS |
2775 | |
2776 | static errno_t |
2777 | utun_nxdp_init(__unused kern_nexus_domain_provider_t domprov) |
2778 | { |
2779 | return 0; |
2780 | } |
2781 | |
2782 | static void |
2783 | utun_nxdp_fini(__unused kern_nexus_domain_provider_t domprov) |
2784 | { |
2785 | // Ignore |
2786 | } |
2787 | |
2788 | static errno_t |
2789 | utun_register_nexus(void) |
2790 | { |
2791 | const struct kern_nexus_domain_provider_init dp_init = { |
2792 | .nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION, |
2793 | .nxdpi_flags = 0, |
2794 | .nxdpi_init = utun_nxdp_init, |
2795 | .nxdpi_fini = utun_nxdp_fini |
2796 | }; |
2797 | errno_t err = 0; |
2798 | |
2799 | /* utun_nxdp_init() is called before this function returns */ |
2800 | err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF, |
2801 | (const uint8_t *) "com.apple.utun" , |
2802 | &dp_init, sizeof(dp_init), |
2803 | &utun_nx_dom_prov); |
2804 | if (err != 0) { |
2805 | printf("%s: failed to register domain provider\n" , __func__); |
2806 | return (err); |
2807 | } |
2808 | return (0); |
2809 | } |
2810 | boolean_t |
2811 | utun_interface_needs_netagent(ifnet_t interface) |
2812 | { |
2813 | struct utun_pcb *pcb = NULL; |
2814 | |
2815 | if (interface == NULL) { |
2816 | return (FALSE); |
2817 | } |
2818 | |
2819 | pcb = ifnet_softc(interface); |
2820 | |
2821 | if (pcb == NULL) { |
2822 | return (FALSE); |
2823 | } |
2824 | |
2825 | return (pcb->utun_needs_netagent == true); |
2826 | } |
2827 | |
2828 | static errno_t |
2829 | utun_ifnet_set_attrs(ifnet_t ifp) |
2830 | { |
2831 | /* Set flags and additional information. */ |
2832 | ifnet_set_mtu(ifp, 1500); |
2833 | ifnet_set_flags(ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff); |
2834 | |
2835 | /* The interface must generate its own IPv6 LinkLocal address, |
2836 | * if possible following the recommendation of RFC2472 to the 64bit interface ID |
2837 | */ |
2838 | ifnet_set_eflags(ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL); |
2839 | |
2840 | return (0); |
2841 | } |
2842 | |
2843 | static errno_t |
2844 | utun_netif_prepare(kern_nexus_t nexus, ifnet_t ifp) |
2845 | { |
2846 | struct utun_pcb *pcb = kern_nexus_get_context(nexus); |
2847 | pcb->utun_netif_nexus = nexus; |
2848 | return (utun_ifnet_set_attrs(ifp)); |
2849 | } |
2850 | |
2851 | static errno_t |
2852 | utun_nexus_pre_connect(kern_nexus_provider_t nxprov, |
2853 | proc_t p, kern_nexus_t nexus, |
2854 | nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx) |
2855 | { |
2856 | #pragma unused(nxprov, p) |
2857 | #pragma unused(nexus, nexus_port, channel, ch_ctx) |
2858 | return (0); |
2859 | } |
2860 | |
2861 | static errno_t |
2862 | utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
2863 | kern_channel_t channel) |
2864 | { |
2865 | #pragma unused(nxprov, channel) |
2866 | struct utun_pcb *pcb = kern_nexus_get_context(nexus); |
2867 | boolean_t ok = ifnet_is_attached(pcb->utun_ifp, 1); |
2868 | return (ok ? 0 : ENXIO); |
2869 | } |
2870 | |
2871 | static void |
2872 | utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
2873 | kern_channel_t channel) |
2874 | { |
2875 | #pragma unused(nxprov, nexus, channel) |
2876 | } |
2877 | |
2878 | static void |
2879 | utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
2880 | kern_channel_t channel) |
2881 | { |
2882 | #pragma unused(nxprov, nexus, channel) |
2883 | } |
2884 | |
2885 | static void |
2886 | utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
2887 | kern_channel_t channel) |
2888 | { |
2889 | #pragma unused(nxprov, channel) |
2890 | struct utun_pcb *pcb = kern_nexus_get_context(nexus); |
2891 | if (pcb->utun_netif_nexus == nexus) { |
2892 | pcb->utun_netif_nexus = NULL; |
2893 | } |
2894 | ifnet_decr_iorefcnt(pcb->utun_ifp); |
2895 | } |
2896 | |
2897 | static errno_t |
2898 | utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
2899 | kern_channel_t channel, kern_channel_ring_t ring, |
2900 | boolean_t is_tx_ring, void **ring_ctx) |
2901 | { |
2902 | #pragma unused(nxprov) |
2903 | #pragma unused(channel) |
2904 | #pragma unused(ring_ctx) |
2905 | struct utun_pcb *pcb = kern_nexus_get_context(nexus); |
2906 | if (!is_tx_ring) { |
2907 | VERIFY(pcb->utun_kpipe_rxring == NULL); |
2908 | pcb->utun_kpipe_rxring = ring; |
2909 | } else { |
2910 | VERIFY(pcb->utun_kpipe_txring == NULL); |
2911 | pcb->utun_kpipe_txring = ring; |
2912 | } |
2913 | return 0; |
2914 | } |
2915 | |
2916 | static void |
2917 | utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
2918 | kern_channel_ring_t ring) |
2919 | { |
2920 | #pragma unused(nxprov) |
2921 | struct utun_pcb *pcb = kern_nexus_get_context(nexus); |
2922 | if (pcb->utun_kpipe_rxring == ring) { |
2923 | pcb->utun_kpipe_rxring = NULL; |
2924 | } else if (pcb->utun_kpipe_txring == ring) { |
2925 | pcb->utun_kpipe_txring = NULL; |
2926 | } |
2927 | } |
2928 | |
2929 | static errno_t |
2930 | utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
2931 | kern_channel_ring_t tx_ring, uint32_t flags) |
2932 | { |
2933 | #pragma unused(nxprov) |
2934 | #pragma unused(flags) |
2935 | struct utun_pcb *pcb = kern_nexus_get_context(nexus); |
2936 | |
2937 | lck_rw_lock_shared(&pcb->utun_pcb_lock); |
2938 | int channel_enabled = pcb->utun_kpipe_enabled; |
2939 | if (!channel_enabled) { |
2940 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
2941 | return 0; |
2942 | } |
2943 | |
2944 | if (pcb->utun_use_netif) { |
2945 | kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); |
2946 | if (tx_slot == NULL) { |
2947 | // Nothing to write, bail |
2948 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
2949 | return 0; |
2950 | } |
2951 | |
2952 | // Signal the netif ring to read |
2953 | kern_channel_ring_t rx_ring = pcb->utun_netif_rxring; |
2954 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
2955 | if (rx_ring != NULL) { |
2956 | kern_channel_notify(rx_ring, 0); |
2957 | } |
2958 | } else { |
2959 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
2960 | |
2961 | struct ifnet_stat_increment_param incs = {}; |
2962 | struct kern_channel_ring_stat_increment tx_ring_stats = {}; |
2963 | MBUFQ_HEAD(mbufq) mbq; |
2964 | MBUFQ_INIT(&mbq); |
2965 | kern_channel_slot_t tx_pslot = NULL; |
2966 | kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); |
2967 | while (tx_slot != NULL) { |
2968 | kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); |
2969 | |
2970 | // Advance TX ring |
2971 | tx_pslot = tx_slot; |
2972 | tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); |
2973 | |
2974 | if (tx_ph == 0) { |
2975 | continue; |
2976 | } |
2977 | |
2978 | kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); |
2979 | VERIFY(tx_buf != NULL); |
2980 | uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); |
2981 | VERIFY(tx_baddr != 0); |
2982 | tx_baddr += kern_buflet_get_data_offset(tx_buf); |
2983 | |
2984 | size_t length = MIN(kern_packet_get_data_length(tx_ph), |
2985 | pcb->utun_slot_size); |
2986 | |
2987 | mbuf_t data = NULL; |
2988 | if (length >= UTUN_HEADER_SIZE(pcb) && |
2989 | !(pcb->utun_flags & UTUN_FLAGS_NO_INPUT)) { |
2990 | errno_t error = mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_HEADER, &data); |
2991 | VERIFY(0 == error); |
2992 | error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_WAITOK); |
2993 | VERIFY(0 == error); |
2994 | /* |
2995 | * The userland ABI requires the first four bytes have |
2996 | * the protocol family in network byte order: swap them |
2997 | */ |
2998 | *(uint32_t *)mbuf_data(data) = ntohl(*(uint32_t *)mbuf_data(data)); |
2999 | mbuf_pkthdr_setrcvif(data, pcb->utun_ifp); |
3000 | bpf_tap_in(pcb->utun_ifp, DLT_NULL, data, 0, 0); |
3001 | incs.packets_in++; |
3002 | incs.bytes_in += length; |
3003 | MBUFQ_ENQUEUE(&mbq, data); |
3004 | } |
3005 | } |
3006 | if (tx_pslot) { |
3007 | kern_channel_advance_slot(tx_ring, tx_pslot); |
3008 | tx_ring_stats.kcrsi_slots_transferred = incs.packets_in; |
3009 | tx_ring_stats.kcrsi_bytes_transferred = incs.bytes_in; |
3010 | kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats); |
3011 | (void) kern_channel_reclaim(tx_ring); |
3012 | } |
3013 | if (!MBUFQ_EMPTY(&mbq)) { |
3014 | (void) ifnet_input_extended(pcb->utun_ifp, MBUFQ_FIRST(&mbq), |
3015 | MBUFQ_LAST(&mbq), &incs); |
3016 | MBUFQ_INIT(&mbq); |
3017 | } |
3018 | } |
3019 | |
3020 | return 0; |
3021 | } |
3022 | |
3023 | static errno_t |
3024 | utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
3025 | kern_channel_ring_t rx_ring, uint32_t flags) |
3026 | { |
3027 | #pragma unused(nxprov) |
3028 | #pragma unused(flags) |
3029 | struct utun_pcb *pcb = kern_nexus_get_context(nexus); |
3030 | struct kern_channel_ring_stat_increment rx_ring_stats = {}; |
3031 | |
3032 | lck_rw_lock_shared(&pcb->utun_pcb_lock); |
3033 | |
3034 | int channel_enabled = pcb->utun_kpipe_enabled; |
3035 | if (!channel_enabled) { |
3036 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
3037 | return 0; |
3038 | } |
3039 | |
3040 | /* reclaim user-released slots */ |
3041 | (void) kern_channel_reclaim(rx_ring); |
3042 | |
3043 | uint32_t avail = kern_channel_available_slot_count(rx_ring); |
3044 | if (avail == 0) { |
3045 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
3046 | return 0; |
3047 | } |
3048 | |
3049 | if (pcb->utun_use_netif) { |
3050 | kern_channel_ring_t tx_ring = pcb->utun_netif_txring; |
3051 | if (tx_ring == NULL || |
3052 | pcb->utun_netif_nexus == NULL) { |
3053 | // Net-If TX ring not set up yet, nothing to read |
3054 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
3055 | return 0; |
3056 | } |
3057 | |
3058 | struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->utun_netif_nexus)->nif_stats; |
3059 | |
3060 | // Unlock utun before entering ring |
3061 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
3062 | |
3063 | (void)kr_enter(tx_ring, TRUE); |
3064 | |
3065 | // Lock again after entering and validate |
3066 | lck_rw_lock_shared(&pcb->utun_pcb_lock); |
3067 | if (tx_ring != pcb->utun_netif_txring) { |
3068 | // Ring no longer valid |
3069 | // Unlock first, then exit ring |
3070 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
3071 | kr_exit(tx_ring); |
3072 | return 0; |
3073 | } |
3074 | |
3075 | struct kern_channel_ring_stat_increment tx_ring_stats; |
3076 | bzero(&tx_ring_stats, sizeof(tx_ring_stats)); |
3077 | kern_channel_slot_t tx_pslot = NULL; |
3078 | kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); |
3079 | if (tx_slot == NULL) { |
3080 | // Nothing to read, don't bother signalling |
3081 | // Unlock first, then exit ring |
3082 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
3083 | kr_exit(tx_ring); |
3084 | return 0; |
3085 | } |
3086 | |
3087 | struct kern_pbufpool *rx_pp = rx_ring->ckr_pp; |
3088 | VERIFY(rx_pp != NULL); |
3089 | kern_channel_slot_t rx_pslot = NULL; |
3090 | kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL); |
3091 | |
3092 | while (rx_slot != NULL && tx_slot != NULL) { |
3093 | size_t length; |
3094 | kern_buflet_t rx_buf; |
3095 | void *rx_baddr; |
3096 | |
3097 | kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); |
3098 | |
3099 | // Advance TX ring |
3100 | tx_pslot = tx_slot; |
3101 | tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); |
3102 | |
3103 | /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */ |
3104 | if (tx_ph == 0) { |
3105 | continue; |
3106 | } |
3107 | |
3108 | // Allocate rx packet |
3109 | kern_packet_t rx_ph = 0; |
3110 | errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); |
3111 | if (__improbable(error != 0)) { |
3112 | printf("utun_kpipe_sync_rx %s: failed to allocate packet\n" , |
3113 | pcb->utun_ifp->if_xname); |
3114 | break; |
3115 | } |
3116 | |
3117 | kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); |
3118 | VERIFY(tx_buf != NULL); |
3119 | uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); |
3120 | VERIFY(tx_baddr != NULL); |
3121 | tx_baddr += kern_buflet_get_data_offset(tx_buf); |
3122 | |
3123 | bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0); |
3124 | |
3125 | length = MIN(kern_packet_get_data_length(tx_ph) + UTUN_HEADER_SIZE(pcb), |
3126 | pcb->utun_slot_size); |
3127 | |
3128 | tx_ring_stats.kcrsi_slots_transferred++; |
3129 | tx_ring_stats.kcrsi_bytes_transferred += length; |
3130 | |
3131 | if (length < UTUN_HEADER_SIZE(pcb) || |
3132 | length > pcb->utun_slot_size || |
3133 | length > rx_pp->pp_buflet_size || |
3134 | (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) { |
3135 | /* flush data */ |
3136 | kern_pbufpool_free(rx_pp, rx_ph); |
3137 | printf("utun_kpipe_sync_rx %s: invalid length %zu header_size %zu\n" , |
3138 | pcb->utun_ifp->if_xname, length, UTUN_HEADER_SIZE(pcb)); |
3139 | STATS_INC(nifs, NETIF_STATS_BADLEN); |
3140 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
3141 | continue; |
3142 | } |
3143 | |
3144 | /* fillout packet */ |
3145 | rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); |
3146 | VERIFY(rx_buf != NULL); |
3147 | rx_baddr = kern_buflet_get_object_address(rx_buf); |
3148 | VERIFY(rx_baddr != NULL); |
3149 | |
3150 | // Find family |
3151 | uint32_t af = 0; |
3152 | uint8_t vhl = *(uint8_t *)(tx_baddr); |
3153 | u_int ip_version = (vhl >> 4); |
3154 | switch (ip_version) { |
3155 | case 4: { |
3156 | af = AF_INET; |
3157 | break; |
3158 | } |
3159 | case 6: { |
3160 | af = AF_INET6; |
3161 | break; |
3162 | } |
3163 | default: { |
3164 | printf("utun_kpipe_sync_rx %s: unknown ip version %u vhl %u header_size %zu\n" , |
3165 | pcb->utun_ifp->if_xname, ip_version, vhl, UTUN_HEADER_SIZE(pcb)); |
3166 | break; |
3167 | } |
3168 | } |
3169 | |
3170 | // Copy header |
3171 | af = htonl(af); |
3172 | memcpy((void *)rx_baddr, &af, sizeof(af)); |
3173 | if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) { |
3174 | kern_packet_get_euuid(tx_ph, (void *)(rx_baddr + sizeof(af))); |
3175 | } |
3176 | |
3177 | // Copy data from tx to rx |
3178 | memcpy((void *)(rx_baddr + UTUN_HEADER_SIZE(pcb)), (void *)tx_baddr, length - UTUN_HEADER_SIZE(pcb)); |
3179 | kern_packet_clear_flow_uuid(rx_ph); // zero flow id |
3180 | |
3181 | /* finalize and attach the packet */ |
3182 | error = kern_buflet_set_data_offset(rx_buf, 0); |
3183 | VERIFY(error == 0); |
3184 | error = kern_buflet_set_data_length(rx_buf, length); |
3185 | VERIFY(error == 0); |
3186 | error = kern_packet_finalize(rx_ph); |
3187 | VERIFY(error == 0); |
3188 | error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); |
3189 | VERIFY(error == 0); |
3190 | |
3191 | STATS_INC(nifs, NETIF_STATS_TXPKTS); |
3192 | STATS_INC(nifs, NETIF_STATS_TXCOPY_DIRECT); |
3193 | |
3194 | rx_ring_stats.kcrsi_slots_transferred++; |
3195 | rx_ring_stats.kcrsi_bytes_transferred += length; |
3196 | |
3197 | rx_pslot = rx_slot; |
3198 | rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); |
3199 | } |
3200 | |
3201 | if (rx_pslot) { |
3202 | kern_channel_advance_slot(rx_ring, rx_pslot); |
3203 | kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats); |
3204 | } |
3205 | |
3206 | if (tx_pslot) { |
3207 | kern_channel_advance_slot(tx_ring, tx_pslot); |
3208 | kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats); |
3209 | (void)kern_channel_reclaim(tx_ring); |
3210 | } |
3211 | |
3212 | /* just like utun_ctl_rcvd(), always reenable output */ |
3213 | errno_t error = ifnet_enable_output(pcb->utun_ifp); |
3214 | if (error != 0) { |
3215 | printf("utun_kpipe_sync_rx: ifnet_enable_output returned error %d\n" , error); |
3216 | } |
3217 | |
3218 | // Unlock first, then exit ring |
3219 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
3220 | |
3221 | if (tx_pslot != NULL) { |
3222 | kern_channel_notify(tx_ring, 0); |
3223 | } |
3224 | kr_exit(tx_ring); |
3225 | } else { |
3226 | lck_rw_unlock_shared(&pcb->utun_pcb_lock); |
3227 | |
3228 | uint32_t mb_cnt = 0; |
3229 | uint32_t mb_len = 0; |
3230 | struct mbuf *mb_head = NULL; |
3231 | struct mbuf *mb_tail = NULL; |
3232 | |
3233 | if (ifnet_dequeue_multi(pcb->utun_ifp, avail, &mb_head, |
3234 | &mb_tail, &mb_cnt, &mb_len) != 0) { |
3235 | return 0; |
3236 | } |
3237 | VERIFY(mb_cnt <= avail); |
3238 | |
3239 | struct kern_pbufpool *rx_pp = rx_ring->ckr_pp; |
3240 | VERIFY(rx_pp != NULL); |
3241 | kern_channel_slot_t rx_pslot = NULL; |
3242 | kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL); |
3243 | while (rx_slot) { |
3244 | size_t length = 0; |
3245 | mbuf_t data = NULL; |
3246 | if ((data = mb_head) == NULL) { |
3247 | VERIFY(mb_cnt == 0); |
3248 | break; |
3249 | } |
3250 | mb_head = mbuf_nextpkt(mb_head); |
3251 | mbuf_setnextpkt(data, NULL); |
3252 | VERIFY(mb_cnt != 0); |
3253 | --mb_cnt; |
3254 | length = mbuf_pkthdr_len(data); |
3255 | if (length < UTUN_HEADER_SIZE(pcb) || |
3256 | length > pcb->utun_slot_size || |
3257 | (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) { |
3258 | /* flush data */ |
3259 | mbuf_freem(data); |
3260 | continue; |
3261 | } |
3262 | bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0); |
3263 | |
3264 | // Allocate rx packet |
3265 | kern_packet_t rx_ph = 0; |
3266 | errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); |
3267 | if (__improbable(error != 0)) { |
3268 | printf("utun_kpipe_sync_rx %s: failed to allocate packet\n" , |
3269 | pcb->utun_ifp->if_xname); |
3270 | break; |
3271 | } |
3272 | |
3273 | /* |
3274 | * The ABI requires the protocol in network byte order |
3275 | */ |
3276 | *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data)); |
3277 | |
3278 | // Fillout rx packet |
3279 | kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); |
3280 | VERIFY(rx_buf != NULL); |
3281 | void *rx_baddr = kern_buflet_get_object_address(rx_buf); |
3282 | VERIFY(rx_baddr != NULL); |
3283 | |
3284 | // Copy-in data from mbuf to buflet |
3285 | mbuf_copydata(data, 0, length, (void *)rx_baddr); |
3286 | kern_packet_clear_flow_uuid(rx_ph); // Zero flow id |
3287 | |
3288 | // Finalize and attach the packet |
3289 | error = kern_buflet_set_data_offset(rx_buf, 0); |
3290 | VERIFY(error == 0); |
3291 | error = kern_buflet_set_data_length(rx_buf, length); |
3292 | VERIFY(error == 0); |
3293 | error = kern_packet_finalize(rx_ph); |
3294 | VERIFY(error == 0); |
3295 | error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); |
3296 | VERIFY(error == 0); |
3297 | |
3298 | rx_ring_stats.kcrsi_slots_transferred++; |
3299 | rx_ring_stats.kcrsi_bytes_transferred += length; |
3300 | |
3301 | if (!pcb->utun_ext_ifdata_stats) { |
3302 | ifnet_stat_increment_out(pcb->utun_ifp, 1, length, 0); |
3303 | } |
3304 | |
3305 | mbuf_freem(data); |
3306 | |
3307 | rx_pslot = rx_slot; |
3308 | rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); |
3309 | } |
3310 | if (rx_pslot) { |
3311 | kern_channel_advance_slot(rx_ring, rx_pslot); |
3312 | kern_channel_increment_ring_stats(rx_ring, &rx_ring_stats); |
3313 | } |
3314 | if (mb_head != NULL) { |
3315 | VERIFY(mb_cnt != 0); |
3316 | mbuf_freem_list(mb_head); |
3317 | } |
3318 | } |
3319 | |
3320 | return 0; |
3321 | } |
3322 | |
3323 | #endif // UTUN_NEXUS |
3324 | |
3325 | |
3326 | /* |
3327 | * These are place holders until coreTLS kext stops calling them |
3328 | */ |
3329 | errno_t utun_ctl_register_dtls (void *reg); |
3330 | int utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family); |
3331 | void utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb); |
3332 | |
3333 | errno_t |
3334 | utun_ctl_register_dtls (void *reg) |
3335 | { |
3336 | #pragma unused(reg) |
3337 | return 0; |
3338 | } |
3339 | |
3340 | int |
3341 | utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family) |
3342 | { |
3343 | #pragma unused(pcb) |
3344 | #pragma unused(pkt) |
3345 | #pragma unused(family) |
3346 | return 0; |
3347 | } |
3348 | |
3349 | void |
3350 | utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb) |
3351 | { |
3352 | #pragma unused(pcb) |
3353 | } |
3354 | |