1 | /* |
2 | * Copyright (c) 2012-2018 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | |
30 | #include <sys/systm.h> |
31 | #include <sys/kern_control.h> |
32 | #include <net/kpi_protocol.h> |
33 | #include <net/kpi_interface.h> |
34 | #include <sys/socket.h> |
35 | #include <sys/socketvar.h> |
36 | #include <net/if.h> |
37 | #include <net/if_types.h> |
38 | #include <net/bpf.h> |
39 | #include <net/if_ipsec.h> |
40 | #include <sys/mbuf.h> |
41 | #include <sys/sockio.h> |
42 | #include <netinet/in.h> |
43 | #include <netinet/ip6.h> |
44 | #include <netinet6/in6_var.h> |
45 | #include <netinet6/ip6_var.h> |
46 | #include <sys/kauth.h> |
47 | #include <netinet6/ipsec.h> |
48 | #include <netinet6/ipsec6.h> |
49 | #include <netinet6/esp.h> |
50 | #include <netinet6/esp6.h> |
51 | #include <netinet/ip.h> |
52 | #include <net/flowadv.h> |
53 | #include <net/necp.h> |
54 | #include <netkey/key.h> |
55 | #include <net/pktap.h> |
56 | #include <kern/zalloc.h> |
57 | |
58 | #define IPSEC_NEXUS 0 |
59 | |
60 | extern int net_qos_policy_restricted; |
61 | extern int net_qos_policy_restrict_avapps; |
62 | |
63 | /* Kernel Control functions */ |
64 | static errno_t ipsec_ctl_bind(kern_ctl_ref kctlref, struct sockaddr_ctl *sac, |
65 | void **unitinfo); |
66 | static errno_t ipsec_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac, |
67 | void **unitinfo); |
68 | static errno_t ipsec_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit, |
69 | void *unitinfo); |
70 | static errno_t ipsec_ctl_send(kern_ctl_ref kctlref, u_int32_t unit, |
71 | void *unitinfo, mbuf_t m, int flags); |
72 | static errno_t ipsec_ctl_getopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, |
73 | int opt, void *data, size_t *len); |
74 | static errno_t ipsec_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, |
75 | int opt, void *data, size_t len); |
76 | |
77 | /* Network Interface functions */ |
78 | static void ipsec_start(ifnet_t interface); |
79 | static errno_t ipsec_output(ifnet_t interface, mbuf_t data); |
80 | static errno_t ipsec_demux(ifnet_t interface, mbuf_t data, char *, |
81 | protocol_family_t *protocol); |
82 | static errno_t ipsec_add_proto(ifnet_t interface, protocol_family_t protocol, |
83 | const struct ifnet_demux_desc *demux_array, |
84 | u_int32_t demux_count); |
85 | static errno_t ipsec_del_proto(ifnet_t interface, protocol_family_t protocol); |
86 | static errno_t ipsec_ioctl(ifnet_t interface, u_long cmd, void *data); |
87 | static void ipsec_detached(ifnet_t interface); |
88 | |
89 | /* Protocol handlers */ |
90 | static errno_t ipsec_attach_proto(ifnet_t interface, protocol_family_t proto); |
91 | static errno_t ipsec_proto_input(ifnet_t interface, protocol_family_t protocol, |
92 | mbuf_t m, char *); |
93 | static errno_t ipsec_proto_pre_output(ifnet_t interface, protocol_family_t protocol, |
94 | mbuf_t *packet, const struct sockaddr *dest, void *route, |
95 | char *frame_type, char *link_layer_dest); |
96 | |
97 | static kern_ctl_ref ipsec_kctlref; |
98 | static u_int32_t ipsec_family; |
99 | static lck_attr_t *ipsec_lck_attr; |
100 | static lck_grp_attr_t *ipsec_lck_grp_attr; |
101 | static lck_grp_t *ipsec_lck_grp; |
102 | static lck_mtx_t ipsec_lock; |
103 | |
104 | #if IPSEC_NEXUS |
105 | |
106 | SYSCTL_DECL(_net_ipsec); |
107 | SYSCTL_NODE(_net, OID_AUTO, ipsec, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "IPsec" ); |
108 | static int if_ipsec_verify_interface_creation = 0; |
109 | SYSCTL_INT(_net_ipsec, OID_AUTO, verify_interface_creation, CTLFLAG_RW | CTLFLAG_LOCKED, &if_ipsec_verify_interface_creation, 0, "" ); |
110 | |
111 | #define IPSEC_IF_VERIFY(_e) if (__improbable(if_ipsec_verify_interface_creation)) { VERIFY(_e); } |
112 | |
113 | #define IPSEC_IF_DEFAULT_SLOT_SIZE 2048 |
114 | #define IPSEC_IF_DEFAULT_RING_SIZE 64 |
115 | #define IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE 64 |
116 | #define IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE 128 |
117 | #define IPSEC_IF_DEFAULT_BUF_SEG_SIZE skmem_usr_buf_seg_size |
118 | |
119 | #define IPSEC_IF_MIN_RING_SIZE 16 |
120 | #define IPSEC_IF_MAX_RING_SIZE 1024 |
121 | |
122 | #define IPSEC_IF_MIN_SLOT_SIZE 1024 |
123 | #define IPSEC_IF_MAX_SLOT_SIZE 4096 |
124 | |
125 | static int sysctl_if_ipsec_ring_size SYSCTL_HANDLER_ARGS; |
126 | static int sysctl_if_ipsec_tx_fsw_ring_size SYSCTL_HANDLER_ARGS; |
127 | static int sysctl_if_ipsec_rx_fsw_ring_size SYSCTL_HANDLER_ARGS; |
128 | |
129 | static int if_ipsec_ring_size = IPSEC_IF_DEFAULT_RING_SIZE; |
130 | static int if_ipsec_tx_fsw_ring_size = IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE; |
131 | static int if_ipsec_rx_fsw_ring_size = IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE; |
132 | |
133 | SYSCTL_PROC(_net_ipsec, OID_AUTO, ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW, |
134 | &if_ipsec_ring_size, IPSEC_IF_DEFAULT_RING_SIZE, &sysctl_if_ipsec_ring_size, "I" , "" ); |
135 | SYSCTL_PROC(_net_ipsec, OID_AUTO, tx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW, |
136 | &if_ipsec_tx_fsw_ring_size, IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE, &sysctl_if_ipsec_tx_fsw_ring_size, "I" , "" ); |
137 | SYSCTL_PROC(_net_ipsec, OID_AUTO, rx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW, |
138 | &if_ipsec_rx_fsw_ring_size, IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE, &sysctl_if_ipsec_rx_fsw_ring_size, "I" , "" ); |
139 | |
140 | static errno_t |
141 | ipsec_register_nexus(void); |
142 | |
143 | typedef struct ipsec_nx { |
144 | uuid_t if_provider; |
145 | uuid_t if_instance; |
146 | uuid_t ms_provider; |
147 | uuid_t ms_instance; |
148 | uuid_t ms_device; |
149 | uuid_t ms_host; |
150 | uuid_t ms_agent; |
151 | } *ipsec_nx_t; |
152 | |
153 | static nexus_controller_t ipsec_ncd; |
154 | static int ipsec_ncd_refcount; |
155 | static uuid_t ipsec_kpipe_uuid; |
156 | |
157 | #endif // IPSEC_NEXUS |
158 | |
159 | /* Control block allocated for each kernel control connection */ |
160 | struct ipsec_pcb { |
161 | TAILQ_ENTRY(ipsec_pcb) ipsec_chain; |
162 | kern_ctl_ref ipsec_ctlref; |
163 | ifnet_t ipsec_ifp; |
164 | u_int32_t ipsec_unit; |
165 | u_int32_t ipsec_unique_id; |
166 | u_int32_t ipsec_flags; |
167 | u_int32_t ipsec_input_frag_size; |
168 | bool ipsec_frag_size_set; |
169 | int ipsec_ext_ifdata_stats; |
170 | mbuf_svc_class_t ipsec_output_service_class; |
171 | char ipsec_if_xname[IFXNAMSIZ]; |
172 | char ipsec_unique_name[IFXNAMSIZ]; |
173 | // PCB lock protects state fields, like ipsec_kpipe_enabled |
174 | decl_lck_rw_data(, ipsec_pcb_lock); |
175 | |
176 | #if IPSEC_NEXUS |
177 | lck_mtx_t ipsec_input_chain_lock; |
178 | struct mbuf * ipsec_input_chain; |
179 | struct mbuf * ipsec_input_chain_last; |
180 | // Input chain lock protects the list of input mbufs |
181 | // The input chain lock must be taken AFTER the PCB lock if both are held |
182 | struct ipsec_nx ipsec_nx; |
183 | int ipsec_kpipe_enabled; |
184 | uuid_t ipsec_kpipe_uuid; |
185 | void * ipsec_kpipe_rxring; |
186 | void * ipsec_kpipe_txring; |
187 | kern_pbufpool_t ipsec_kpipe_pp; |
188 | |
189 | kern_nexus_t ipsec_netif_nexus; |
190 | kern_pbufpool_t ipsec_netif_pp; |
191 | void * ipsec_netif_rxring; |
192 | void * ipsec_netif_txring; |
193 | uint64_t ipsec_netif_txring_size; |
194 | |
195 | u_int32_t ipsec_slot_size; |
196 | u_int32_t ipsec_netif_ring_size; |
197 | u_int32_t ipsec_tx_fsw_ring_size; |
198 | u_int32_t ipsec_rx_fsw_ring_size; |
199 | bool ipsec_use_netif; |
200 | bool ipsec_needs_netagent; |
201 | #endif // IPSEC_NEXUS |
202 | }; |
203 | |
204 | TAILQ_HEAD(ipsec_list, ipsec_pcb) ipsec_head; |
205 | |
206 | #define IPSEC_PCB_ZONE_MAX 32 |
207 | #define IPSEC_PCB_ZONE_NAME "net.if_ipsec" |
208 | |
209 | static unsigned int ipsec_pcb_size; /* size of zone element */ |
210 | static struct zone *ipsec_pcb_zone; /* zone for ipsec_pcb */ |
211 | |
212 | #define IPSECQ_MAXLEN 256 |
213 | |
214 | #if IPSEC_NEXUS |
215 | static int |
216 | sysctl_if_ipsec_ring_size SYSCTL_HANDLER_ARGS |
217 | { |
218 | #pragma unused(arg1, arg2) |
219 | int value = if_ipsec_ring_size; |
220 | |
221 | int error = sysctl_handle_int(oidp, &value, 0, req); |
222 | if (error || !req->newptr) { |
223 | return (error); |
224 | } |
225 | |
226 | if (value < IPSEC_IF_MIN_RING_SIZE || |
227 | value > IPSEC_IF_MAX_RING_SIZE) { |
228 | return (EINVAL); |
229 | } |
230 | |
231 | if_ipsec_ring_size = value; |
232 | |
233 | return (0); |
234 | } |
235 | |
236 | static int |
237 | sysctl_if_ipsec_tx_fsw_ring_size SYSCTL_HANDLER_ARGS |
238 | { |
239 | #pragma unused(arg1, arg2) |
240 | int value = if_ipsec_tx_fsw_ring_size; |
241 | |
242 | int error = sysctl_handle_int(oidp, &value, 0, req); |
243 | if (error || !req->newptr) { |
244 | return (error); |
245 | } |
246 | |
247 | if (value < IPSEC_IF_MIN_RING_SIZE || |
248 | value > IPSEC_IF_MAX_RING_SIZE) { |
249 | return (EINVAL); |
250 | } |
251 | |
252 | if_ipsec_tx_fsw_ring_size = value; |
253 | |
254 | return (0); |
255 | } |
256 | |
257 | static int |
258 | sysctl_if_ipsec_rx_fsw_ring_size SYSCTL_HANDLER_ARGS |
259 | { |
260 | #pragma unused(arg1, arg2) |
261 | int value = if_ipsec_rx_fsw_ring_size; |
262 | |
263 | int error = sysctl_handle_int(oidp, &value, 0, req); |
264 | if (error || !req->newptr) { |
265 | return (error); |
266 | } |
267 | |
268 | if (value < IPSEC_IF_MIN_RING_SIZE || |
269 | value > IPSEC_IF_MAX_RING_SIZE) { |
270 | return (EINVAL); |
271 | } |
272 | |
273 | if_ipsec_rx_fsw_ring_size = value; |
274 | |
275 | return (0); |
276 | } |
277 | #endif // IPSEC_NEXUS |
278 | |
279 | errno_t |
280 | ipsec_register_control(void) |
281 | { |
282 | struct kern_ctl_reg kern_ctl; |
283 | errno_t result = 0; |
284 | |
285 | /* Find a unique value for our interface family */ |
286 | result = mbuf_tag_id_find(IPSEC_CONTROL_NAME, &ipsec_family); |
287 | if (result != 0) { |
288 | printf("ipsec_register_control - mbuf_tag_id_find_internal failed: %d\n" , result); |
289 | return result; |
290 | } |
291 | |
292 | ipsec_pcb_size = sizeof(struct ipsec_pcb); |
293 | ipsec_pcb_zone = zinit(ipsec_pcb_size, |
294 | IPSEC_PCB_ZONE_MAX * ipsec_pcb_size, |
295 | 0, IPSEC_PCB_ZONE_NAME); |
296 | if (ipsec_pcb_zone == NULL) { |
297 | printf("ipsec_register_control - zinit(ipsec_pcb) failed" ); |
298 | return ENOMEM; |
299 | } |
300 | |
301 | #if IPSEC_NEXUS |
302 | ipsec_register_nexus(); |
303 | #endif // IPSEC_NEXUS |
304 | |
305 | TAILQ_INIT(&ipsec_head); |
306 | |
307 | bzero(&kern_ctl, sizeof(kern_ctl)); |
308 | strlcpy(kern_ctl.ctl_name, IPSEC_CONTROL_NAME, sizeof(kern_ctl.ctl_name)); |
309 | kern_ctl.ctl_name[sizeof(kern_ctl.ctl_name) - 1] = 0; |
310 | kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED; /* Require root */ |
311 | kern_ctl.ctl_sendsize = 64 * 1024; |
312 | kern_ctl.ctl_recvsize = 64 * 1024; |
313 | kern_ctl.ctl_bind = ipsec_ctl_bind; |
314 | kern_ctl.ctl_connect = ipsec_ctl_connect; |
315 | kern_ctl.ctl_disconnect = ipsec_ctl_disconnect; |
316 | kern_ctl.ctl_send = ipsec_ctl_send; |
317 | kern_ctl.ctl_setopt = ipsec_ctl_setopt; |
318 | kern_ctl.ctl_getopt = ipsec_ctl_getopt; |
319 | |
320 | result = ctl_register(&kern_ctl, &ipsec_kctlref); |
321 | if (result != 0) { |
322 | printf("ipsec_register_control - ctl_register failed: %d\n" , result); |
323 | return result; |
324 | } |
325 | |
326 | /* Register the protocol plumbers */ |
327 | if ((result = proto_register_plumber(PF_INET, ipsec_family, |
328 | ipsec_attach_proto, NULL)) != 0) { |
329 | printf("ipsec_register_control - proto_register_plumber(PF_INET, %d) failed: %d\n" , |
330 | ipsec_family, result); |
331 | ctl_deregister(ipsec_kctlref); |
332 | return result; |
333 | } |
334 | |
335 | /* Register the protocol plumbers */ |
336 | if ((result = proto_register_plumber(PF_INET6, ipsec_family, |
337 | ipsec_attach_proto, NULL)) != 0) { |
338 | proto_unregister_plumber(PF_INET, ipsec_family); |
339 | ctl_deregister(ipsec_kctlref); |
340 | printf("ipsec_register_control - proto_register_plumber(PF_INET6, %d) failed: %d\n" , |
341 | ipsec_family, result); |
342 | return result; |
343 | } |
344 | |
345 | ipsec_lck_attr = lck_attr_alloc_init(); |
346 | ipsec_lck_grp_attr = lck_grp_attr_alloc_init(); |
347 | ipsec_lck_grp = lck_grp_alloc_init("ipsec" , ipsec_lck_grp_attr); |
348 | lck_mtx_init(&ipsec_lock, ipsec_lck_grp, ipsec_lck_attr); |
349 | |
350 | return 0; |
351 | } |
352 | |
353 | /* Helpers */ |
354 | int |
355 | ipsec_interface_isvalid (ifnet_t interface) |
356 | { |
357 | struct ipsec_pcb *pcb = NULL; |
358 | |
359 | if (interface == NULL) |
360 | return 0; |
361 | |
362 | pcb = ifnet_softc(interface); |
363 | |
364 | if (pcb == NULL) |
365 | return 0; |
366 | |
367 | /* When ctl disconnects, ipsec_unit is set to 0 */ |
368 | if (pcb->ipsec_unit == 0) |
369 | return 0; |
370 | |
371 | return 1; |
372 | } |
373 | |
374 | #if IPSEC_NEXUS |
375 | boolean_t |
376 | ipsec_interface_needs_netagent(ifnet_t interface) |
377 | { |
378 | struct ipsec_pcb *pcb = NULL; |
379 | |
380 | if (interface == NULL) { |
381 | return (FALSE); |
382 | } |
383 | |
384 | pcb = ifnet_softc(interface); |
385 | |
386 | if (pcb == NULL) { |
387 | return (FALSE); |
388 | } |
389 | |
390 | return (pcb->ipsec_needs_netagent == true); |
391 | } |
392 | #endif // IPSEC_NEXUS |
393 | |
394 | static errno_t |
395 | ipsec_ifnet_set_attrs(ifnet_t ifp) |
396 | { |
397 | /* Set flags and additional information. */ |
398 | ifnet_set_mtu(ifp, 1500); |
399 | ifnet_set_flags(ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff); |
400 | |
401 | /* The interface must generate its own IPv6 LinkLocal address, |
402 | * if possible following the recommendation of RFC2472 to the 64bit interface ID |
403 | */ |
404 | ifnet_set_eflags(ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL); |
405 | |
406 | #if !IPSEC_NEXUS |
407 | /* Reset the stats in case as the interface may have been recycled */ |
408 | struct ifnet_stats_param stats; |
409 | bzero(&stats, sizeof(struct ifnet_stats_param)); |
410 | ifnet_set_stat(ifp, &stats); |
411 | #endif // !IPSEC_NEXUS |
412 | |
413 | return (0); |
414 | } |
415 | |
416 | #if IPSEC_NEXUS |
417 | |
418 | static uuid_t ipsec_nx_dom_prov; |
419 | |
420 | static errno_t |
421 | ipsec_nxdp_init(__unused kern_nexus_domain_provider_t domprov) |
422 | { |
423 | return 0; |
424 | } |
425 | |
426 | static void |
427 | ipsec_nxdp_fini(__unused kern_nexus_domain_provider_t domprov) |
428 | { |
429 | // Ignore |
430 | } |
431 | |
432 | static errno_t |
433 | ipsec_register_nexus(void) |
434 | { |
435 | const struct kern_nexus_domain_provider_init dp_init = { |
436 | .nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION, |
437 | .nxdpi_flags = 0, |
438 | .nxdpi_init = ipsec_nxdp_init, |
439 | .nxdpi_fini = ipsec_nxdp_fini |
440 | }; |
441 | errno_t err = 0; |
442 | |
443 | /* ipsec_nxdp_init() is called before this function returns */ |
444 | err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF, |
445 | (const uint8_t *) "com.apple.ipsec" , |
446 | &dp_init, sizeof(dp_init), |
447 | &ipsec_nx_dom_prov); |
448 | if (err != 0) { |
449 | printf("%s: failed to register domain provider\n" , __func__); |
450 | return (err); |
451 | } |
452 | return (0); |
453 | } |
454 | |
455 | static errno_t |
456 | ipsec_netif_prepare(kern_nexus_t nexus, ifnet_t ifp) |
457 | { |
458 | struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); |
459 | pcb->ipsec_netif_nexus = nexus; |
460 | return (ipsec_ifnet_set_attrs(ifp)); |
461 | } |
462 | |
463 | static errno_t |
464 | ipsec_nexus_pre_connect(kern_nexus_provider_t nxprov, |
465 | proc_t p, kern_nexus_t nexus, |
466 | nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx) |
467 | { |
468 | #pragma unused(nxprov, p) |
469 | #pragma unused(nexus, nexus_port, channel, ch_ctx) |
470 | return (0); |
471 | } |
472 | |
473 | static errno_t |
474 | ipsec_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
475 | kern_channel_t channel) |
476 | { |
477 | #pragma unused(nxprov, channel) |
478 | struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); |
479 | boolean_t ok = ifnet_is_attached(pcb->ipsec_ifp, 1); |
480 | return (ok ? 0 : ENXIO); |
481 | } |
482 | |
483 | static void |
484 | ipsec_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
485 | kern_channel_t channel) |
486 | { |
487 | #pragma unused(nxprov, nexus, channel) |
488 | } |
489 | |
490 | static void |
491 | ipsec_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
492 | kern_channel_t channel) |
493 | { |
494 | #pragma unused(nxprov, nexus, channel) |
495 | } |
496 | |
497 | static void |
498 | ipsec_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
499 | kern_channel_t channel) |
500 | { |
501 | #pragma unused(nxprov, channel) |
502 | struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); |
503 | if (pcb->ipsec_netif_nexus == nexus) { |
504 | pcb->ipsec_netif_nexus = NULL; |
505 | } |
506 | ifnet_decr_iorefcnt(pcb->ipsec_ifp); |
507 | } |
508 | |
509 | static errno_t |
510 | ipsec_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
511 | kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring, |
512 | void **ring_ctx) |
513 | { |
514 | #pragma unused(nxprov) |
515 | #pragma unused(channel) |
516 | #pragma unused(ring_ctx) |
517 | struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); |
518 | if (!is_tx_ring) { |
519 | VERIFY(pcb->ipsec_kpipe_rxring == NULL); |
520 | pcb->ipsec_kpipe_rxring = ring; |
521 | } else { |
522 | VERIFY(pcb->ipsec_kpipe_txring == NULL); |
523 | pcb->ipsec_kpipe_txring = ring; |
524 | } |
525 | return 0; |
526 | } |
527 | |
528 | static void |
529 | ipsec_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
530 | kern_channel_ring_t ring) |
531 | { |
532 | #pragma unused(nxprov) |
533 | struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); |
534 | if (pcb->ipsec_kpipe_rxring == ring) { |
535 | pcb->ipsec_kpipe_rxring = NULL; |
536 | } else if (pcb->ipsec_kpipe_txring == ring) { |
537 | pcb->ipsec_kpipe_txring = NULL; |
538 | } |
539 | } |
540 | |
541 | static errno_t |
542 | ipsec_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
543 | kern_channel_ring_t tx_ring, uint32_t flags) |
544 | { |
545 | #pragma unused(nxprov) |
546 | #pragma unused(flags) |
547 | struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); |
548 | |
549 | lck_rw_lock_shared(&pcb->ipsec_pcb_lock); |
550 | int channel_enabled = pcb->ipsec_kpipe_enabled; |
551 | if (!channel_enabled) { |
552 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
553 | return 0; |
554 | } |
555 | |
556 | kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); |
557 | if (tx_slot == NULL) { |
558 | // Nothing to write, bail |
559 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
560 | return 0; |
561 | } |
562 | |
563 | // Signal the netif ring to read |
564 | kern_channel_ring_t rx_ring = pcb->ipsec_netif_rxring; |
565 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
566 | |
567 | if (rx_ring != NULL) { |
568 | kern_channel_notify(rx_ring, 0); |
569 | } |
570 | return 0; |
571 | } |
572 | |
573 | static mbuf_t |
574 | ipsec_encrypt_mbuf(ifnet_t interface, |
575 | mbuf_t data) |
576 | { |
577 | struct ipsec_output_state ipsec_state; |
578 | int error = 0; |
579 | uint32_t af; |
580 | |
581 | // Make sure this packet isn't looping through the interface |
582 | if (necp_get_last_interface_index_from_packet(data) == interface->if_index) { |
583 | error = -1; |
584 | goto ipsec_output_err; |
585 | } |
586 | |
587 | // Mark the interface so NECP can evaluate tunnel policy |
588 | necp_mark_packet_from_interface(data, interface); |
589 | |
590 | struct ip *ip = mtod(data, struct ip *); |
591 | u_int ip_version = ip->ip_v; |
592 | |
593 | switch (ip_version) { |
594 | case 4: { |
595 | af = AF_INET; |
596 | |
597 | memset(&ipsec_state, 0, sizeof(ipsec_state)); |
598 | ipsec_state.m = data; |
599 | ipsec_state.dst = (struct sockaddr *)&ip->ip_dst; |
600 | memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro)); |
601 | |
602 | error = ipsec4_interface_output(&ipsec_state, interface); |
603 | if (error == 0 && ipsec_state.tunneled == 6) { |
604 | // Tunneled in IPv6 - packet is gone |
605 | // TODO: Don't lose mbuf |
606 | data = NULL; |
607 | goto done; |
608 | } |
609 | |
610 | data = ipsec_state.m; |
611 | if (error || data == NULL) { |
612 | if (error) { |
613 | printf("ipsec_encrypt_mbuf: ipsec4_output error %d\n" , error); |
614 | } |
615 | goto ipsec_output_err; |
616 | } |
617 | goto done; |
618 | } |
619 | case 6: { |
620 | af = AF_INET6; |
621 | |
622 | data = ipsec6_splithdr(data); |
623 | if (data == NULL) { |
624 | printf("ipsec_encrypt_mbuf: ipsec6_splithdr returned NULL\n" ); |
625 | goto ipsec_output_err; |
626 | } |
627 | |
628 | struct ip6_hdr *ip6 = mtod(data, struct ip6_hdr *); |
629 | |
630 | memset(&ipsec_state, 0, sizeof(ipsec_state)); |
631 | ipsec_state.m = data; |
632 | ipsec_state.dst = (struct sockaddr *)&ip6->ip6_dst; |
633 | memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro)); |
634 | |
635 | error = ipsec6_interface_output(&ipsec_state, interface, &ip6->ip6_nxt, ipsec_state.m); |
636 | if (error == 0 && ipsec_state.tunneled == 4) { |
637 | // Tunneled in IPv4 - packet is gone |
638 | // TODO: Don't lose mbuf |
639 | data = NULL; |
640 | goto done; |
641 | } |
642 | data = ipsec_state.m; |
643 | if (error || data == NULL) { |
644 | if (error) { |
645 | printf("ipsec_encrypt_mbuf: ipsec6_output error %d\n" , error); |
646 | } |
647 | goto ipsec_output_err; |
648 | } |
649 | goto done; |
650 | } |
651 | default: { |
652 | printf("ipsec_encrypt_mbuf: Received unknown packet version %d\n" , ip_version); |
653 | error = -1; |
654 | goto ipsec_output_err; |
655 | } |
656 | } |
657 | |
658 | done: |
659 | return data; |
660 | |
661 | ipsec_output_err: |
662 | if (data) { |
663 | mbuf_freem(data); |
664 | } |
665 | return NULL; |
666 | } |
667 | |
668 | static errno_t |
669 | ipsec_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
670 | kern_channel_ring_t rx_ring, uint32_t flags) |
671 | { |
672 | #pragma unused(nxprov) |
673 | #pragma unused(flags) |
674 | struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); |
675 | struct kern_channel_ring_stat_increment rx_ring_stats; |
676 | |
677 | lck_rw_lock_shared(&pcb->ipsec_pcb_lock); |
678 | |
679 | int channel_enabled = pcb->ipsec_kpipe_enabled; |
680 | if (!channel_enabled) { |
681 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
682 | return 0; |
683 | } |
684 | |
685 | // Reclaim user-released slots |
686 | (void) kern_channel_reclaim(rx_ring); |
687 | |
688 | uint32_t avail = kern_channel_available_slot_count(rx_ring); |
689 | if (avail == 0) { |
690 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
691 | return 0; |
692 | } |
693 | |
694 | kern_channel_ring_t tx_ring = pcb->ipsec_netif_txring; |
695 | if (tx_ring == NULL) { |
696 | // Net-If TX ring not set up yet, nothing to read |
697 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
698 | return 0; |
699 | } |
700 | |
701 | struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->ipsec_netif_nexus)->nif_stats; |
702 | |
703 | // Unlock ipsec before entering ring |
704 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
705 | |
706 | (void)kr_enter(tx_ring, TRUE); |
707 | |
708 | // Lock again after entering and validate |
709 | lck_rw_lock_shared(&pcb->ipsec_pcb_lock); |
710 | if (tx_ring != pcb->ipsec_netif_txring) { |
711 | // Ring no longer valid |
712 | // Unlock first, then exit ring |
713 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
714 | kr_exit(tx_ring); |
715 | return 0; |
716 | } |
717 | |
718 | |
719 | struct kern_channel_ring_stat_increment tx_ring_stats; |
720 | bzero(&tx_ring_stats, sizeof(tx_ring_stats)); |
721 | kern_channel_slot_t tx_pslot = NULL; |
722 | kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); |
723 | if (tx_slot == NULL) { |
724 | // Nothing to read, don't bother signalling |
725 | // Unlock first, then exit ring |
726 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
727 | kr_exit(tx_ring); |
728 | return 0; |
729 | } |
730 | |
731 | struct kern_pbufpool *rx_pp = rx_ring->ckr_pp; |
732 | VERIFY(rx_pp != NULL); |
733 | bzero(&rx_ring_stats, sizeof(rx_ring_stats)); |
734 | kern_channel_slot_t rx_pslot = NULL; |
735 | kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL); |
736 | |
737 | while (rx_slot != NULL && tx_slot != NULL) { |
738 | size_t length = 0; |
739 | mbuf_t data = NULL; |
740 | errno_t error = 0; |
741 | |
742 | // Allocate rx packet |
743 | kern_packet_t rx_ph = 0; |
744 | error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); |
745 | if (__improbable(error != 0)) { |
746 | printf("ipsec_kpipe_sync_rx %s: failed to allocate packet\n" , |
747 | pcb->ipsec_ifp->if_xname); |
748 | break; |
749 | } |
750 | |
751 | kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); |
752 | |
753 | // Advance TX ring |
754 | tx_pslot = tx_slot; |
755 | tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); |
756 | |
757 | if (tx_ph == 0) { |
758 | kern_pbufpool_free(rx_pp, rx_ph); |
759 | continue; |
760 | } |
761 | |
762 | kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); |
763 | VERIFY(tx_buf != NULL); |
764 | uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); |
765 | VERIFY(tx_baddr != NULL); |
766 | tx_baddr += kern_buflet_get_data_offset(tx_buf); |
767 | |
768 | bpf_tap_packet_out(pcb->ipsec_ifp, DLT_RAW, tx_ph, NULL, 0); |
769 | |
770 | length = MIN(kern_packet_get_data_length(tx_ph), |
771 | pcb->ipsec_slot_size); |
772 | |
773 | // Increment TX stats |
774 | tx_ring_stats.kcrsi_slots_transferred++; |
775 | tx_ring_stats.kcrsi_bytes_transferred += length; |
776 | |
777 | if (length > 0) { |
778 | error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data); |
779 | if (error == 0) { |
780 | error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT); |
781 | if (error == 0) { |
782 | // Encrypt and send packet |
783 | data = ipsec_encrypt_mbuf(pcb->ipsec_ifp, data); |
784 | } else { |
785 | printf("ipsec_kpipe_sync_rx %s - mbuf_copyback(%zu) error %d\n" , pcb->ipsec_ifp->if_xname, length, error); |
786 | STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); |
787 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
788 | mbuf_freem(data); |
789 | data = NULL; |
790 | } |
791 | } else { |
792 | printf("ipsec_kpipe_sync_rx %s - mbuf_gethdr error %d\n" , pcb->ipsec_ifp->if_xname, error); |
793 | STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); |
794 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
795 | } |
796 | } else { |
797 | printf("ipsec_kpipe_sync_rx %s - 0 length packet\n" , pcb->ipsec_ifp->if_xname); |
798 | STATS_INC(nifs, NETIF_STATS_BADLEN); |
799 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
800 | } |
801 | |
802 | if (data == NULL) { |
803 | printf("ipsec_kpipe_sync_rx %s: no encrypted packet to send\n" , pcb->ipsec_ifp->if_xname); |
804 | kern_pbufpool_free(rx_pp, rx_ph); |
805 | break; |
806 | } |
807 | |
808 | length = mbuf_pkthdr_len(data); |
809 | if (length > rx_pp->pp_buflet_size) { |
810 | // Flush data |
811 | mbuf_freem(data); |
812 | kern_pbufpool_free(rx_pp, rx_ph); |
813 | printf("ipsec_kpipe_sync_rx %s: encrypted packet length %zu > %u\n" , |
814 | pcb->ipsec_ifp->if_xname, length, rx_pp->pp_buflet_size); |
815 | continue; |
816 | } |
817 | |
818 | // Fillout rx packet |
819 | kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); |
820 | VERIFY(rx_buf != NULL); |
821 | void *rx_baddr = kern_buflet_get_object_address(rx_buf); |
822 | VERIFY(rx_baddr != NULL); |
823 | |
824 | // Copy-in data from mbuf to buflet |
825 | mbuf_copydata(data, 0, length, (void *)rx_baddr); |
826 | kern_packet_clear_flow_uuid(rx_ph); // Zero flow id |
827 | |
828 | // Finalize and attach the packet |
829 | error = kern_buflet_set_data_offset(rx_buf, 0); |
830 | VERIFY(error == 0); |
831 | error = kern_buflet_set_data_length(rx_buf, length); |
832 | VERIFY(error == 0); |
833 | error = kern_packet_finalize(rx_ph); |
834 | VERIFY(error == 0); |
835 | error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); |
836 | VERIFY(error == 0); |
837 | |
838 | STATS_INC(nifs, NETIF_STATS_TXPKTS); |
839 | STATS_INC(nifs, NETIF_STATS_TXCOPY_DIRECT); |
840 | |
841 | rx_ring_stats.kcrsi_slots_transferred++; |
842 | rx_ring_stats.kcrsi_bytes_transferred += length; |
843 | |
844 | if (!pcb->ipsec_ext_ifdata_stats) { |
845 | ifnet_stat_increment_out(pcb->ipsec_ifp, 1, length, 0); |
846 | } |
847 | |
848 | mbuf_freem(data); |
849 | |
850 | rx_pslot = rx_slot; |
851 | rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); |
852 | } |
853 | |
854 | if (rx_pslot) { |
855 | kern_channel_advance_slot(rx_ring, rx_pslot); |
856 | kern_channel_increment_ring_net_stats(rx_ring, pcb->ipsec_ifp, &rx_ring_stats); |
857 | } |
858 | |
859 | if (tx_pslot) { |
860 | kern_channel_advance_slot(tx_ring, tx_pslot); |
861 | kern_channel_increment_ring_net_stats(tx_ring, pcb->ipsec_ifp, &tx_ring_stats); |
862 | (void)kern_channel_reclaim(tx_ring); |
863 | } |
864 | |
865 | /* always reenable output */ |
866 | errno_t error = ifnet_enable_output(pcb->ipsec_ifp); |
867 | if (error != 0) { |
868 | printf("ipsec_kpipe_sync_rx: ifnet_enable_output returned error %d\n" , error); |
869 | } |
870 | |
871 | // Unlock first, then exit ring |
872 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
873 | |
874 | if (tx_pslot != NULL) { |
875 | kern_channel_notify(tx_ring, 0); |
876 | } |
877 | kr_exit(tx_ring); |
878 | |
879 | return 0; |
880 | } |
881 | |
882 | static errno_t |
883 | ipsec_netif_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
884 | kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring, |
885 | void **ring_ctx) |
886 | { |
887 | #pragma unused(nxprov) |
888 | #pragma unused(channel) |
889 | #pragma unused(ring_ctx) |
890 | struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); |
891 | if (!is_tx_ring) { |
892 | VERIFY(pcb->ipsec_netif_rxring == NULL); |
893 | pcb->ipsec_netif_rxring = ring; |
894 | } else { |
895 | VERIFY(pcb->ipsec_netif_txring == NULL); |
896 | pcb->ipsec_netif_txring = ring; |
897 | } |
898 | return 0; |
899 | } |
900 | |
901 | static void |
902 | ipsec_netif_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
903 | kern_channel_ring_t ring) |
904 | { |
905 | #pragma unused(nxprov) |
906 | struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); |
907 | if (pcb->ipsec_netif_rxring == ring) { |
908 | pcb->ipsec_netif_rxring = NULL; |
909 | } else if (pcb->ipsec_netif_txring == ring) { |
910 | pcb->ipsec_netif_txring = NULL; |
911 | } |
912 | } |
913 | |
914 | static bool |
915 | ipsec_netif_check_policy(mbuf_t data) |
916 | { |
917 | necp_kernel_policy_result necp_result = 0; |
918 | necp_kernel_policy_result_parameter necp_result_parameter = {}; |
919 | uint32_t necp_matched_policy_id = 0; |
920 | |
921 | // This packet has been marked with IP level policy, do not mark again. |
922 | if (data && data->m_pkthdr.necp_mtag.necp_policy_id >= NECP_KERNEL_POLICY_ID_FIRST_VALID_IP) { |
923 | return (true); |
924 | } |
925 | |
926 | size_t length = mbuf_pkthdr_len(data); |
927 | if (length < sizeof(struct ip)) { |
928 | return (false); |
929 | } |
930 | |
931 | struct ip *ip = mtod(data, struct ip *); |
932 | u_int ip_version = ip->ip_v; |
933 | switch (ip_version) { |
934 | case 4: { |
935 | necp_matched_policy_id = necp_ip_output_find_policy_match(data, 0, NULL, |
936 | &necp_result, &necp_result_parameter); |
937 | break; |
938 | } |
939 | case 6: { |
940 | necp_matched_policy_id = necp_ip6_output_find_policy_match(data, 0, NULL, |
941 | &necp_result, &necp_result_parameter); |
942 | break; |
943 | } |
944 | default: { |
945 | return (false); |
946 | } |
947 | } |
948 | |
949 | if (necp_result == NECP_KERNEL_POLICY_RESULT_DROP || |
950 | necp_result == NECP_KERNEL_POLICY_RESULT_SOCKET_DIVERT) { |
951 | /* Drop and flow divert packets should be blocked at the IP layer */ |
952 | return (false); |
953 | } |
954 | |
955 | necp_mark_packet_from_ip(data, necp_matched_policy_id); |
956 | return (true); |
957 | } |
958 | |
959 | static errno_t |
960 | ipsec_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
961 | kern_channel_ring_t tx_ring, uint32_t flags) |
962 | { |
963 | #pragma unused(nxprov) |
964 | #pragma unused(flags) |
965 | struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); |
966 | |
967 | struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats; |
968 | |
969 | lck_rw_lock_shared(&pcb->ipsec_pcb_lock); |
970 | |
971 | struct kern_channel_ring_stat_increment tx_ring_stats; |
972 | bzero(&tx_ring_stats, sizeof(tx_ring_stats)); |
973 | kern_channel_slot_t tx_pslot = NULL; |
974 | kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); |
975 | |
976 | STATS_INC(nifs, NETIF_STATS_TXSYNC); |
977 | |
978 | if (tx_slot == NULL) { |
979 | // Nothing to write, don't bother signalling |
980 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
981 | return 0; |
982 | } |
983 | |
984 | if (pcb->ipsec_kpipe_enabled) { |
985 | kern_channel_ring_t rx_ring = pcb->ipsec_kpipe_rxring; |
986 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
987 | |
988 | // Signal the kernel pipe ring to read |
989 | if (rx_ring != NULL) { |
990 | kern_channel_notify(rx_ring, 0); |
991 | } |
992 | return 0; |
993 | } |
994 | |
995 | // If we're here, we're injecting into the BSD stack |
996 | while (tx_slot != NULL) { |
997 | size_t length = 0; |
998 | mbuf_t data = NULL; |
999 | |
1000 | kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); |
1001 | |
1002 | // Advance TX ring |
1003 | tx_pslot = tx_slot; |
1004 | tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); |
1005 | |
1006 | if (tx_ph == 0) { |
1007 | continue; |
1008 | } |
1009 | |
1010 | kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); |
1011 | VERIFY(tx_buf != NULL); |
1012 | uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); |
1013 | VERIFY(tx_baddr != 0); |
1014 | tx_baddr += kern_buflet_get_data_offset(tx_buf); |
1015 | |
1016 | bpf_tap_packet_out(pcb->ipsec_ifp, DLT_RAW, tx_ph, NULL, 0); |
1017 | |
1018 | length = MIN(kern_packet_get_data_length(tx_ph), |
1019 | pcb->ipsec_slot_size); |
1020 | |
1021 | if (length > 0) { |
1022 | errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data); |
1023 | if (error == 0) { |
1024 | error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT); |
1025 | if (error == 0) { |
1026 | // Mark packet from policy |
1027 | uint32_t policy_id = kern_packet_get_policy_id(tx_ph); |
1028 | necp_mark_packet_from_ip(data, policy_id); |
1029 | |
1030 | // Check policy with NECP |
1031 | if (!ipsec_netif_check_policy(data)) { |
1032 | printf("ipsec_netif_sync_tx %s - failed policy check\n" , pcb->ipsec_ifp->if_xname); |
1033 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
1034 | mbuf_freem(data); |
1035 | data = NULL; |
1036 | } else { |
1037 | // Send through encryption |
1038 | error = ipsec_output(pcb->ipsec_ifp, data); |
1039 | if (error != 0) { |
1040 | printf("ipsec_netif_sync_tx %s - ipsec_output error %d\n" , pcb->ipsec_ifp->if_xname, error); |
1041 | } |
1042 | } |
1043 | } else { |
1044 | printf("ipsec_netif_sync_tx %s - mbuf_copyback(%zu) error %d\n" , pcb->ipsec_ifp->if_xname, length, error); |
1045 | STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); |
1046 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
1047 | mbuf_freem(data); |
1048 | data = NULL; |
1049 | } |
1050 | } else { |
1051 | printf("ipsec_netif_sync_tx %s - mbuf_gethdr error %d\n" , pcb->ipsec_ifp->if_xname, error); |
1052 | STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); |
1053 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
1054 | } |
1055 | } else { |
1056 | printf("ipsec_netif_sync_tx %s - 0 length packet\n" , pcb->ipsec_ifp->if_xname); |
1057 | STATS_INC(nifs, NETIF_STATS_BADLEN); |
1058 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
1059 | } |
1060 | |
1061 | if (data == NULL) { |
1062 | printf("ipsec_netif_sync_tx %s: no encrypted packet to send\n" , pcb->ipsec_ifp->if_xname); |
1063 | break; |
1064 | } |
1065 | |
1066 | STATS_INC(nifs, NETIF_STATS_TXPKTS); |
1067 | STATS_INC(nifs, NETIF_STATS_TXCOPY_MBUF); |
1068 | |
1069 | tx_ring_stats.kcrsi_slots_transferred++; |
1070 | tx_ring_stats.kcrsi_bytes_transferred += length; |
1071 | } |
1072 | |
1073 | if (tx_pslot) { |
1074 | kern_channel_advance_slot(tx_ring, tx_pslot); |
1075 | kern_channel_increment_ring_net_stats(tx_ring, pcb->ipsec_ifp, &tx_ring_stats); |
1076 | (void)kern_channel_reclaim(tx_ring); |
1077 | } |
1078 | |
1079 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
1080 | |
1081 | return 0; |
1082 | } |
1083 | |
1084 | static errno_t |
1085 | ipsec_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
1086 | kern_channel_ring_t ring, __unused uint32_t flags) |
1087 | { |
1088 | #pragma unused(nxprov) |
1089 | struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); |
1090 | boolean_t more = false; |
1091 | errno_t rc = 0; |
1092 | |
1093 | /* |
1094 | * Refill and sync the ring; we may be racing against another thread doing |
1095 | * an RX sync that also wants to do kr_enter(), and so use the blocking |
1096 | * variant here. |
1097 | */ |
1098 | rc = kern_channel_tx_refill_canblock(ring, UINT32_MAX, UINT32_MAX, true, &more); |
1099 | if (rc != 0 && rc != EAGAIN && rc != EBUSY) { |
1100 | printf("%s, tx refill failed %d\n" , __func__, rc); |
1101 | } |
1102 | |
1103 | (void) kr_enter(ring, TRUE); |
1104 | lck_rw_lock_shared(&pcb->ipsec_pcb_lock); |
1105 | |
1106 | if (pcb->ipsec_kpipe_enabled) { |
1107 | uint32_t tx_available = kern_channel_available_slot_count(ring); |
1108 | if (pcb->ipsec_netif_txring_size > 0 && |
1109 | tx_available >= pcb->ipsec_netif_txring_size - 1) { |
1110 | // No room left in tx ring, disable output for now |
1111 | errno_t error = ifnet_disable_output(pcb->ipsec_ifp); |
1112 | if (error != 0) { |
1113 | printf("ipsec_netif_tx_doorbell: ifnet_disable_output returned error %d\n" , error); |
1114 | } |
1115 | } |
1116 | } |
1117 | |
1118 | if (pcb->ipsec_kpipe_enabled) { |
1119 | kern_channel_ring_t rx_ring = pcb->ipsec_kpipe_rxring; |
1120 | |
1121 | // Unlock while calling notify |
1122 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
1123 | // Signal the kernel pipe ring to read |
1124 | if (rx_ring != NULL) { |
1125 | kern_channel_notify(rx_ring, 0); |
1126 | } |
1127 | } else { |
1128 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
1129 | } |
1130 | |
1131 | kr_exit(ring); |
1132 | |
1133 | return (0); |
1134 | } |
1135 | |
1136 | static errno_t |
1137 | ipsec_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, |
1138 | kern_channel_ring_t rx_ring, uint32_t flags) |
1139 | { |
1140 | #pragma unused(nxprov) |
1141 | #pragma unused(flags) |
1142 | struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); |
1143 | struct kern_channel_ring_stat_increment rx_ring_stats; |
1144 | |
1145 | struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats; |
1146 | |
1147 | lck_rw_lock_shared(&pcb->ipsec_pcb_lock); |
1148 | |
1149 | // Reclaim user-released slots |
1150 | (void) kern_channel_reclaim(rx_ring); |
1151 | |
1152 | STATS_INC(nifs, NETIF_STATS_RXSYNC); |
1153 | |
1154 | uint32_t avail = kern_channel_available_slot_count(rx_ring); |
1155 | if (avail == 0) { |
1156 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
1157 | return 0; |
1158 | } |
1159 | |
1160 | struct kern_pbufpool *rx_pp = rx_ring->ckr_pp; |
1161 | VERIFY(rx_pp != NULL); |
1162 | bzero(&rx_ring_stats, sizeof(rx_ring_stats)); |
1163 | kern_channel_slot_t rx_pslot = NULL; |
1164 | kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL); |
1165 | |
1166 | while (rx_slot != NULL) { |
1167 | // Check for a waiting packet |
1168 | lck_mtx_lock(&pcb->ipsec_input_chain_lock); |
1169 | mbuf_t data = pcb->ipsec_input_chain; |
1170 | if (data == NULL) { |
1171 | lck_mtx_unlock(&pcb->ipsec_input_chain_lock); |
1172 | break; |
1173 | } |
1174 | |
1175 | // Allocate rx packet |
1176 | kern_packet_t rx_ph = 0; |
1177 | errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); |
1178 | if (__improbable(error != 0)) { |
1179 | STATS_INC(nifs, NETIF_STATS_NOMEM_PKT); |
1180 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
1181 | lck_mtx_unlock(&pcb->ipsec_input_chain_lock); |
1182 | break; |
1183 | } |
1184 | |
1185 | // Advance waiting packets |
1186 | pcb->ipsec_input_chain = data->m_nextpkt; |
1187 | data->m_nextpkt = NULL; |
1188 | if (pcb->ipsec_input_chain == NULL) { |
1189 | pcb->ipsec_input_chain_last = NULL; |
1190 | } |
1191 | lck_mtx_unlock(&pcb->ipsec_input_chain_lock); |
1192 | |
1193 | size_t length = mbuf_pkthdr_len(data); |
1194 | |
1195 | if (length < sizeof(struct ip)) { |
1196 | // Flush data |
1197 | mbuf_freem(data); |
1198 | kern_pbufpool_free(rx_pp, rx_ph); |
1199 | STATS_INC(nifs, NETIF_STATS_BADLEN); |
1200 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
1201 | printf("ipsec_netif_sync_rx %s: legacy decrypted packet length cannot hold IP %zu < %zu\n" , |
1202 | pcb->ipsec_ifp->if_xname, length, sizeof(struct ip)); |
1203 | continue; |
1204 | } |
1205 | |
1206 | uint32_t af = 0; |
1207 | struct ip *ip = mtod(data, struct ip *); |
1208 | u_int ip_version = ip->ip_v; |
1209 | switch (ip_version) { |
1210 | case 4: { |
1211 | af = AF_INET; |
1212 | break; |
1213 | } |
1214 | case 6: { |
1215 | af = AF_INET6; |
1216 | break; |
1217 | } |
1218 | default: { |
1219 | printf("ipsec_netif_sync_rx %s: legacy unknown ip version %u\n" , |
1220 | pcb->ipsec_ifp->if_xname, ip_version); |
1221 | break; |
1222 | } |
1223 | } |
1224 | |
1225 | if (length > rx_pp->pp_buflet_size || |
1226 | (pcb->ipsec_frag_size_set && length > pcb->ipsec_input_frag_size)) { |
1227 | |
1228 | // We need to fragment to send up into the netif |
1229 | |
1230 | u_int32_t fragment_mtu = rx_pp->pp_buflet_size; |
1231 | if (pcb->ipsec_frag_size_set && |
1232 | pcb->ipsec_input_frag_size < rx_pp->pp_buflet_size) { |
1233 | fragment_mtu = pcb->ipsec_input_frag_size; |
1234 | } |
1235 | |
1236 | mbuf_t fragment_chain = NULL; |
1237 | switch (af) { |
1238 | case AF_INET: { |
1239 | // ip_fragment expects the length in host order |
1240 | ip->ip_len = ntohs(ip->ip_len); |
1241 | |
1242 | // ip_fragment will modify the original data, don't free |
1243 | int fragment_error = ip_fragment(data, pcb->ipsec_ifp, fragment_mtu, TRUE); |
1244 | if (fragment_error == 0 && data != NULL) { |
1245 | fragment_chain = data; |
1246 | } else { |
1247 | STATS_INC(nifs, NETIF_STATS_BADLEN); |
1248 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
1249 | printf("ipsec_netif_sync_rx %s: failed to fragment IPv4 packet of length %zu (%d)\n" , |
1250 | pcb->ipsec_ifp->if_xname, length, fragment_error); |
1251 | } |
1252 | break; |
1253 | } |
1254 | case AF_INET6: { |
1255 | if (length < sizeof(struct ip6_hdr)) { |
1256 | mbuf_freem(data); |
1257 | STATS_INC(nifs, NETIF_STATS_BADLEN); |
1258 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
1259 | printf("ipsec_netif_sync_rx %s: failed to fragment IPv6 packet of length %zu < %zu\n" , |
1260 | pcb->ipsec_ifp->if_xname, length, sizeof(struct ip6_hdr)); |
1261 | } else { |
1262 | |
1263 | // ip6_do_fragmentation will free the original data on success only |
1264 | struct ip6_hdr *ip6 = mtod(data, struct ip6_hdr *); |
1265 | struct ip6_exthdrs exthdrs; |
1266 | memset(&exthdrs, 0, sizeof(exthdrs)); |
1267 | |
1268 | int fragment_error = ip6_do_fragmentation(&data, 0, pcb->ipsec_ifp, sizeof(struct ip6_hdr), |
1269 | ip6, &exthdrs, fragment_mtu, ip6->ip6_nxt); |
1270 | if (fragment_error == 0 && data != NULL) { |
1271 | fragment_chain = data; |
1272 | } else { |
1273 | mbuf_freem(data); |
1274 | STATS_INC(nifs, NETIF_STATS_BADLEN); |
1275 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
1276 | printf("ipsec_netif_sync_rx %s: failed to fragment IPv6 packet of length %zu (%d)\n" , |
1277 | pcb->ipsec_ifp->if_xname, length, fragment_error); |
1278 | } |
1279 | } |
1280 | break; |
1281 | } |
1282 | default: { |
1283 | // Cannot fragment unknown families |
1284 | mbuf_freem(data); |
1285 | STATS_INC(nifs, NETIF_STATS_BADLEN); |
1286 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
1287 | printf("ipsec_netif_sync_rx %s: uknown legacy decrypted packet length %zu > %u\n" , |
1288 | pcb->ipsec_ifp->if_xname, length, rx_pp->pp_buflet_size); |
1289 | break; |
1290 | } |
1291 | } |
1292 | |
1293 | if (fragment_chain != NULL) { |
1294 | // Add fragments to chain before continuing |
1295 | lck_mtx_lock(&pcb->ipsec_input_chain_lock); |
1296 | if (pcb->ipsec_input_chain != NULL) { |
1297 | pcb->ipsec_input_chain_last->m_nextpkt = fragment_chain; |
1298 | } else { |
1299 | pcb->ipsec_input_chain = fragment_chain; |
1300 | } |
1301 | while (fragment_chain->m_nextpkt) { |
1302 | VERIFY(fragment_chain != fragment_chain->m_nextpkt); |
1303 | fragment_chain = fragment_chain->m_nextpkt; |
1304 | } |
1305 | pcb->ipsec_input_chain_last = fragment_chain; |
1306 | lck_mtx_unlock(&pcb->ipsec_input_chain_lock); |
1307 | } |
1308 | |
1309 | // Make sure to free unused rx packet |
1310 | kern_pbufpool_free(rx_pp, rx_ph); |
1311 | |
1312 | continue; |
1313 | } |
1314 | |
1315 | mbuf_pkthdr_setrcvif(data, pcb->ipsec_ifp); |
1316 | |
1317 | // Fillout rx packet |
1318 | kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); |
1319 | VERIFY(rx_buf != NULL); |
1320 | void *rx_baddr = kern_buflet_get_object_address(rx_buf); |
1321 | VERIFY(rx_baddr != NULL); |
1322 | |
1323 | // Copy-in data from mbuf to buflet |
1324 | mbuf_copydata(data, 0, length, (void *)rx_baddr); |
1325 | kern_packet_clear_flow_uuid(rx_ph); // Zero flow id |
1326 | |
1327 | // Finalize and attach the packet |
1328 | error = kern_buflet_set_data_offset(rx_buf, 0); |
1329 | VERIFY(error == 0); |
1330 | error = kern_buflet_set_data_length(rx_buf, length); |
1331 | VERIFY(error == 0); |
1332 | error = kern_packet_set_link_header_offset(rx_ph, 0); |
1333 | VERIFY(error == 0); |
1334 | error = kern_packet_set_network_header_offset(rx_ph, 0); |
1335 | VERIFY(error == 0); |
1336 | error = kern_packet_finalize(rx_ph); |
1337 | VERIFY(error == 0); |
1338 | error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); |
1339 | VERIFY(error == 0); |
1340 | |
1341 | STATS_INC(nifs, NETIF_STATS_RXPKTS); |
1342 | STATS_INC(nifs, NETIF_STATS_RXCOPY_MBUF); |
1343 | bpf_tap_packet_in(pcb->ipsec_ifp, DLT_RAW, rx_ph, NULL, 0); |
1344 | |
1345 | rx_ring_stats.kcrsi_slots_transferred++; |
1346 | rx_ring_stats.kcrsi_bytes_transferred += length; |
1347 | |
1348 | if (!pcb->ipsec_ext_ifdata_stats) { |
1349 | ifnet_stat_increment_in(pcb->ipsec_ifp, 1, length, 0); |
1350 | } |
1351 | |
1352 | mbuf_freem(data); |
1353 | |
1354 | // Advance ring |
1355 | rx_pslot = rx_slot; |
1356 | rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); |
1357 | } |
1358 | |
1359 | struct kern_channel_ring_stat_increment tx_ring_stats; |
1360 | bzero(&tx_ring_stats, sizeof(tx_ring_stats)); |
1361 | kern_channel_ring_t tx_ring = pcb->ipsec_kpipe_txring; |
1362 | kern_channel_slot_t tx_pslot = NULL; |
1363 | kern_channel_slot_t tx_slot = NULL; |
1364 | if (tx_ring == NULL) { |
1365 | // Net-If TX ring not set up yet, nothing to read |
1366 | goto done; |
1367 | } |
1368 | |
1369 | |
1370 | // Unlock ipsec before entering ring |
1371 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
1372 | |
1373 | (void)kr_enter(tx_ring, TRUE); |
1374 | |
1375 | // Lock again after entering and validate |
1376 | lck_rw_lock_shared(&pcb->ipsec_pcb_lock); |
1377 | |
1378 | if (tx_ring != pcb->ipsec_kpipe_txring) { |
1379 | goto done; |
1380 | } |
1381 | |
1382 | tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); |
1383 | if (tx_slot == NULL) { |
1384 | // Nothing to read, don't bother signalling |
1385 | goto done; |
1386 | } |
1387 | |
1388 | while (rx_slot != NULL && tx_slot != NULL) { |
1389 | size_t length = 0; |
1390 | mbuf_t data = NULL; |
1391 | errno_t error = 0; |
1392 | uint32_t af; |
1393 | |
1394 | // Allocate rx packet |
1395 | kern_packet_t rx_ph = 0; |
1396 | error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); |
1397 | if (__improbable(error != 0)) { |
1398 | STATS_INC(nifs, NETIF_STATS_NOMEM_PKT); |
1399 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
1400 | break; |
1401 | } |
1402 | |
1403 | kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); |
1404 | |
1405 | // Advance TX ring |
1406 | tx_pslot = tx_slot; |
1407 | tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); |
1408 | |
1409 | if (tx_ph == 0) { |
1410 | kern_pbufpool_free(rx_pp, rx_ph); |
1411 | continue; |
1412 | } |
1413 | |
1414 | kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); |
1415 | VERIFY(tx_buf != NULL); |
1416 | uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); |
1417 | VERIFY(tx_baddr != 0); |
1418 | tx_baddr += kern_buflet_get_data_offset(tx_buf); |
1419 | |
1420 | length = MIN(kern_packet_get_data_length(tx_ph), |
1421 | pcb->ipsec_slot_size); |
1422 | |
1423 | // Increment TX stats |
1424 | tx_ring_stats.kcrsi_slots_transferred++; |
1425 | tx_ring_stats.kcrsi_bytes_transferred += length; |
1426 | |
1427 | if (length >= sizeof(struct ip)) { |
1428 | error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data); |
1429 | if (error == 0) { |
1430 | error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT); |
1431 | if (error == 0) { |
1432 | struct ip *ip = mtod(data, struct ip *); |
1433 | u_int ip_version = ip->ip_v; |
1434 | switch (ip_version) { |
1435 | case 4: { |
1436 | af = AF_INET; |
1437 | ip->ip_len = ntohs(ip->ip_len) - sizeof(struct ip); |
1438 | ip->ip_off = ntohs(ip->ip_off); |
1439 | |
1440 | if (length < ip->ip_len) { |
1441 | printf("ipsec_netif_sync_rx %s: IPv4 packet length too short (%zu < %u)\n" , |
1442 | pcb->ipsec_ifp->if_xname, length, ip->ip_len); |
1443 | STATS_INC(nifs, NETIF_STATS_BADLEN); |
1444 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
1445 | mbuf_freem(data); |
1446 | data = NULL; |
1447 | } else { |
1448 | data = esp4_input_extended(data, sizeof(struct ip), pcb->ipsec_ifp); |
1449 | } |
1450 | break; |
1451 | } |
1452 | case 6: { |
1453 | if (length < sizeof(struct ip6_hdr)) { |
1454 | printf("ipsec_netif_sync_rx %s: IPv6 packet length too short for header %zu\n" , |
1455 | pcb->ipsec_ifp->if_xname, length); |
1456 | STATS_INC(nifs, NETIF_STATS_BADLEN); |
1457 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
1458 | mbuf_freem(data); |
1459 | data = NULL; |
1460 | } else { |
1461 | af = AF_INET6; |
1462 | struct ip6_hdr *ip6 = mtod(data, struct ip6_hdr *); |
1463 | const size_t ip6_len = sizeof(*ip6) + ntohs(ip6->ip6_plen); |
1464 | if (length < ip6_len) { |
1465 | printf("ipsec_netif_sync_rx %s: IPv6 packet length too short (%zu < %zu)\n" , |
1466 | pcb->ipsec_ifp->if_xname, length, ip6_len); |
1467 | STATS_INC(nifs, NETIF_STATS_BADLEN); |
1468 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
1469 | mbuf_freem(data); |
1470 | data = NULL; |
1471 | } else { |
1472 | int offset = sizeof(struct ip6_hdr); |
1473 | esp6_input_extended(&data, &offset, ip6->ip6_nxt, pcb->ipsec_ifp); |
1474 | } |
1475 | } |
1476 | break; |
1477 | } |
1478 | default: { |
1479 | printf("ipsec_netif_sync_rx %s: unknown ip version %u\n" , |
1480 | pcb->ipsec_ifp->if_xname, ip_version); |
1481 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
1482 | mbuf_freem(data); |
1483 | data = NULL; |
1484 | break; |
1485 | } |
1486 | } |
1487 | } else { |
1488 | printf("ipsec_netif_sync_rx %s - mbuf_copyback(%zu) error %d\n" , pcb->ipsec_ifp->if_xname, length, error); |
1489 | STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); |
1490 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
1491 | mbuf_freem(data); |
1492 | data = NULL; |
1493 | } |
1494 | } else { |
1495 | printf("ipsec_netif_sync_rx %s - mbuf_gethdr error %d\n" , pcb->ipsec_ifp->if_xname, error); |
1496 | STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); |
1497 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
1498 | } |
1499 | } else { |
1500 | printf("ipsec_netif_sync_rx %s - bad packet length %zu\n" , pcb->ipsec_ifp->if_xname, length); |
1501 | STATS_INC(nifs, NETIF_STATS_BADLEN); |
1502 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
1503 | } |
1504 | |
1505 | if (data == NULL) { |
1506 | // Failed to get decrypted data data |
1507 | kern_pbufpool_free(rx_pp, rx_ph); |
1508 | continue; |
1509 | } |
1510 | |
1511 | length = mbuf_pkthdr_len(data); |
1512 | if (length > rx_pp->pp_buflet_size) { |
1513 | // Flush data |
1514 | mbuf_freem(data); |
1515 | kern_pbufpool_free(rx_pp, rx_ph); |
1516 | STATS_INC(nifs, NETIF_STATS_BADLEN); |
1517 | STATS_INC(nifs, NETIF_STATS_DROPPED); |
1518 | printf("ipsec_netif_sync_rx %s: decrypted packet length %zu > %u\n" , |
1519 | pcb->ipsec_ifp->if_xname, length, rx_pp->pp_buflet_size); |
1520 | continue; |
1521 | } |
1522 | |
1523 | mbuf_pkthdr_setrcvif(data, pcb->ipsec_ifp); |
1524 | |
1525 | // Fillout rx packet |
1526 | kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); |
1527 | VERIFY(rx_buf != NULL); |
1528 | void *rx_baddr = kern_buflet_get_object_address(rx_buf); |
1529 | VERIFY(rx_baddr != NULL); |
1530 | |
1531 | // Copy-in data from mbuf to buflet |
1532 | mbuf_copydata(data, 0, length, (void *)rx_baddr); |
1533 | kern_packet_clear_flow_uuid(rx_ph); // Zero flow id |
1534 | |
1535 | // Finalize and attach the packet |
1536 | error = kern_buflet_set_data_offset(rx_buf, 0); |
1537 | VERIFY(error == 0); |
1538 | error = kern_buflet_set_data_length(rx_buf, length); |
1539 | VERIFY(error == 0); |
1540 | error = kern_packet_set_link_header_offset(rx_ph, 0); |
1541 | VERIFY(error == 0); |
1542 | error = kern_packet_set_network_header_offset(rx_ph, 0); |
1543 | VERIFY(error == 0); |
1544 | error = kern_packet_finalize(rx_ph); |
1545 | VERIFY(error == 0); |
1546 | error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); |
1547 | VERIFY(error == 0); |
1548 | |
1549 | STATS_INC(nifs, NETIF_STATS_RXPKTS); |
1550 | STATS_INC(nifs, NETIF_STATS_RXCOPY_DIRECT); |
1551 | bpf_tap_packet_in(pcb->ipsec_ifp, DLT_RAW, rx_ph, NULL, 0); |
1552 | |
1553 | rx_ring_stats.kcrsi_slots_transferred++; |
1554 | rx_ring_stats.kcrsi_bytes_transferred += length; |
1555 | |
1556 | if (!pcb->ipsec_ext_ifdata_stats) { |
1557 | ifnet_stat_increment_in(pcb->ipsec_ifp, 1, length, 0); |
1558 | } |
1559 | |
1560 | mbuf_freem(data); |
1561 | |
1562 | rx_pslot = rx_slot; |
1563 | rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); |
1564 | } |
1565 | |
1566 | done: |
1567 | if (rx_pslot) { |
1568 | kern_channel_advance_slot(rx_ring, rx_pslot); |
1569 | kern_channel_increment_ring_net_stats(rx_ring, pcb->ipsec_ifp, &rx_ring_stats); |
1570 | } |
1571 | |
1572 | if (tx_pslot) { |
1573 | kern_channel_advance_slot(tx_ring, tx_pslot); |
1574 | kern_channel_increment_ring_net_stats(tx_ring, pcb->ipsec_ifp, &tx_ring_stats); |
1575 | (void)kern_channel_reclaim(tx_ring); |
1576 | } |
1577 | |
1578 | // Unlock first, then exit ring |
1579 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
1580 | if (tx_ring != NULL) { |
1581 | if (tx_pslot != NULL) { |
1582 | kern_channel_notify(tx_ring, 0); |
1583 | } |
1584 | kr_exit(tx_ring); |
1585 | } |
1586 | |
1587 | return 0; |
1588 | } |
1589 | |
1590 | static errno_t |
1591 | ipsec_nexus_ifattach(struct ipsec_pcb *pcb, |
1592 | struct ifnet_init_eparams *init_params, |
1593 | struct ifnet **ifp) |
1594 | { |
1595 | errno_t err; |
1596 | nexus_controller_t controller = kern_nexus_shared_controller(); |
1597 | struct kern_nexus_net_init net_init; |
1598 | struct kern_pbufpool_init pp_init; |
1599 | |
1600 | nexus_name_t provider_name; |
1601 | snprintf((char *)provider_name, sizeof(provider_name), |
1602 | "com.apple.netif.%s" , pcb->ipsec_if_xname); |
1603 | |
1604 | struct kern_nexus_provider_init prov_init = { |
1605 | .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION, |
1606 | .nxpi_flags = NXPIF_VIRTUAL_DEVICE, |
1607 | .nxpi_pre_connect = ipsec_nexus_pre_connect, |
1608 | .nxpi_connected = ipsec_nexus_connected, |
1609 | .nxpi_pre_disconnect = ipsec_netif_pre_disconnect, |
1610 | .nxpi_disconnected = ipsec_nexus_disconnected, |
1611 | .nxpi_ring_init = ipsec_netif_ring_init, |
1612 | .nxpi_ring_fini = ipsec_netif_ring_fini, |
1613 | .nxpi_slot_init = NULL, |
1614 | .nxpi_slot_fini = NULL, |
1615 | .nxpi_sync_tx = ipsec_netif_sync_tx, |
1616 | .nxpi_sync_rx = ipsec_netif_sync_rx, |
1617 | .nxpi_tx_doorbell = ipsec_netif_tx_doorbell, |
1618 | }; |
1619 | |
1620 | nexus_attr_t nxa = NULL; |
1621 | err = kern_nexus_attr_create(&nxa); |
1622 | IPSEC_IF_VERIFY(err == 0); |
1623 | if (err != 0) { |
1624 | printf("%s: kern_nexus_attr_create failed: %d\n" , |
1625 | __func__, err); |
1626 | goto failed; |
1627 | } |
1628 | |
1629 | uint64_t slot_buffer_size = pcb->ipsec_slot_size; |
1630 | err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); |
1631 | VERIFY(err == 0); |
1632 | |
1633 | // Reset ring size for netif nexus to limit memory usage |
1634 | uint64_t ring_size = pcb->ipsec_netif_ring_size; |
1635 | err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size); |
1636 | VERIFY(err == 0); |
1637 | err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size); |
1638 | VERIFY(err == 0); |
1639 | |
1640 | pcb->ipsec_netif_txring_size = ring_size; |
1641 | |
1642 | bzero(&pp_init, sizeof (pp_init)); |
1643 | pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION; |
1644 | pp_init.kbi_packets = pcb->ipsec_netif_ring_size * 2; |
1645 | pp_init.kbi_bufsize = pcb->ipsec_slot_size; |
1646 | pp_init.kbi_buf_seg_size = IPSEC_IF_DEFAULT_BUF_SEG_SIZE; |
1647 | pp_init.kbi_max_frags = 1; |
1648 | (void) snprintf((char *)pp_init.kbi_name, sizeof (pp_init.kbi_name), |
1649 | "%s" , provider_name); |
1650 | |
1651 | err = kern_pbufpool_create(&pp_init, &pp_init, &pcb->ipsec_netif_pp, NULL); |
1652 | if (err != 0) { |
1653 | printf("%s pbufbool create failed, error %d\n" , __func__, err); |
1654 | goto failed; |
1655 | } |
1656 | |
1657 | err = kern_nexus_controller_register_provider(controller, |
1658 | ipsec_nx_dom_prov, |
1659 | provider_name, |
1660 | &prov_init, |
1661 | sizeof(prov_init), |
1662 | nxa, |
1663 | &pcb->ipsec_nx.if_provider); |
1664 | IPSEC_IF_VERIFY(err == 0); |
1665 | if (err != 0) { |
1666 | printf("%s register provider failed, error %d\n" , |
1667 | __func__, err); |
1668 | goto failed; |
1669 | } |
1670 | |
1671 | bzero(&net_init, sizeof(net_init)); |
1672 | net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION; |
1673 | net_init.nxneti_flags = 0; |
1674 | net_init.nxneti_eparams = init_params; |
1675 | net_init.nxneti_lladdr = NULL; |
1676 | net_init.nxneti_prepare = ipsec_netif_prepare; |
1677 | net_init.nxneti_tx_pbufpool = pcb->ipsec_netif_pp; |
1678 | err = kern_nexus_controller_alloc_net_provider_instance(controller, |
1679 | pcb->ipsec_nx.if_provider, |
1680 | pcb, |
1681 | &pcb->ipsec_nx.if_instance, |
1682 | &net_init, |
1683 | ifp); |
1684 | IPSEC_IF_VERIFY(err == 0); |
1685 | if (err != 0) { |
1686 | printf("%s alloc_net_provider_instance failed, %d\n" , |
1687 | __func__, err); |
1688 | kern_nexus_controller_deregister_provider(controller, |
1689 | pcb->ipsec_nx.if_provider); |
1690 | uuid_clear(pcb->ipsec_nx.if_provider); |
1691 | goto failed; |
1692 | } |
1693 | |
1694 | failed: |
1695 | if (nxa) { |
1696 | kern_nexus_attr_destroy(nxa); |
1697 | } |
1698 | if (err && pcb->ipsec_netif_pp != NULL) { |
1699 | kern_pbufpool_destroy(pcb->ipsec_netif_pp); |
1700 | pcb->ipsec_netif_pp = NULL; |
1701 | } |
1702 | return (err); |
1703 | } |
1704 | |
1705 | static void |
1706 | ipsec_detach_provider_and_instance(uuid_t provider, uuid_t instance) |
1707 | { |
1708 | nexus_controller_t controller = kern_nexus_shared_controller(); |
1709 | errno_t err; |
1710 | |
1711 | if (!uuid_is_null(instance)) { |
1712 | err = kern_nexus_controller_free_provider_instance(controller, |
1713 | instance); |
1714 | if (err != 0) { |
1715 | printf("%s free_provider_instance failed %d\n" , |
1716 | __func__, err); |
1717 | } |
1718 | uuid_clear(instance); |
1719 | } |
1720 | if (!uuid_is_null(provider)) { |
1721 | err = kern_nexus_controller_deregister_provider(controller, |
1722 | provider); |
1723 | if (err != 0) { |
1724 | printf("%s deregister_provider %d\n" , __func__, err); |
1725 | } |
1726 | uuid_clear(provider); |
1727 | } |
1728 | return; |
1729 | } |
1730 | |
1731 | static void |
1732 | ipsec_nexus_detach(struct ipsec_pcb *pcb) |
1733 | { |
1734 | ipsec_nx_t nx = &pcb->ipsec_nx; |
1735 | nexus_controller_t controller = kern_nexus_shared_controller(); |
1736 | errno_t err; |
1737 | |
1738 | if (!uuid_is_null(nx->ms_host)) { |
1739 | err = kern_nexus_ifdetach(controller, |
1740 | nx->ms_instance, |
1741 | nx->ms_host); |
1742 | if (err != 0) { |
1743 | printf("%s: kern_nexus_ifdetach ms host failed %d\n" , |
1744 | __func__, err); |
1745 | } |
1746 | } |
1747 | |
1748 | if (!uuid_is_null(nx->ms_device)) { |
1749 | err = kern_nexus_ifdetach(controller, |
1750 | nx->ms_instance, |
1751 | nx->ms_device); |
1752 | if (err != 0) { |
1753 | printf("%s: kern_nexus_ifdetach ms device failed %d\n" , |
1754 | __func__, err); |
1755 | } |
1756 | } |
1757 | |
1758 | ipsec_detach_provider_and_instance(nx->if_provider, |
1759 | nx->if_instance); |
1760 | ipsec_detach_provider_and_instance(nx->ms_provider, |
1761 | nx->ms_instance); |
1762 | |
1763 | if (pcb->ipsec_netif_pp != NULL) { |
1764 | kern_pbufpool_destroy(pcb->ipsec_netif_pp); |
1765 | pcb->ipsec_netif_pp = NULL; |
1766 | |
1767 | } |
1768 | memset(nx, 0, sizeof(*nx)); |
1769 | } |
1770 | |
1771 | static errno_t |
1772 | ipsec_create_fs_provider_and_instance(struct ipsec_pcb *pcb, |
1773 | uint32_t subtype, const char *type_name, |
1774 | const char *ifname, |
1775 | uuid_t *provider, uuid_t *instance) |
1776 | { |
1777 | nexus_attr_t attr = NULL; |
1778 | nexus_controller_t controller = kern_nexus_shared_controller(); |
1779 | uuid_t dom_prov; |
1780 | errno_t err; |
1781 | struct kern_nexus_init init; |
1782 | nexus_name_t provider_name; |
1783 | |
1784 | err = kern_nexus_get_builtin_domain_provider(NEXUS_TYPE_FLOW_SWITCH, |
1785 | &dom_prov); |
1786 | IPSEC_IF_VERIFY(err == 0); |
1787 | if (err != 0) { |
1788 | printf("%s can't get %s provider, error %d\n" , |
1789 | __func__, type_name, err); |
1790 | goto failed; |
1791 | } |
1792 | |
1793 | err = kern_nexus_attr_create(&attr); |
1794 | IPSEC_IF_VERIFY(err == 0); |
1795 | if (err != 0) { |
1796 | printf("%s: kern_nexus_attr_create failed: %d\n" , |
1797 | __func__, err); |
1798 | goto failed; |
1799 | } |
1800 | |
1801 | err = kern_nexus_attr_set(attr, NEXUS_ATTR_EXTENSIONS, subtype); |
1802 | VERIFY(err == 0); |
1803 | |
1804 | uint64_t slot_buffer_size = pcb->ipsec_slot_size; |
1805 | err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); |
1806 | VERIFY(err == 0); |
1807 | |
1808 | // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif. |
1809 | uint64_t tx_ring_size = pcb->ipsec_tx_fsw_ring_size; |
1810 | err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size); |
1811 | VERIFY(err == 0); |
1812 | uint64_t rx_ring_size = pcb->ipsec_rx_fsw_ring_size; |
1813 | err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size); |
1814 | VERIFY(err == 0); |
1815 | |
1816 | snprintf((char *)provider_name, sizeof(provider_name), |
1817 | "com.apple.%s.%s" , type_name, ifname); |
1818 | err = kern_nexus_controller_register_provider(controller, |
1819 | dom_prov, |
1820 | provider_name, |
1821 | NULL, |
1822 | 0, |
1823 | attr, |
1824 | provider); |
1825 | kern_nexus_attr_destroy(attr); |
1826 | attr = NULL; |
1827 | IPSEC_IF_VERIFY(err == 0); |
1828 | if (err != 0) { |
1829 | printf("%s register %s provider failed, error %d\n" , |
1830 | __func__, type_name, err); |
1831 | goto failed; |
1832 | } |
1833 | bzero(&init, sizeof (init)); |
1834 | init.nxi_version = KERN_NEXUS_CURRENT_VERSION; |
1835 | err = kern_nexus_controller_alloc_provider_instance(controller, |
1836 | *provider, |
1837 | NULL, |
1838 | instance, &init); |
1839 | IPSEC_IF_VERIFY(err == 0); |
1840 | if (err != 0) { |
1841 | printf("%s alloc_provider_instance %s failed, %d\n" , |
1842 | __func__, type_name, err); |
1843 | kern_nexus_controller_deregister_provider(controller, |
1844 | *provider); |
1845 | uuid_clear(*provider); |
1846 | } |
1847 | failed: |
1848 | return (err); |
1849 | } |
1850 | |
1851 | static errno_t |
1852 | ipsec_multistack_attach(struct ipsec_pcb *pcb) |
1853 | { |
1854 | nexus_controller_t controller = kern_nexus_shared_controller(); |
1855 | errno_t err = 0; |
1856 | ipsec_nx_t nx = &pcb->ipsec_nx; |
1857 | |
1858 | // Allocate multistack flowswitch |
1859 | err = ipsec_create_fs_provider_and_instance(pcb, |
1860 | NEXUS_EXTENSION_FSW_TYPE_MULTISTACK, |
1861 | "multistack" , |
1862 | pcb->ipsec_ifp->if_xname, |
1863 | &nx->ms_provider, |
1864 | &nx->ms_instance); |
1865 | if (err != 0) { |
1866 | printf("%s: failed to create bridge provider and instance\n" , |
1867 | __func__); |
1868 | goto failed; |
1869 | } |
1870 | |
1871 | // Attach multistack to device port |
1872 | err = kern_nexus_ifattach(controller, nx->ms_instance, |
1873 | NULL, nx->if_instance, |
1874 | FALSE, &nx->ms_device); |
1875 | if (err != 0) { |
1876 | printf("%s kern_nexus_ifattach ms device %d\n" , __func__, err); |
1877 | goto failed; |
1878 | } |
1879 | |
1880 | // Attach multistack to host port |
1881 | err = kern_nexus_ifattach(controller, nx->ms_instance, |
1882 | NULL, nx->if_instance, |
1883 | TRUE, &nx->ms_host); |
1884 | if (err != 0) { |
1885 | printf("%s kern_nexus_ifattach ms host %d\n" , __func__, err); |
1886 | goto failed; |
1887 | } |
1888 | |
1889 | // Extract the agent UUID and save for later |
1890 | struct kern_nexus *multistack_nx = nx_find(nx->ms_instance, false); |
1891 | if (multistack_nx != NULL) { |
1892 | struct nx_flowswitch *flowswitch = NX_FSW_PRIVATE(multistack_nx); |
1893 | if (flowswitch != NULL) { |
1894 | FSW_RLOCK(flowswitch); |
1895 | struct fsw_ms_context *ms_context = (struct fsw_ms_context *)flowswitch->fsw_ops_private; |
1896 | if (ms_context != NULL) { |
1897 | uuid_copy(nx->ms_agent, ms_context->mc_agent_uuid); |
1898 | } else { |
1899 | printf("ipsec_multistack_attach - fsw_ms_context is NULL\n" ); |
1900 | } |
1901 | FSW_UNLOCK(flowswitch); |
1902 | } else { |
1903 | printf("ipsec_multistack_attach - flowswitch is NULL\n" ); |
1904 | } |
1905 | nx_release(multistack_nx); |
1906 | } else { |
1907 | printf("ipsec_multistack_attach - unable to find multistack nexus\n" ); |
1908 | } |
1909 | |
1910 | return (0); |
1911 | |
1912 | failed: |
1913 | ipsec_nexus_detach(pcb); |
1914 | |
1915 | errno_t detach_error = 0; |
1916 | if ((detach_error = ifnet_detach(pcb->ipsec_ifp)) != 0) { |
1917 | panic("ipsec_multistack_attach - ifnet_detach failed: %d\n" , detach_error); |
1918 | /* NOT REACHED */ |
1919 | } |
1920 | |
1921 | return (err); |
1922 | } |
1923 | |
1924 | #pragma mark Kernel Pipe Nexus |
1925 | |
1926 | static errno_t |
1927 | ipsec_register_kernel_pipe_nexus(void) |
1928 | { |
1929 | nexus_attr_t nxa = NULL; |
1930 | errno_t result; |
1931 | |
1932 | lck_mtx_lock(&ipsec_lock); |
1933 | if (ipsec_ncd_refcount++) { |
1934 | lck_mtx_unlock(&ipsec_lock); |
1935 | return 0; |
1936 | } |
1937 | |
1938 | result = kern_nexus_controller_create(&ipsec_ncd); |
1939 | if (result) { |
1940 | printf("%s: kern_nexus_controller_create failed: %d\n" , |
1941 | __FUNCTION__, result); |
1942 | goto done; |
1943 | } |
1944 | |
1945 | uuid_t dom_prov; |
1946 | result = kern_nexus_get_builtin_domain_provider( |
1947 | NEXUS_TYPE_KERNEL_PIPE, &dom_prov); |
1948 | if (result) { |
1949 | printf("%s: kern_nexus_get_builtin_domain_provider failed: %d\n" , |
1950 | __FUNCTION__, result); |
1951 | goto done; |
1952 | } |
1953 | |
1954 | struct kern_nexus_provider_init prov_init = { |
1955 | .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION, |
1956 | .nxpi_flags = NXPIF_VIRTUAL_DEVICE, |
1957 | .nxpi_pre_connect = ipsec_nexus_pre_connect, |
1958 | .nxpi_connected = ipsec_nexus_connected, |
1959 | .nxpi_pre_disconnect = ipsec_nexus_pre_disconnect, |
1960 | .nxpi_disconnected = ipsec_nexus_disconnected, |
1961 | .nxpi_ring_init = ipsec_kpipe_ring_init, |
1962 | .nxpi_ring_fini = ipsec_kpipe_ring_fini, |
1963 | .nxpi_slot_init = NULL, |
1964 | .nxpi_slot_fini = NULL, |
1965 | .nxpi_sync_tx = ipsec_kpipe_sync_tx, |
1966 | .nxpi_sync_rx = ipsec_kpipe_sync_rx, |
1967 | .nxpi_tx_doorbell = NULL, |
1968 | }; |
1969 | |
1970 | result = kern_nexus_attr_create(&nxa); |
1971 | if (result) { |
1972 | printf("%s: kern_nexus_attr_create failed: %d\n" , |
1973 | __FUNCTION__, result); |
1974 | goto done; |
1975 | } |
1976 | |
1977 | uint64_t slot_buffer_size = IPSEC_IF_DEFAULT_SLOT_SIZE; |
1978 | result = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); |
1979 | VERIFY(result == 0); |
1980 | |
1981 | // Reset ring size for kernel pipe nexus to limit memory usage |
1982 | uint64_t ring_size = if_ipsec_ring_size; |
1983 | result = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size); |
1984 | VERIFY(result == 0); |
1985 | result = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size); |
1986 | VERIFY(result == 0); |
1987 | |
1988 | result = kern_nexus_controller_register_provider(ipsec_ncd, |
1989 | dom_prov, |
1990 | (const uint8_t *)"com.apple.nexus.ipsec.kpipe" , |
1991 | &prov_init, |
1992 | sizeof(prov_init), |
1993 | nxa, |
1994 | &ipsec_kpipe_uuid); |
1995 | if (result) { |
1996 | printf("%s: kern_nexus_controller_register_provider failed: %d\n" , |
1997 | __FUNCTION__, result); |
1998 | goto done; |
1999 | } |
2000 | |
2001 | done: |
2002 | if (nxa) { |
2003 | kern_nexus_attr_destroy(nxa); |
2004 | } |
2005 | |
2006 | if (result) { |
2007 | if (ipsec_ncd) { |
2008 | kern_nexus_controller_destroy(ipsec_ncd); |
2009 | ipsec_ncd = NULL; |
2010 | } |
2011 | ipsec_ncd_refcount = 0; |
2012 | } |
2013 | |
2014 | lck_mtx_unlock(&ipsec_lock); |
2015 | |
2016 | return result; |
2017 | } |
2018 | |
2019 | static void |
2020 | ipsec_unregister_kernel_pipe_nexus(void) |
2021 | { |
2022 | lck_mtx_lock(&ipsec_lock); |
2023 | |
2024 | VERIFY(ipsec_ncd_refcount > 0); |
2025 | |
2026 | if (--ipsec_ncd_refcount == 0) { |
2027 | kern_nexus_controller_destroy(ipsec_ncd); |
2028 | ipsec_ncd = NULL; |
2029 | } |
2030 | |
2031 | lck_mtx_unlock(&ipsec_lock); |
2032 | } |
2033 | |
2034 | // For use by socket option, not internally |
2035 | static errno_t |
2036 | ipsec_disable_channel(struct ipsec_pcb *pcb) |
2037 | { |
2038 | errno_t result; |
2039 | int enabled; |
2040 | uuid_t uuid; |
2041 | |
2042 | lck_rw_lock_exclusive(&pcb->ipsec_pcb_lock); |
2043 | |
2044 | enabled = pcb->ipsec_kpipe_enabled; |
2045 | uuid_copy(uuid, pcb->ipsec_kpipe_uuid); |
2046 | |
2047 | VERIFY(uuid_is_null(pcb->ipsec_kpipe_uuid) == !enabled); |
2048 | |
2049 | pcb->ipsec_kpipe_enabled = 0; |
2050 | uuid_clear(pcb->ipsec_kpipe_uuid); |
2051 | |
2052 | lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock); |
2053 | |
2054 | if (enabled) { |
2055 | result = kern_nexus_controller_free_provider_instance(ipsec_ncd, uuid); |
2056 | } else { |
2057 | result = ENXIO; |
2058 | } |
2059 | |
2060 | if (!result) { |
2061 | if (pcb->ipsec_kpipe_pp != NULL) { |
2062 | kern_pbufpool_destroy(pcb->ipsec_kpipe_pp); |
2063 | pcb->ipsec_kpipe_pp = NULL; |
2064 | } |
2065 | ipsec_unregister_kernel_pipe_nexus(); |
2066 | } |
2067 | |
2068 | return result; |
2069 | } |
2070 | |
2071 | static errno_t |
2072 | ipsec_enable_channel(struct ipsec_pcb *pcb, struct proc *proc) |
2073 | { |
2074 | struct kern_nexus_init init; |
2075 | struct kern_pbufpool_init pp_init; |
2076 | errno_t result; |
2077 | |
2078 | kauth_cred_t cred = kauth_cred_get(); |
2079 | result = priv_check_cred(cred, PRIV_SKYWALK_REGISTER_KERNEL_PIPE, 0); |
2080 | if (result) { |
2081 | return result; |
2082 | } |
2083 | |
2084 | result = ipsec_register_kernel_pipe_nexus(); |
2085 | if (result) { |
2086 | return result; |
2087 | } |
2088 | |
2089 | VERIFY(ipsec_ncd); |
2090 | |
2091 | lck_rw_lock_exclusive(&pcb->ipsec_pcb_lock); |
2092 | |
2093 | /* ipsec driver doesn't support channels without a netif */ |
2094 | if (!pcb->ipsec_use_netif) { |
2095 | result = EOPNOTSUPP; |
2096 | goto done; |
2097 | } |
2098 | |
2099 | if (pcb->ipsec_kpipe_enabled) { |
2100 | result = EEXIST; // return success instead? |
2101 | goto done; |
2102 | } |
2103 | |
2104 | bzero(&pp_init, sizeof (pp_init)); |
2105 | pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION; |
2106 | pp_init.kbi_packets = pcb->ipsec_netif_ring_size * 2; |
2107 | pp_init.kbi_bufsize = pcb->ipsec_slot_size; |
2108 | pp_init.kbi_buf_seg_size = IPSEC_IF_DEFAULT_BUF_SEG_SIZE; |
2109 | pp_init.kbi_max_frags = 1; |
2110 | pp_init.kbi_flags |= KBIF_QUANTUM; |
2111 | (void) snprintf((char *)pp_init.kbi_name, sizeof (pp_init.kbi_name), |
2112 | "com.apple.kpipe.%s" , pcb->ipsec_if_xname); |
2113 | |
2114 | result = kern_pbufpool_create(&pp_init, &pp_init, &pcb->ipsec_kpipe_pp, |
2115 | NULL); |
2116 | if (result != 0) { |
2117 | printf("%s pbufbool create failed, error %d\n" , __func__, result); |
2118 | goto done; |
2119 | } |
2120 | |
2121 | VERIFY(uuid_is_null(pcb->ipsec_kpipe_uuid)); |
2122 | bzero(&init, sizeof (init)); |
2123 | init.nxi_version = KERN_NEXUS_CURRENT_VERSION; |
2124 | init.nxi_tx_pbufpool = pcb->ipsec_kpipe_pp; |
2125 | result = kern_nexus_controller_alloc_provider_instance(ipsec_ncd, |
2126 | ipsec_kpipe_uuid, pcb, &pcb->ipsec_kpipe_uuid, &init); |
2127 | if (result) { |
2128 | goto done; |
2129 | } |
2130 | |
2131 | nexus_port_t port = NEXUS_PORT_KERNEL_PIPE_CLIENT; |
2132 | result = kern_nexus_controller_bind_provider_instance(ipsec_ncd, |
2133 | pcb->ipsec_kpipe_uuid, &port, |
2134 | proc_pid(proc), NULL, NULL, 0, NEXUS_BIND_PID); |
2135 | if (result) { |
2136 | kern_nexus_controller_free_provider_instance(ipsec_ncd, |
2137 | pcb->ipsec_kpipe_uuid); |
2138 | uuid_clear(pcb->ipsec_kpipe_uuid); |
2139 | goto done; |
2140 | } |
2141 | |
2142 | pcb->ipsec_kpipe_enabled = 1; |
2143 | |
2144 | done: |
2145 | lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock); |
2146 | |
2147 | if (result) { |
2148 | if (pcb->ipsec_kpipe_pp != NULL) { |
2149 | kern_pbufpool_destroy(pcb->ipsec_kpipe_pp); |
2150 | pcb->ipsec_kpipe_pp = NULL; |
2151 | } |
2152 | ipsec_unregister_kernel_pipe_nexus(); |
2153 | } |
2154 | |
2155 | return result; |
2156 | } |
2157 | |
2158 | #endif // IPSEC_NEXUS |
2159 | |
2160 | |
2161 | /* Kernel control functions */ |
2162 | |
2163 | static inline void |
2164 | ipsec_free_pcb(struct ipsec_pcb *pcb, bool in_list) |
2165 | { |
2166 | #if IPSEC_NEXUS |
2167 | mbuf_freem_list(pcb->ipsec_input_chain); |
2168 | lck_mtx_destroy(&pcb->ipsec_input_chain_lock, ipsec_lck_grp); |
2169 | #endif // IPSEC_NEXUS |
2170 | lck_rw_destroy(&pcb->ipsec_pcb_lock, ipsec_lck_grp); |
2171 | if (in_list) { |
2172 | lck_mtx_lock(&ipsec_lock); |
2173 | TAILQ_REMOVE(&ipsec_head, pcb, ipsec_chain); |
2174 | lck_mtx_unlock(&ipsec_lock); |
2175 | } |
2176 | zfree(ipsec_pcb_zone, pcb); |
2177 | } |
2178 | |
2179 | static errno_t |
2180 | ipsec_ctl_bind(kern_ctl_ref kctlref, |
2181 | struct sockaddr_ctl *sac, |
2182 | void **unitinfo) |
2183 | { |
2184 | struct ipsec_pcb *pcb = zalloc(ipsec_pcb_zone); |
2185 | memset(pcb, 0, sizeof(*pcb)); |
2186 | |
2187 | /* Setup the protocol control block */ |
2188 | *unitinfo = pcb; |
2189 | pcb->ipsec_ctlref = kctlref; |
2190 | pcb->ipsec_unit = sac->sc_unit; |
2191 | pcb->ipsec_output_service_class = MBUF_SC_OAM; |
2192 | |
2193 | #if IPSEC_NEXUS |
2194 | pcb->ipsec_use_netif = false; |
2195 | pcb->ipsec_slot_size = IPSEC_IF_DEFAULT_SLOT_SIZE; |
2196 | pcb->ipsec_netif_ring_size = IPSEC_IF_DEFAULT_RING_SIZE; |
2197 | pcb->ipsec_tx_fsw_ring_size = IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE; |
2198 | pcb->ipsec_rx_fsw_ring_size = IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE; |
2199 | #endif // IPSEC_NEXUS |
2200 | |
2201 | lck_rw_init(&pcb->ipsec_pcb_lock, ipsec_lck_grp, ipsec_lck_attr); |
2202 | #if IPSEC_NEXUS |
2203 | lck_mtx_init(&pcb->ipsec_input_chain_lock, ipsec_lck_grp, ipsec_lck_attr); |
2204 | #endif // IPSEC_NEXUS |
2205 | |
2206 | return (0); |
2207 | } |
2208 | |
2209 | static errno_t |
2210 | ipsec_ctl_connect(kern_ctl_ref kctlref, |
2211 | struct sockaddr_ctl *sac, |
2212 | void **unitinfo) |
2213 | { |
2214 | struct ifnet_init_eparams ipsec_init = {}; |
2215 | errno_t result = 0; |
2216 | |
2217 | if (*unitinfo == NULL) { |
2218 | (void)ipsec_ctl_bind(kctlref, sac, unitinfo); |
2219 | } |
2220 | |
2221 | struct ipsec_pcb *pcb = *unitinfo; |
2222 | |
2223 | lck_mtx_lock(&ipsec_lock); |
2224 | |
2225 | /* Find some open interface id */ |
2226 | u_int32_t chosen_unique_id = 1; |
2227 | struct ipsec_pcb *next_pcb = TAILQ_LAST(&ipsec_head, ipsec_list); |
2228 | if (next_pcb != NULL) { |
2229 | /* List was not empty, add one to the last item */ |
2230 | chosen_unique_id = next_pcb->ipsec_unique_id + 1; |
2231 | next_pcb = NULL; |
2232 | |
2233 | /* |
2234 | * If this wrapped the id number, start looking at |
2235 | * the front of the list for an unused id. |
2236 | */ |
2237 | if (chosen_unique_id == 0) { |
2238 | /* Find the next unused ID */ |
2239 | chosen_unique_id = 1; |
2240 | TAILQ_FOREACH(next_pcb, &ipsec_head, ipsec_chain) { |
2241 | if (next_pcb->ipsec_unique_id > chosen_unique_id) { |
2242 | /* We found a gap */ |
2243 | break; |
2244 | } |
2245 | |
2246 | chosen_unique_id = next_pcb->ipsec_unique_id + 1; |
2247 | } |
2248 | } |
2249 | } |
2250 | |
2251 | pcb->ipsec_unique_id = chosen_unique_id; |
2252 | |
2253 | if (next_pcb != NULL) { |
2254 | TAILQ_INSERT_BEFORE(next_pcb, pcb, ipsec_chain); |
2255 | } else { |
2256 | TAILQ_INSERT_TAIL(&ipsec_head, pcb, ipsec_chain); |
2257 | } |
2258 | lck_mtx_unlock(&ipsec_lock); |
2259 | |
2260 | snprintf(pcb->ipsec_if_xname, sizeof(pcb->ipsec_if_xname), "ipsec%d" , pcb->ipsec_unit - 1); |
2261 | snprintf(pcb->ipsec_unique_name, sizeof(pcb->ipsec_unique_name), "ipsecid%d" , pcb->ipsec_unique_id - 1); |
2262 | printf("ipsec_ctl_connect: creating interface %s (id %s)\n" , pcb->ipsec_if_xname, pcb->ipsec_unique_name); |
2263 | |
2264 | /* Create the interface */ |
2265 | bzero(&ipsec_init, sizeof(ipsec_init)); |
2266 | ipsec_init.ver = IFNET_INIT_CURRENT_VERSION; |
2267 | ipsec_init.len = sizeof (ipsec_init); |
2268 | |
2269 | #if IPSEC_NEXUS |
2270 | if (pcb->ipsec_use_netif) { |
2271 | ipsec_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO); |
2272 | } else |
2273 | #endif // IPSEC_NEXUS |
2274 | { |
2275 | ipsec_init.flags = IFNET_INIT_NX_NOAUTO; |
2276 | ipsec_init.start = ipsec_start; |
2277 | } |
2278 | ipsec_init.name = "ipsec" ; |
2279 | ipsec_init.unit = pcb->ipsec_unit - 1; |
2280 | ipsec_init.uniqueid = pcb->ipsec_unique_name; |
2281 | ipsec_init.uniqueid_len = strlen(pcb->ipsec_unique_name); |
2282 | ipsec_init.family = ipsec_family; |
2283 | ipsec_init.subfamily = IFNET_SUBFAMILY_IPSEC; |
2284 | ipsec_init.type = IFT_OTHER; |
2285 | ipsec_init.demux = ipsec_demux; |
2286 | ipsec_init.add_proto = ipsec_add_proto; |
2287 | ipsec_init.del_proto = ipsec_del_proto; |
2288 | ipsec_init.softc = pcb; |
2289 | ipsec_init.ioctl = ipsec_ioctl; |
2290 | ipsec_init.detach = ipsec_detached; |
2291 | |
2292 | #if IPSEC_NEXUS |
2293 | if (pcb->ipsec_use_netif) { |
2294 | result = ipsec_nexus_ifattach(pcb, &ipsec_init, &pcb->ipsec_ifp); |
2295 | if (result != 0) { |
2296 | printf("ipsec_ctl_connect - ipsec_nexus_ifattach failed: %d\n" , result); |
2297 | ipsec_free_pcb(pcb, true); |
2298 | *unitinfo = NULL; |
2299 | return result; |
2300 | } |
2301 | |
2302 | result = ipsec_multistack_attach(pcb); |
2303 | if (result != 0) { |
2304 | printf("ipsec_ctl_connect - ipsec_multistack_attach failed: %d\n" , result); |
2305 | *unitinfo = NULL; |
2306 | return result; |
2307 | } |
2308 | |
2309 | /* Attach to bpf */ |
2310 | bpfattach(pcb->ipsec_ifp, DLT_RAW, 0); |
2311 | } else |
2312 | #endif // IPSEC_NEXUS |
2313 | { |
2314 | result = ifnet_allocate_extended(&ipsec_init, &pcb->ipsec_ifp); |
2315 | if (result != 0) { |
2316 | printf("ipsec_ctl_connect - ifnet_allocate failed: %d\n" , result); |
2317 | ipsec_free_pcb(pcb, true); |
2318 | *unitinfo = NULL; |
2319 | return result; |
2320 | } |
2321 | ipsec_ifnet_set_attrs(pcb->ipsec_ifp); |
2322 | |
2323 | /* Attach the interface */ |
2324 | result = ifnet_attach(pcb->ipsec_ifp, NULL); |
2325 | if (result != 0) { |
2326 | printf("ipsec_ctl_connect - ifnet_attach failed: %d\n" , result); |
2327 | ifnet_release(pcb->ipsec_ifp); |
2328 | ipsec_free_pcb(pcb, true); |
2329 | *unitinfo = NULL; |
2330 | return (result); |
2331 | } |
2332 | |
2333 | /* Attach to bpf */ |
2334 | bpfattach(pcb->ipsec_ifp, DLT_NULL, 0); |
2335 | } |
2336 | |
2337 | /* The interfaces resoures allocated, mark it as running */ |
2338 | ifnet_set_flags(pcb->ipsec_ifp, IFF_RUNNING, IFF_RUNNING); |
2339 | |
2340 | return (0); |
2341 | } |
2342 | |
2343 | static errno_t |
2344 | ipsec_detach_ip(ifnet_t interface, |
2345 | protocol_family_t protocol, |
2346 | socket_t pf_socket) |
2347 | { |
2348 | errno_t result = EPROTONOSUPPORT; |
2349 | |
2350 | /* Attempt a detach */ |
2351 | if (protocol == PF_INET) { |
2352 | struct ifreq ifr; |
2353 | |
2354 | bzero(&ifr, sizeof(ifr)); |
2355 | snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d" , |
2356 | ifnet_name(interface), ifnet_unit(interface)); |
2357 | |
2358 | result = sock_ioctl(pf_socket, SIOCPROTODETACH, &ifr); |
2359 | } |
2360 | else if (protocol == PF_INET6) { |
2361 | struct in6_ifreq ifr6; |
2362 | |
2363 | bzero(&ifr6, sizeof(ifr6)); |
2364 | snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d" , |
2365 | ifnet_name(interface), ifnet_unit(interface)); |
2366 | |
2367 | result = sock_ioctl(pf_socket, SIOCPROTODETACH_IN6, &ifr6); |
2368 | } |
2369 | |
2370 | return result; |
2371 | } |
2372 | |
2373 | static void |
2374 | ipsec_remove_address(ifnet_t interface, |
2375 | protocol_family_t protocol, |
2376 | ifaddr_t address, |
2377 | socket_t pf_socket) |
2378 | { |
2379 | errno_t result = 0; |
2380 | |
2381 | /* Attempt a detach */ |
2382 | if (protocol == PF_INET) { |
2383 | struct ifreq ifr; |
2384 | |
2385 | bzero(&ifr, sizeof(ifr)); |
2386 | snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d" , |
2387 | ifnet_name(interface), ifnet_unit(interface)); |
2388 | result = ifaddr_address(address, &ifr.ifr_addr, sizeof(ifr.ifr_addr)); |
2389 | if (result != 0) { |
2390 | printf("ipsec_remove_address - ifaddr_address failed: %d" , result); |
2391 | } |
2392 | else { |
2393 | result = sock_ioctl(pf_socket, SIOCDIFADDR, &ifr); |
2394 | if (result != 0) { |
2395 | printf("ipsec_remove_address - SIOCDIFADDR failed: %d" , result); |
2396 | } |
2397 | } |
2398 | } |
2399 | else if (protocol == PF_INET6) { |
2400 | struct in6_ifreq ifr6; |
2401 | |
2402 | bzero(&ifr6, sizeof(ifr6)); |
2403 | snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d" , |
2404 | ifnet_name(interface), ifnet_unit(interface)); |
2405 | result = ifaddr_address(address, (struct sockaddr*)&ifr6.ifr_addr, |
2406 | sizeof(ifr6.ifr_addr)); |
2407 | if (result != 0) { |
2408 | printf("ipsec_remove_address - ifaddr_address failed (v6): %d" , |
2409 | result); |
2410 | } |
2411 | else { |
2412 | result = sock_ioctl(pf_socket, SIOCDIFADDR_IN6, &ifr6); |
2413 | if (result != 0) { |
2414 | printf("ipsec_remove_address - SIOCDIFADDR_IN6 failed: %d" , |
2415 | result); |
2416 | } |
2417 | } |
2418 | } |
2419 | } |
2420 | |
2421 | static void |
2422 | ipsec_cleanup_family(ifnet_t interface, |
2423 | protocol_family_t protocol) |
2424 | { |
2425 | errno_t result = 0; |
2426 | socket_t pf_socket = NULL; |
2427 | ifaddr_t *addresses = NULL; |
2428 | int i; |
2429 | |
2430 | if (protocol != PF_INET && protocol != PF_INET6) { |
2431 | printf("ipsec_cleanup_family - invalid protocol family %d\n" , protocol); |
2432 | return; |
2433 | } |
2434 | |
2435 | /* Create a socket for removing addresses and detaching the protocol */ |
2436 | result = sock_socket(protocol, SOCK_DGRAM, 0, NULL, NULL, &pf_socket); |
2437 | if (result != 0) { |
2438 | if (result != EAFNOSUPPORT) |
2439 | printf("ipsec_cleanup_family - failed to create %s socket: %d\n" , |
2440 | protocol == PF_INET ? "IP" : "IPv6" , result); |
2441 | goto cleanup; |
2442 | } |
2443 | |
2444 | /* always set SS_PRIV, we want to close and detach regardless */ |
2445 | sock_setpriv(pf_socket, 1); |
2446 | |
2447 | result = ipsec_detach_ip(interface, protocol, pf_socket); |
2448 | if (result == 0 || result == ENXIO) { |
2449 | /* We are done! We either detached or weren't attached. */ |
2450 | goto cleanup; |
2451 | } |
2452 | else if (result != EBUSY) { |
2453 | /* Uh, not really sure what happened here... */ |
2454 | printf("ipsec_cleanup_family - ipsec_detach_ip failed: %d\n" , result); |
2455 | goto cleanup; |
2456 | } |
2457 | |
2458 | /* |
2459 | * At this point, we received an EBUSY error. This means there are |
2460 | * addresses attached. We should detach them and then try again. |
2461 | */ |
2462 | result = ifnet_get_address_list_family(interface, &addresses, protocol); |
2463 | if (result != 0) { |
2464 | printf("fnet_get_address_list_family(%s%d, 0xblah, %s) - failed: %d\n" , |
2465 | ifnet_name(interface), ifnet_unit(interface), |
2466 | protocol == PF_INET ? "PF_INET" : "PF_INET6" , result); |
2467 | goto cleanup; |
2468 | } |
2469 | |
2470 | for (i = 0; addresses[i] != 0; i++) { |
2471 | ipsec_remove_address(interface, protocol, addresses[i], pf_socket); |
2472 | } |
2473 | ifnet_free_address_list(addresses); |
2474 | addresses = NULL; |
2475 | |
2476 | /* |
2477 | * The addresses should be gone, we should try the remove again. |
2478 | */ |
2479 | result = ipsec_detach_ip(interface, protocol, pf_socket); |
2480 | if (result != 0 && result != ENXIO) { |
2481 | printf("ipsec_cleanup_family - ipsec_detach_ip failed: %d\n" , result); |
2482 | } |
2483 | |
2484 | cleanup: |
2485 | if (pf_socket != NULL) |
2486 | sock_close(pf_socket); |
2487 | |
2488 | if (addresses != NULL) |
2489 | ifnet_free_address_list(addresses); |
2490 | } |
2491 | |
2492 | static errno_t |
2493 | ipsec_ctl_disconnect(__unused kern_ctl_ref kctlref, |
2494 | __unused u_int32_t unit, |
2495 | void *unitinfo) |
2496 | { |
2497 | struct ipsec_pcb *pcb = unitinfo; |
2498 | ifnet_t ifp = NULL; |
2499 | errno_t result = 0; |
2500 | |
2501 | if (pcb == NULL) { |
2502 | return EINVAL; |
2503 | } |
2504 | |
2505 | #if IPSEC_NEXUS |
2506 | // Tell the nexus to stop all rings |
2507 | if (pcb->ipsec_netif_nexus != NULL) { |
2508 | kern_nexus_stop(pcb->ipsec_netif_nexus); |
2509 | } |
2510 | #endif // IPSEC_NEXUS |
2511 | |
2512 | lck_rw_lock_exclusive(&pcb->ipsec_pcb_lock); |
2513 | |
2514 | #if IPSEC_NEXUS |
2515 | uuid_t kpipe_uuid; |
2516 | uuid_copy(kpipe_uuid, pcb->ipsec_kpipe_uuid); |
2517 | uuid_clear(pcb->ipsec_kpipe_uuid); |
2518 | pcb->ipsec_kpipe_enabled = FALSE; |
2519 | #endif // IPSEC_NEXUS |
2520 | |
2521 | pcb->ipsec_ctlref = NULL; |
2522 | |
2523 | ifp = pcb->ipsec_ifp; |
2524 | if (ifp != NULL) { |
2525 | #if IPSEC_NEXUS |
2526 | if (pcb->ipsec_netif_nexus != NULL) { |
2527 | /* |
2528 | * Quiesce the interface and flush any pending outbound packets. |
2529 | */ |
2530 | if_down(ifp); |
2531 | |
2532 | /* Increment refcnt, but detach interface */ |
2533 | ifnet_incr_iorefcnt(ifp); |
2534 | if ((result = ifnet_detach(ifp)) != 0) { |
2535 | panic("ipsec_ctl_disconnect - ifnet_detach failed: %d\n" , result); |
2536 | /* NOT REACHED */ |
2537 | } |
2538 | |
2539 | /* |
2540 | * We want to do everything in our power to ensure that the interface |
2541 | * really goes away when the socket is closed. We must remove IP/IPv6 |
2542 | * addresses and detach the protocols. Finally, we can remove and |
2543 | * release the interface. |
2544 | */ |
2545 | key_delsp_for_ipsec_if(ifp); |
2546 | |
2547 | ipsec_cleanup_family(ifp, AF_INET); |
2548 | ipsec_cleanup_family(ifp, AF_INET6); |
2549 | |
2550 | lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock); |
2551 | |
2552 | if (!uuid_is_null(kpipe_uuid)) { |
2553 | if (kern_nexus_controller_free_provider_instance(ipsec_ncd, kpipe_uuid) == 0) { |
2554 | if (pcb->ipsec_kpipe_pp != NULL) { |
2555 | kern_pbufpool_destroy(pcb->ipsec_kpipe_pp); |
2556 | pcb->ipsec_kpipe_pp = NULL; |
2557 | } |
2558 | ipsec_unregister_kernel_pipe_nexus(); |
2559 | } |
2560 | } |
2561 | ipsec_nexus_detach(pcb); |
2562 | |
2563 | /* Decrement refcnt to finish detaching and freeing */ |
2564 | ifnet_decr_iorefcnt(ifp); |
2565 | } else |
2566 | #endif // IPSEC_NEXUS |
2567 | { |
2568 | lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock); |
2569 | |
2570 | #if IPSEC_NEXUS |
2571 | if (!uuid_is_null(kpipe_uuid)) { |
2572 | if (kern_nexus_controller_free_provider_instance(ipsec_ncd, kpipe_uuid) == 0) { |
2573 | if (pcb->ipsec_kpipe_pp != NULL) { |
2574 | kern_pbufpool_destroy(pcb->ipsec_kpipe_pp); |
2575 | pcb->ipsec_kpipe_pp = NULL; |
2576 | } |
2577 | ipsec_unregister_kernel_pipe_nexus(); |
2578 | } |
2579 | } |
2580 | #endif // IPSEC_NEXUS |
2581 | |
2582 | /* |
2583 | * We want to do everything in our power to ensure that the interface |
2584 | * really goes away when the socket is closed. We must remove IP/IPv6 |
2585 | * addresses and detach the protocols. Finally, we can remove and |
2586 | * release the interface. |
2587 | */ |
2588 | key_delsp_for_ipsec_if(ifp); |
2589 | |
2590 | ipsec_cleanup_family(ifp, AF_INET); |
2591 | ipsec_cleanup_family(ifp, AF_INET6); |
2592 | |
2593 | /* |
2594 | * Detach now; ipsec_detach() will be called asynchronously once |
2595 | * the I/O reference count drops to 0. There we will invoke |
2596 | * ifnet_release(). |
2597 | */ |
2598 | if ((result = ifnet_detach(ifp)) != 0) { |
2599 | printf("ipsec_ctl_disconnect - ifnet_detach failed: %d\n" , result); |
2600 | } |
2601 | } |
2602 | } else { |
2603 | // Bound, but not connected |
2604 | lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock); |
2605 | ipsec_free_pcb(pcb, false); |
2606 | } |
2607 | |
2608 | return 0; |
2609 | } |
2610 | |
2611 | static errno_t |
2612 | ipsec_ctl_send(__unused kern_ctl_ref kctlref, |
2613 | __unused u_int32_t unit, |
2614 | __unused void *unitinfo, |
2615 | mbuf_t m, |
2616 | __unused int flags) |
2617 | { |
2618 | /* Receive messages from the control socket. Currently unused. */ |
2619 | mbuf_freem(m); |
2620 | return 0; |
2621 | } |
2622 | |
2623 | static errno_t |
2624 | ipsec_ctl_setopt(__unused kern_ctl_ref kctlref, |
2625 | __unused u_int32_t unit, |
2626 | void *unitinfo, |
2627 | int opt, |
2628 | void *data, |
2629 | size_t len) |
2630 | { |
2631 | struct ipsec_pcb *pcb = unitinfo; |
2632 | errno_t result = 0; |
2633 | |
2634 | /* check for privileges for privileged options */ |
2635 | switch (opt) { |
2636 | case IPSEC_OPT_FLAGS: |
2637 | case IPSEC_OPT_EXT_IFDATA_STATS: |
2638 | case IPSEC_OPT_SET_DELEGATE_INTERFACE: |
2639 | case IPSEC_OPT_OUTPUT_TRAFFIC_CLASS: |
2640 | if (kauth_cred_issuser(kauth_cred_get()) == 0) { |
2641 | return EPERM; |
2642 | } |
2643 | break; |
2644 | } |
2645 | |
2646 | switch (opt) { |
2647 | case IPSEC_OPT_FLAGS: |
2648 | if (len != sizeof(u_int32_t)) { |
2649 | result = EMSGSIZE; |
2650 | } else { |
2651 | pcb->ipsec_flags = *(u_int32_t *)data; |
2652 | } |
2653 | break; |
2654 | |
2655 | case IPSEC_OPT_EXT_IFDATA_STATS: |
2656 | if (len != sizeof(int)) { |
2657 | result = EMSGSIZE; |
2658 | break; |
2659 | } |
2660 | if (pcb->ipsec_ifp == NULL) { |
2661 | // Only can set after connecting |
2662 | result = EINVAL; |
2663 | break; |
2664 | } |
2665 | pcb->ipsec_ext_ifdata_stats = (*(int *)data) ? 1 : 0; |
2666 | break; |
2667 | |
2668 | case IPSEC_OPT_INC_IFDATA_STATS_IN: |
2669 | case IPSEC_OPT_INC_IFDATA_STATS_OUT: { |
2670 | struct ipsec_stats_param *utsp = (struct ipsec_stats_param *)data; |
2671 | |
2672 | if (utsp == NULL || len < sizeof(struct ipsec_stats_param)) { |
2673 | result = EINVAL; |
2674 | break; |
2675 | } |
2676 | if (pcb->ipsec_ifp == NULL) { |
2677 | // Only can set after connecting |
2678 | result = EINVAL; |
2679 | break; |
2680 | } |
2681 | if (!pcb->ipsec_ext_ifdata_stats) { |
2682 | result = EINVAL; |
2683 | break; |
2684 | } |
2685 | if (opt == IPSEC_OPT_INC_IFDATA_STATS_IN) |
2686 | ifnet_stat_increment_in(pcb->ipsec_ifp, utsp->utsp_packets, |
2687 | utsp->utsp_bytes, utsp->utsp_errors); |
2688 | else |
2689 | ifnet_stat_increment_out(pcb->ipsec_ifp, utsp->utsp_packets, |
2690 | utsp->utsp_bytes, utsp->utsp_errors); |
2691 | break; |
2692 | } |
2693 | |
2694 | case IPSEC_OPT_SET_DELEGATE_INTERFACE: { |
2695 | ifnet_t del_ifp = NULL; |
2696 | char name[IFNAMSIZ]; |
2697 | |
2698 | if (len > IFNAMSIZ - 1) { |
2699 | result = EMSGSIZE; |
2700 | break; |
2701 | } |
2702 | if (pcb->ipsec_ifp == NULL) { |
2703 | // Only can set after connecting |
2704 | result = EINVAL; |
2705 | break; |
2706 | } |
2707 | if (len != 0) { /* if len==0, del_ifp will be NULL causing the delegate to be removed */ |
2708 | bcopy(data, name, len); |
2709 | name[len] = 0; |
2710 | result = ifnet_find_by_name(name, &del_ifp); |
2711 | } |
2712 | if (result == 0) { |
2713 | printf("%s IPSEC_OPT_SET_DELEGATE_INTERFACE %s to %s\n" , |
2714 | __func__, pcb->ipsec_ifp->if_xname, |
2715 | del_ifp ? del_ifp->if_xname : "NULL" ); |
2716 | |
2717 | result = ifnet_set_delegate(pcb->ipsec_ifp, del_ifp); |
2718 | if (del_ifp) |
2719 | ifnet_release(del_ifp); |
2720 | } |
2721 | break; |
2722 | } |
2723 | |
2724 | case IPSEC_OPT_OUTPUT_TRAFFIC_CLASS: { |
2725 | if (len != sizeof(int)) { |
2726 | result = EMSGSIZE; |
2727 | break; |
2728 | } |
2729 | if (pcb->ipsec_ifp == NULL) { |
2730 | // Only can set after connecting |
2731 | result = EINVAL; |
2732 | break; |
2733 | } |
2734 | mbuf_svc_class_t output_service_class = so_tc2msc(*(int *)data); |
2735 | if (output_service_class == MBUF_SC_UNSPEC) { |
2736 | pcb->ipsec_output_service_class = MBUF_SC_OAM; |
2737 | } else { |
2738 | pcb->ipsec_output_service_class = output_service_class; |
2739 | } |
2740 | printf("%s IPSEC_OPT_OUTPUT_TRAFFIC_CLASS %s svc %d\n" , |
2741 | __func__, pcb->ipsec_ifp->if_xname, |
2742 | pcb->ipsec_output_service_class); |
2743 | break; |
2744 | } |
2745 | |
2746 | #if IPSEC_NEXUS |
2747 | case IPSEC_OPT_ENABLE_CHANNEL: { |
2748 | if (len != sizeof(int)) { |
2749 | result = EMSGSIZE; |
2750 | break; |
2751 | } |
2752 | if (pcb->ipsec_ifp == NULL) { |
2753 | // Only can set after connecting |
2754 | result = EINVAL; |
2755 | break; |
2756 | } |
2757 | if (*(int *)data) { |
2758 | result = ipsec_enable_channel(pcb, current_proc()); |
2759 | } else { |
2760 | result = ipsec_disable_channel(pcb); |
2761 | } |
2762 | break; |
2763 | } |
2764 | |
2765 | case IPSEC_OPT_ENABLE_FLOWSWITCH: { |
2766 | if (len != sizeof(int)) { |
2767 | result = EMSGSIZE; |
2768 | break; |
2769 | } |
2770 | if (pcb->ipsec_ifp == NULL) { |
2771 | // Only can set after connecting |
2772 | result = EINVAL; |
2773 | break; |
2774 | } |
2775 | if (!if_is_netagent_enabled()) { |
2776 | result = ENOTSUP; |
2777 | break; |
2778 | } |
2779 | if (uuid_is_null(pcb->ipsec_nx.ms_agent)) { |
2780 | result = ENOENT; |
2781 | break; |
2782 | } |
2783 | |
2784 | if (*(int *)data) { |
2785 | if_add_netagent(pcb->ipsec_ifp, pcb->ipsec_nx.ms_agent); |
2786 | pcb->ipsec_needs_netagent = true; |
2787 | } else { |
2788 | pcb->ipsec_needs_netagent = false; |
2789 | if_delete_netagent(pcb->ipsec_ifp, pcb->ipsec_nx.ms_agent); |
2790 | } |
2791 | break; |
2792 | } |
2793 | |
2794 | case IPSEC_OPT_INPUT_FRAG_SIZE: { |
2795 | if (len != sizeof(u_int32_t)) { |
2796 | result = EMSGSIZE; |
2797 | break; |
2798 | } |
2799 | u_int32_t input_frag_size = *(u_int32_t *)data; |
2800 | if (input_frag_size <= sizeof(struct ip6_hdr)) { |
2801 | pcb->ipsec_frag_size_set = FALSE; |
2802 | pcb->ipsec_input_frag_size = 0; |
2803 | } else { |
2804 | printf("SET FRAG SIZE TO %u\n" , input_frag_size); |
2805 | pcb->ipsec_frag_size_set = TRUE; |
2806 | pcb->ipsec_input_frag_size = input_frag_size; |
2807 | } |
2808 | break; |
2809 | } |
2810 | case IPSEC_OPT_ENABLE_NETIF: { |
2811 | if (len != sizeof(int)) { |
2812 | result = EMSGSIZE; |
2813 | break; |
2814 | } |
2815 | if (pcb->ipsec_ifp != NULL) { |
2816 | // Only can set before connecting |
2817 | result = EINVAL; |
2818 | break; |
2819 | } |
2820 | lck_rw_lock_exclusive(&pcb->ipsec_pcb_lock); |
2821 | pcb->ipsec_use_netif = !!(*(int *)data); |
2822 | lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock); |
2823 | break; |
2824 | } |
2825 | case IPSEC_OPT_SLOT_SIZE: { |
2826 | if (len != sizeof(u_int32_t)) { |
2827 | result = EMSGSIZE; |
2828 | break; |
2829 | } |
2830 | if (pcb->ipsec_ifp != NULL) { |
2831 | // Only can set before connecting |
2832 | result = EINVAL; |
2833 | break; |
2834 | } |
2835 | u_int32_t slot_size = *(u_int32_t *)data; |
2836 | if (slot_size < IPSEC_IF_MIN_SLOT_SIZE || |
2837 | slot_size > IPSEC_IF_MAX_SLOT_SIZE) { |
2838 | return (EINVAL); |
2839 | } |
2840 | pcb->ipsec_slot_size = slot_size; |
2841 | break; |
2842 | } |
2843 | case IPSEC_OPT_NETIF_RING_SIZE: { |
2844 | if (len != sizeof(u_int32_t)) { |
2845 | result = EMSGSIZE; |
2846 | break; |
2847 | } |
2848 | if (pcb->ipsec_ifp != NULL) { |
2849 | // Only can set before connecting |
2850 | result = EINVAL; |
2851 | break; |
2852 | } |
2853 | u_int32_t ring_size = *(u_int32_t *)data; |
2854 | if (ring_size < IPSEC_IF_MIN_RING_SIZE || |
2855 | ring_size > IPSEC_IF_MAX_RING_SIZE) { |
2856 | return (EINVAL); |
2857 | } |
2858 | pcb->ipsec_netif_ring_size = ring_size; |
2859 | break; |
2860 | } |
2861 | case IPSEC_OPT_TX_FSW_RING_SIZE: { |
2862 | if (len != sizeof(u_int32_t)) { |
2863 | result = EMSGSIZE; |
2864 | break; |
2865 | } |
2866 | if (pcb->ipsec_ifp != NULL) { |
2867 | // Only can set before connecting |
2868 | result = EINVAL; |
2869 | break; |
2870 | } |
2871 | u_int32_t ring_size = *(u_int32_t *)data; |
2872 | if (ring_size < IPSEC_IF_MIN_RING_SIZE || |
2873 | ring_size > IPSEC_IF_MAX_RING_SIZE) { |
2874 | return (EINVAL); |
2875 | } |
2876 | pcb->ipsec_tx_fsw_ring_size = ring_size; |
2877 | break; |
2878 | } |
2879 | case IPSEC_OPT_RX_FSW_RING_SIZE: { |
2880 | if (len != sizeof(u_int32_t)) { |
2881 | result = EMSGSIZE; |
2882 | break; |
2883 | } |
2884 | if (pcb->ipsec_ifp != NULL) { |
2885 | // Only can set before connecting |
2886 | result = EINVAL; |
2887 | break; |
2888 | } |
2889 | u_int32_t ring_size = *(u_int32_t *)data; |
2890 | if (ring_size < IPSEC_IF_MIN_RING_SIZE || |
2891 | ring_size > IPSEC_IF_MAX_RING_SIZE) { |
2892 | return (EINVAL); |
2893 | } |
2894 | pcb->ipsec_rx_fsw_ring_size = ring_size; |
2895 | break; |
2896 | } |
2897 | |
2898 | #endif // IPSEC_NEXUS |
2899 | |
2900 | default: |
2901 | result = ENOPROTOOPT; |
2902 | break; |
2903 | } |
2904 | |
2905 | return result; |
2906 | } |
2907 | |
2908 | static errno_t |
2909 | ipsec_ctl_getopt(__unused kern_ctl_ref kctlref, |
2910 | __unused u_int32_t unit, |
2911 | void *unitinfo, |
2912 | int opt, |
2913 | void *data, |
2914 | size_t *len) |
2915 | { |
2916 | struct ipsec_pcb *pcb = unitinfo; |
2917 | errno_t result = 0; |
2918 | |
2919 | switch (opt) { |
2920 | case IPSEC_OPT_FLAGS: { |
2921 | if (*len != sizeof(u_int32_t)) { |
2922 | result = EMSGSIZE; |
2923 | } else { |
2924 | *(u_int32_t *)data = pcb->ipsec_flags; |
2925 | } |
2926 | break; |
2927 | } |
2928 | |
2929 | case IPSEC_OPT_EXT_IFDATA_STATS: { |
2930 | if (*len != sizeof(int)) { |
2931 | result = EMSGSIZE; |
2932 | } else { |
2933 | *(int *)data = (pcb->ipsec_ext_ifdata_stats) ? 1 : 0; |
2934 | } |
2935 | break; |
2936 | } |
2937 | |
2938 | case IPSEC_OPT_IFNAME: { |
2939 | if (*len < MIN(strlen(pcb->ipsec_if_xname) + 1, sizeof(pcb->ipsec_if_xname))) { |
2940 | result = EMSGSIZE; |
2941 | } else { |
2942 | if (pcb->ipsec_ifp == NULL) { |
2943 | // Only can get after connecting |
2944 | result = EINVAL; |
2945 | break; |
2946 | } |
2947 | *len = snprintf(data, *len, "%s" , pcb->ipsec_if_xname) + 1; |
2948 | } |
2949 | break; |
2950 | } |
2951 | |
2952 | case IPSEC_OPT_OUTPUT_TRAFFIC_CLASS: { |
2953 | if (*len != sizeof(int)) { |
2954 | result = EMSGSIZE; |
2955 | } else { |
2956 | *(int *)data = so_svc2tc(pcb->ipsec_output_service_class); |
2957 | } |
2958 | break; |
2959 | } |
2960 | |
2961 | #if IPSEC_NEXUS |
2962 | |
2963 | case IPSEC_OPT_ENABLE_CHANNEL: { |
2964 | if (*len != sizeof(int)) { |
2965 | result = EMSGSIZE; |
2966 | } else { |
2967 | lck_rw_lock_shared(&pcb->ipsec_pcb_lock); |
2968 | *(int *)data = pcb->ipsec_kpipe_enabled; |
2969 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
2970 | } |
2971 | break; |
2972 | } |
2973 | |
2974 | case IPSEC_OPT_ENABLE_FLOWSWITCH: { |
2975 | if (*len != sizeof(int)) { |
2976 | result = EMSGSIZE; |
2977 | } else { |
2978 | *(int *)data = if_check_netagent(pcb->ipsec_ifp, pcb->ipsec_nx.ms_agent); |
2979 | } |
2980 | break; |
2981 | } |
2982 | |
2983 | case IPSEC_OPT_ENABLE_NETIF: { |
2984 | if (*len != sizeof(int)) { |
2985 | result = EMSGSIZE; |
2986 | } else { |
2987 | lck_rw_lock_shared(&pcb->ipsec_pcb_lock); |
2988 | *(int *)data = !!pcb->ipsec_use_netif; |
2989 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
2990 | } |
2991 | break; |
2992 | } |
2993 | |
2994 | case IPSEC_OPT_GET_CHANNEL_UUID: { |
2995 | lck_rw_lock_shared(&pcb->ipsec_pcb_lock); |
2996 | if (uuid_is_null(pcb->ipsec_kpipe_uuid)) { |
2997 | result = ENXIO; |
2998 | } else if (*len != sizeof(uuid_t)) { |
2999 | result = EMSGSIZE; |
3000 | } else { |
3001 | uuid_copy(data, pcb->ipsec_kpipe_uuid); |
3002 | } |
3003 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
3004 | break; |
3005 | } |
3006 | |
3007 | case IPSEC_OPT_INPUT_FRAG_SIZE: { |
3008 | if (*len != sizeof(u_int32_t)) { |
3009 | result = EMSGSIZE; |
3010 | } else { |
3011 | *(u_int32_t *)data = pcb->ipsec_input_frag_size; |
3012 | } |
3013 | break; |
3014 | } |
3015 | case IPSEC_OPT_SLOT_SIZE: { |
3016 | if (*len != sizeof(u_int32_t)) { |
3017 | result = EMSGSIZE; |
3018 | } else { |
3019 | *(u_int32_t *)data = pcb->ipsec_slot_size; |
3020 | } |
3021 | break; |
3022 | } |
3023 | case IPSEC_OPT_NETIF_RING_SIZE: { |
3024 | if (*len != sizeof(u_int32_t)) { |
3025 | result = EMSGSIZE; |
3026 | } else { |
3027 | *(u_int32_t *)data = pcb->ipsec_netif_ring_size; |
3028 | } |
3029 | break; |
3030 | } |
3031 | case IPSEC_OPT_TX_FSW_RING_SIZE: { |
3032 | if (*len != sizeof(u_int32_t)) { |
3033 | result = EMSGSIZE; |
3034 | } else { |
3035 | *(u_int32_t *)data = pcb->ipsec_tx_fsw_ring_size; |
3036 | } |
3037 | break; |
3038 | } |
3039 | case IPSEC_OPT_RX_FSW_RING_SIZE: { |
3040 | if (*len != sizeof(u_int32_t)) { |
3041 | result = EMSGSIZE; |
3042 | } else { |
3043 | *(u_int32_t *)data = pcb->ipsec_rx_fsw_ring_size; |
3044 | } |
3045 | break; |
3046 | } |
3047 | |
3048 | #endif // IPSEC_NEXUS |
3049 | |
3050 | default: { |
3051 | result = ENOPROTOOPT; |
3052 | break; |
3053 | } |
3054 | } |
3055 | |
3056 | return result; |
3057 | } |
3058 | |
3059 | /* Network Interface functions */ |
3060 | static errno_t |
3061 | ipsec_output(ifnet_t interface, |
3062 | mbuf_t data) |
3063 | { |
3064 | struct ipsec_pcb *pcb = ifnet_softc(interface); |
3065 | struct ipsec_output_state ipsec_state; |
3066 | struct route ro; |
3067 | struct route_in6 ro6; |
3068 | int length; |
3069 | struct ip *ip; |
3070 | struct ip6_hdr *ip6; |
3071 | struct ip_out_args ipoa; |
3072 | struct ip6_out_args ip6oa; |
3073 | int error = 0; |
3074 | u_int ip_version = 0; |
3075 | int flags = 0; |
3076 | struct flowadv *adv = NULL; |
3077 | |
3078 | // Make sure this packet isn't looping through the interface |
3079 | if (necp_get_last_interface_index_from_packet(data) == interface->if_index) { |
3080 | error = EINVAL; |
3081 | goto ipsec_output_err; |
3082 | } |
3083 | |
3084 | // Mark the interface so NECP can evaluate tunnel policy |
3085 | necp_mark_packet_from_interface(data, interface); |
3086 | |
3087 | ip = mtod(data, struct ip *); |
3088 | ip_version = ip->ip_v; |
3089 | |
3090 | switch (ip_version) { |
3091 | case 4: { |
3092 | #if IPSEC_NEXUS |
3093 | if (!pcb->ipsec_use_netif) |
3094 | #endif // IPSEC_NEXUS |
3095 | { |
3096 | int af = AF_INET; |
3097 | bpf_tap_out(pcb->ipsec_ifp, DLT_NULL, data, &af, sizeof(af)); |
3098 | } |
3099 | |
3100 | /* Apply encryption */ |
3101 | memset(&ipsec_state, 0, sizeof(ipsec_state)); |
3102 | ipsec_state.m = data; |
3103 | ipsec_state.dst = (struct sockaddr *)&ip->ip_dst; |
3104 | memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro)); |
3105 | |
3106 | error = ipsec4_interface_output(&ipsec_state, interface); |
3107 | /* Tunneled in IPv6 - packet is gone */ |
3108 | if (error == 0 && ipsec_state.tunneled == 6) { |
3109 | goto done; |
3110 | } |
3111 | |
3112 | data = ipsec_state.m; |
3113 | if (error || data == NULL) { |
3114 | if (error) { |
3115 | printf("ipsec_output: ipsec4_output error %d.\n" , error); |
3116 | } |
3117 | goto ipsec_output_err; |
3118 | } |
3119 | |
3120 | /* Set traffic class, set flow */ |
3121 | m_set_service_class(data, pcb->ipsec_output_service_class); |
3122 | data->m_pkthdr.pkt_flowsrc = FLOWSRC_IFNET; |
3123 | data->m_pkthdr.pkt_flowid = interface->if_flowhash; |
3124 | data->m_pkthdr.pkt_proto = ip->ip_p; |
3125 | data->m_pkthdr.pkt_flags = (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC); |
3126 | |
3127 | /* Flip endian-ness for ip_output */ |
3128 | ip = mtod(data, struct ip *); |
3129 | NTOHS(ip->ip_len); |
3130 | NTOHS(ip->ip_off); |
3131 | |
3132 | /* Increment statistics */ |
3133 | length = mbuf_pkthdr_len(data); |
3134 | ifnet_stat_increment_out(interface, 1, length, 0); |
3135 | |
3136 | /* Send to ip_output */ |
3137 | memset(&ro, 0, sizeof(ro)); |
3138 | |
3139 | flags = (IP_OUTARGS | /* Passing out args to specify interface */ |
3140 | IP_NOIPSEC); /* To ensure the packet doesn't go through ipsec twice */ |
3141 | |
3142 | memset(&ipoa, 0, sizeof(ipoa)); |
3143 | ipoa.ipoa_flowadv.code = 0; |
3144 | ipoa.ipoa_flags = IPOAF_SELECT_SRCIF | IPOAF_BOUND_SRCADDR; |
3145 | if (ipsec_state.outgoing_if) { |
3146 | ipoa.ipoa_boundif = ipsec_state.outgoing_if; |
3147 | ipoa.ipoa_flags |= IPOAF_BOUND_IF; |
3148 | } |
3149 | ipsec_set_ipoa_for_interface(pcb->ipsec_ifp, &ipoa); |
3150 | |
3151 | adv = &ipoa.ipoa_flowadv; |
3152 | |
3153 | (void)ip_output(data, NULL, &ro, flags, NULL, &ipoa); |
3154 | data = NULL; |
3155 | |
3156 | if (adv->code == FADV_FLOW_CONTROLLED || adv->code == FADV_SUSPENDED) { |
3157 | error = ENOBUFS; |
3158 | ifnet_disable_output(interface); |
3159 | } |
3160 | |
3161 | goto done; |
3162 | } |
3163 | case 6: { |
3164 | #if IPSEC_NEXUS |
3165 | if (!pcb->ipsec_use_netif) |
3166 | #endif // IPSEC_NEXUS |
3167 | { |
3168 | int af = AF_INET6; |
3169 | bpf_tap_out(pcb->ipsec_ifp, DLT_NULL, data, &af, sizeof(af)); |
3170 | } |
3171 | |
3172 | data = ipsec6_splithdr(data); |
3173 | if (data == NULL) { |
3174 | printf("ipsec_output: ipsec6_splithdr returned NULL\n" ); |
3175 | goto ipsec_output_err; |
3176 | } |
3177 | |
3178 | ip6 = mtod(data, struct ip6_hdr *); |
3179 | |
3180 | memset(&ipsec_state, 0, sizeof(ipsec_state)); |
3181 | ipsec_state.m = data; |
3182 | ipsec_state.dst = (struct sockaddr *)&ip6->ip6_dst; |
3183 | memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro)); |
3184 | |
3185 | error = ipsec6_interface_output(&ipsec_state, interface, &ip6->ip6_nxt, ipsec_state.m); |
3186 | if (error == 0 && ipsec_state.tunneled == 4) { /* tunneled in IPv4 - packet is gone */ |
3187 | goto done; |
3188 | } |
3189 | data = ipsec_state.m; |
3190 | if (error || data == NULL) { |
3191 | if (error) { |
3192 | printf("ipsec_output: ipsec6_output error %d\n" , error); |
3193 | } |
3194 | goto ipsec_output_err; |
3195 | } |
3196 | |
3197 | /* Set traffic class, set flow */ |
3198 | m_set_service_class(data, pcb->ipsec_output_service_class); |
3199 | data->m_pkthdr.pkt_flowsrc = FLOWSRC_IFNET; |
3200 | data->m_pkthdr.pkt_flowid = interface->if_flowhash; |
3201 | data->m_pkthdr.pkt_proto = ip6->ip6_nxt; |
3202 | data->m_pkthdr.pkt_flags = (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC); |
3203 | |
3204 | /* Increment statistics */ |
3205 | length = mbuf_pkthdr_len(data); |
3206 | ifnet_stat_increment_out(interface, 1, length, 0); |
3207 | |
3208 | /* Send to ip6_output */ |
3209 | memset(&ro6, 0, sizeof(ro6)); |
3210 | |
3211 | flags = IPV6_OUTARGS; |
3212 | |
3213 | memset(&ip6oa, 0, sizeof(ip6oa)); |
3214 | ip6oa.ip6oa_flowadv.code = 0; |
3215 | ip6oa.ip6oa_flags = IP6OAF_SELECT_SRCIF | IP6OAF_BOUND_SRCADDR; |
3216 | if (ipsec_state.outgoing_if) { |
3217 | ip6oa.ip6oa_boundif = ipsec_state.outgoing_if; |
3218 | ip6oa.ip6oa_flags |= IP6OAF_BOUND_IF; |
3219 | } |
3220 | ipsec_set_ip6oa_for_interface(pcb->ipsec_ifp, &ip6oa); |
3221 | |
3222 | adv = &ip6oa.ip6oa_flowadv; |
3223 | |
3224 | (void) ip6_output(data, NULL, &ro6, flags, NULL, NULL, &ip6oa); |
3225 | data = NULL; |
3226 | |
3227 | if (adv->code == FADV_FLOW_CONTROLLED || adv->code == FADV_SUSPENDED) { |
3228 | error = ENOBUFS; |
3229 | ifnet_disable_output(interface); |
3230 | } |
3231 | |
3232 | goto done; |
3233 | } |
3234 | default: { |
3235 | printf("ipsec_output: Received unknown packet version %d.\n" , ip_version); |
3236 | error = EINVAL; |
3237 | goto ipsec_output_err; |
3238 | } |
3239 | } |
3240 | |
3241 | done: |
3242 | return error; |
3243 | |
3244 | ipsec_output_err: |
3245 | if (data) |
3246 | mbuf_freem(data); |
3247 | goto done; |
3248 | } |
3249 | |
3250 | static void |
3251 | ipsec_start(ifnet_t interface) |
3252 | { |
3253 | mbuf_t data; |
3254 | struct ipsec_pcb *pcb = ifnet_softc(interface); |
3255 | |
3256 | VERIFY(pcb != NULL); |
3257 | for (;;) { |
3258 | if (ifnet_dequeue(interface, &data) != 0) |
3259 | break; |
3260 | if (ipsec_output(interface, data) != 0) |
3261 | break; |
3262 | } |
3263 | } |
3264 | |
3265 | /* Network Interface functions */ |
3266 | static errno_t |
3267 | ipsec_demux(__unused ifnet_t interface, |
3268 | mbuf_t data, |
3269 | __unused char *, |
3270 | protocol_family_t *protocol) |
3271 | { |
3272 | struct ip *ip; |
3273 | u_int ip_version; |
3274 | |
3275 | while (data != NULL && mbuf_len(data) < 1) { |
3276 | data = mbuf_next(data); |
3277 | } |
3278 | |
3279 | if (data == NULL) |
3280 | return ENOENT; |
3281 | |
3282 | ip = mtod(data, struct ip *); |
3283 | ip_version = ip->ip_v; |
3284 | |
3285 | switch(ip_version) { |
3286 | case 4: |
3287 | *protocol = PF_INET; |
3288 | return 0; |
3289 | case 6: |
3290 | *protocol = PF_INET6; |
3291 | return 0; |
3292 | default: |
3293 | break; |
3294 | } |
3295 | |
3296 | return 0; |
3297 | } |
3298 | |
3299 | static errno_t |
3300 | ipsec_add_proto(__unused ifnet_t interface, |
3301 | protocol_family_t protocol, |
3302 | __unused const struct ifnet_demux_desc *demux_array, |
3303 | __unused u_int32_t demux_count) |
3304 | { |
3305 | switch(protocol) { |
3306 | case PF_INET: |
3307 | return 0; |
3308 | case PF_INET6: |
3309 | return 0; |
3310 | default: |
3311 | break; |
3312 | } |
3313 | |
3314 | return ENOPROTOOPT; |
3315 | } |
3316 | |
3317 | static errno_t |
3318 | ipsec_del_proto(__unused ifnet_t interface, |
3319 | __unused protocol_family_t protocol) |
3320 | { |
3321 | return 0; |
3322 | } |
3323 | |
3324 | static errno_t |
3325 | ipsec_ioctl(ifnet_t interface, |
3326 | u_long command, |
3327 | void *data) |
3328 | { |
3329 | #if IPSEC_NEXUS |
3330 | struct ipsec_pcb *pcb = ifnet_softc(interface); |
3331 | #endif |
3332 | errno_t result = 0; |
3333 | |
3334 | switch(command) { |
3335 | case SIOCSIFMTU: { |
3336 | #if IPSEC_NEXUS |
3337 | if (pcb->ipsec_use_netif) { |
3338 | // Make sure we can fit packets in the channel buffers |
3339 | if (((uint64_t)((struct ifreq*)data)->ifr_mtu) > pcb->ipsec_slot_size) { |
3340 | result = EINVAL; |
3341 | } else { |
3342 | ifnet_set_mtu(interface, (uint32_t)((struct ifreq*)data)->ifr_mtu); |
3343 | } |
3344 | } else |
3345 | #endif // IPSEC_NEXUS |
3346 | { |
3347 | ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu); |
3348 | } |
3349 | break; |
3350 | } |
3351 | |
3352 | case SIOCSIFFLAGS: |
3353 | /* ifioctl() takes care of it */ |
3354 | break; |
3355 | |
3356 | default: |
3357 | result = EOPNOTSUPP; |
3358 | } |
3359 | |
3360 | return result; |
3361 | } |
3362 | |
3363 | static void |
3364 | ipsec_detached(ifnet_t interface) |
3365 | { |
3366 | struct ipsec_pcb *pcb = ifnet_softc(interface); |
3367 | (void)ifnet_release(interface); |
3368 | ipsec_free_pcb(pcb, true); |
3369 | } |
3370 | |
3371 | /* Protocol Handlers */ |
3372 | |
3373 | static errno_t |
3374 | ipsec_proto_input(ifnet_t interface, |
3375 | protocol_family_t protocol, |
3376 | mbuf_t m, |
3377 | __unused char *) |
3378 | { |
3379 | mbuf_pkthdr_setrcvif(m, interface); |
3380 | |
3381 | #if IPSEC_NEXUS |
3382 | struct ipsec_pcb *pcb = ifnet_softc(interface); |
3383 | if (!pcb->ipsec_use_netif) |
3384 | #endif // IPSEC_NEXUS |
3385 | { |
3386 | uint32_t af = 0; |
3387 | struct ip *ip = mtod(m, struct ip *); |
3388 | if (ip->ip_v == 4) { |
3389 | af = AF_INET; |
3390 | } else if (ip->ip_v == 6) { |
3391 | af = AF_INET6; |
3392 | } |
3393 | bpf_tap_in(interface, DLT_NULL, m, &af, sizeof(af)); |
3394 | pktap_input(interface, protocol, m, NULL); |
3395 | } |
3396 | |
3397 | int32_t pktlen = m->m_pkthdr.len; |
3398 | if (proto_input(protocol, m) != 0) { |
3399 | ifnet_stat_increment_in(interface, 0, 0, 1); |
3400 | m_freem(m); |
3401 | } else { |
3402 | ifnet_stat_increment_in(interface, 1, pktlen, 0); |
3403 | } |
3404 | |
3405 | return 0; |
3406 | } |
3407 | |
3408 | static errno_t |
3409 | ipsec_proto_pre_output(__unused ifnet_t interface, |
3410 | protocol_family_t protocol, |
3411 | __unused mbuf_t *packet, |
3412 | __unused const struct sockaddr *dest, |
3413 | __unused void *route, |
3414 | __unused char *frame_type, |
3415 | __unused char *link_layer_dest) |
3416 | { |
3417 | |
3418 | *(protocol_family_t *)(void *)frame_type = protocol; |
3419 | return 0; |
3420 | } |
3421 | |
3422 | static errno_t |
3423 | ipsec_attach_proto(ifnet_t interface, |
3424 | protocol_family_t protocol) |
3425 | { |
3426 | struct ifnet_attach_proto_param proto; |
3427 | errno_t result; |
3428 | |
3429 | bzero(&proto, sizeof(proto)); |
3430 | proto.input = ipsec_proto_input; |
3431 | proto.pre_output = ipsec_proto_pre_output; |
3432 | |
3433 | result = ifnet_attach_protocol(interface, protocol, &proto); |
3434 | if (result != 0 && result != EEXIST) { |
3435 | printf("ipsec_attach_inet - ifnet_attach_protocol %d failed: %d\n" , |
3436 | protocol, result); |
3437 | } |
3438 | |
3439 | return result; |
3440 | } |
3441 | |
3442 | errno_t |
3443 | ipsec_inject_inbound_packet(ifnet_t interface, |
3444 | mbuf_t packet) |
3445 | { |
3446 | #if IPSEC_NEXUS |
3447 | struct ipsec_pcb *pcb = ifnet_softc(interface); |
3448 | |
3449 | if (pcb->ipsec_use_netif) { |
3450 | lck_rw_lock_shared(&pcb->ipsec_pcb_lock); |
3451 | |
3452 | lck_mtx_lock(&pcb->ipsec_input_chain_lock); |
3453 | if (pcb->ipsec_input_chain != NULL) { |
3454 | pcb->ipsec_input_chain_last->m_nextpkt = packet; |
3455 | } else { |
3456 | pcb->ipsec_input_chain = packet; |
3457 | } |
3458 | while (packet->m_nextpkt) { |
3459 | VERIFY(packet != packet->m_nextpkt); |
3460 | packet = packet->m_nextpkt; |
3461 | } |
3462 | pcb->ipsec_input_chain_last = packet; |
3463 | lck_mtx_unlock(&pcb->ipsec_input_chain_lock); |
3464 | |
3465 | kern_channel_ring_t rx_ring = pcb->ipsec_netif_rxring; |
3466 | lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); |
3467 | |
3468 | if (rx_ring != NULL) { |
3469 | kern_channel_notify(rx_ring, 0); |
3470 | } |
3471 | |
3472 | return (0); |
3473 | } else |
3474 | #endif // IPSEC_NEXUS |
3475 | { |
3476 | errno_t error; |
3477 | protocol_family_t protocol; |
3478 | if ((error = ipsec_demux(interface, packet, NULL, &protocol)) != 0) { |
3479 | return error; |
3480 | } |
3481 | |
3482 | return ipsec_proto_input(interface, protocol, packet, NULL); |
3483 | } |
3484 | } |
3485 | |
3486 | void |
3487 | ipsec_set_pkthdr_for_interface(ifnet_t interface, mbuf_t packet, int family) |
3488 | { |
3489 | if (packet != NULL && interface != NULL) { |
3490 | struct ipsec_pcb *pcb = ifnet_softc(interface); |
3491 | if (pcb != NULL) { |
3492 | /* Set traffic class, set flow */ |
3493 | m_set_service_class(packet, pcb->ipsec_output_service_class); |
3494 | packet->m_pkthdr.pkt_flowsrc = FLOWSRC_IFNET; |
3495 | packet->m_pkthdr.pkt_flowid = interface->if_flowhash; |
3496 | if (family == AF_INET) { |
3497 | struct ip *ip = mtod(packet, struct ip *); |
3498 | packet->m_pkthdr.pkt_proto = ip->ip_p; |
3499 | } else if (family == AF_INET6) { |
3500 | struct ip6_hdr *ip6 = mtod(packet, struct ip6_hdr *); |
3501 | packet->m_pkthdr.pkt_proto = ip6->ip6_nxt; |
3502 | } |
3503 | packet->m_pkthdr.pkt_flags = (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC); |
3504 | } |
3505 | } |
3506 | } |
3507 | |
3508 | void |
3509 | ipsec_set_ipoa_for_interface(ifnet_t interface, struct ip_out_args *ipoa) |
3510 | { |
3511 | struct ipsec_pcb *pcb; |
3512 | |
3513 | if (interface == NULL || ipoa == NULL) |
3514 | return; |
3515 | pcb = ifnet_softc(interface); |
3516 | |
3517 | if (net_qos_policy_restricted == 0) { |
3518 | ipoa->ipoa_flags |= IPOAF_QOSMARKING_ALLOWED; |
3519 | ipoa->ipoa_sotc = so_svc2tc(pcb->ipsec_output_service_class); |
3520 | } else if (pcb->ipsec_output_service_class != MBUF_SC_VO || |
3521 | net_qos_policy_restrict_avapps != 0) { |
3522 | ipoa->ipoa_flags &= ~IPOAF_QOSMARKING_ALLOWED; |
3523 | } else { |
3524 | ipoa->ipoa_flags |= IP6OAF_QOSMARKING_ALLOWED; |
3525 | ipoa->ipoa_sotc = SO_TC_VO; |
3526 | } |
3527 | } |
3528 | |
3529 | void |
3530 | ipsec_set_ip6oa_for_interface(ifnet_t interface, struct ip6_out_args *ip6oa) |
3531 | { |
3532 | struct ipsec_pcb *pcb; |
3533 | |
3534 | if (interface == NULL || ip6oa == NULL) |
3535 | return; |
3536 | pcb = ifnet_softc(interface); |
3537 | |
3538 | if (net_qos_policy_restricted == 0) { |
3539 | ip6oa->ip6oa_flags |= IPOAF_QOSMARKING_ALLOWED; |
3540 | ip6oa->ip6oa_sotc = so_svc2tc(pcb->ipsec_output_service_class); |
3541 | } else if (pcb->ipsec_output_service_class != MBUF_SC_VO || |
3542 | net_qos_policy_restrict_avapps != 0) { |
3543 | ip6oa->ip6oa_flags &= ~IPOAF_QOSMARKING_ALLOWED; |
3544 | } else { |
3545 | ip6oa->ip6oa_flags |= IP6OAF_QOSMARKING_ALLOWED; |
3546 | ip6oa->ip6oa_sotc = SO_TC_VO; |
3547 | } |
3548 | } |
3549 | |