1 | /* |
2 | * Copyright (c) 2012-2017 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <string.h> |
30 | #include <sys/types.h> |
31 | #include <sys/syslog.h> |
32 | #include <sys/queue.h> |
33 | #include <sys/malloc.h> |
34 | #include <sys/socket.h> |
35 | #include <sys/kpi_mbuf.h> |
36 | #include <sys/mbuf.h> |
37 | #include <sys/domain.h> |
38 | #include <sys/protosw.h> |
39 | #include <sys/socketvar.h> |
40 | #include <sys/kernel.h> |
41 | #include <sys/systm.h> |
42 | #include <sys/kern_control.h> |
43 | #include <sys/ubc.h> |
44 | #include <sys/codesign.h> |
45 | #include <libkern/tree.h> |
46 | #include <kern/locks.h> |
47 | #include <kern/debug.h> |
48 | #include <net/if_var.h> |
49 | #include <net/route.h> |
50 | #include <net/flowhash.h> |
51 | #include <net/ntstat.h> |
52 | #include <netinet/in.h> |
53 | #include <netinet/in_var.h> |
54 | #include <netinet/tcp.h> |
55 | #include <netinet/tcp_var.h> |
56 | #include <netinet/tcp_fsm.h> |
57 | #include <netinet/flow_divert.h> |
58 | #include <netinet/flow_divert_proto.h> |
59 | #if INET6 |
60 | #include <netinet6/in6_pcb.h> |
61 | #include <netinet6/ip6protosw.h> |
62 | #endif /* INET6 */ |
63 | #include <dev/random/randomdev.h> |
64 | #include <libkern/crypto/sha1.h> |
65 | #include <libkern/crypto/crypto_internal.h> |
66 | #include <os/log.h> |
67 | |
68 | #define FLOW_DIVERT_CONNECT_STARTED 0x00000001 |
69 | #define FLOW_DIVERT_READ_CLOSED 0x00000002 |
70 | #define FLOW_DIVERT_WRITE_CLOSED 0x00000004 |
71 | #define FLOW_DIVERT_TUNNEL_RD_CLOSED 0x00000008 |
72 | #define FLOW_DIVERT_TUNNEL_WR_CLOSED 0x00000010 |
73 | #define FLOW_DIVERT_TRANSFERRED 0x00000020 |
74 | #define FLOW_DIVERT_HAS_HMAC 0x00000040 |
75 | |
76 | #define FDLOG(level, pcb, format, ...) \ |
77 | os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " format "\n", (pcb)->hash, __VA_ARGS__) |
78 | |
79 | #define FDLOG0(level, pcb, msg) \ |
80 | os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " msg "\n", (pcb)->hash) |
81 | |
82 | #define FDRETAIN(pcb) if ((pcb) != NULL) OSIncrementAtomic(&(pcb)->ref_count) |
83 | #define FDRELEASE(pcb) \ |
84 | do { \ |
85 | if ((pcb) != NULL && 1 == OSDecrementAtomic(&(pcb)->ref_count)) { \ |
86 | flow_divert_pcb_destroy(pcb); \ |
87 | } \ |
88 | } while (0) |
89 | |
90 | #define FDLOCK(pcb) lck_mtx_lock(&(pcb)->mtx) |
91 | #define FDUNLOCK(pcb) lck_mtx_unlock(&(pcb)->mtx) |
92 | |
93 | #define FD_CTL_SENDBUFF_SIZE (128 * 1024) |
94 | #define FD_CTL_RCVBUFF_SIZE (128 * 1024) |
95 | |
96 | #define GROUP_BIT_CTL_ENQUEUE_BLOCKED 0 |
97 | |
98 | #define GROUP_COUNT_MAX 32 |
99 | #define FLOW_DIVERT_MAX_NAME_SIZE 4096 |
100 | #define FLOW_DIVERT_MAX_KEY_SIZE 1024 |
101 | #define FLOW_DIVERT_MAX_TRIE_MEMORY (1024 * 1024) |
102 | |
103 | struct flow_divert_trie_node |
104 | { |
105 | uint16_t start; |
106 | uint16_t length; |
107 | uint16_t child_map; |
108 | }; |
109 | |
110 | #define CHILD_MAP_SIZE 256 |
111 | #define NULL_TRIE_IDX 0xffff |
112 | #define TRIE_NODE(t, i) ((t)->nodes[(i)]) |
113 | #define TRIE_CHILD(t, i, b) (((t)->child_maps + (CHILD_MAP_SIZE * TRIE_NODE(t, i).child_map))[(b)]) |
114 | #define TRIE_BYTE(t, i) ((t)->bytes[(i)]) |
115 | |
116 | static struct flow_divert_pcb nil_pcb; |
117 | |
118 | decl_lck_rw_data(static, g_flow_divert_group_lck); |
119 | static struct flow_divert_group **g_flow_divert_groups = NULL; |
120 | static uint32_t g_active_group_count = 0; |
121 | |
122 | static lck_grp_attr_t *flow_divert_grp_attr = NULL; |
123 | static lck_attr_t *flow_divert_mtx_attr = NULL; |
124 | static lck_grp_t *flow_divert_mtx_grp = NULL; |
125 | static errno_t g_init_result = 0; |
126 | |
127 | static kern_ctl_ref g_flow_divert_kctl_ref = NULL; |
128 | |
129 | static struct protosw g_flow_divert_in_protosw; |
130 | static struct pr_usrreqs g_flow_divert_in_usrreqs; |
131 | static struct protosw g_flow_divert_in_udp_protosw; |
132 | static struct pr_usrreqs g_flow_divert_in_udp_usrreqs; |
133 | #if INET6 |
134 | static struct ip6protosw g_flow_divert_in6_protosw; |
135 | static struct pr_usrreqs g_flow_divert_in6_usrreqs; |
136 | static struct ip6protosw g_flow_divert_in6_udp_protosw; |
137 | static struct pr_usrreqs g_flow_divert_in6_udp_usrreqs; |
138 | #endif /* INET6 */ |
139 | |
140 | static struct protosw *g_tcp_protosw = NULL; |
141 | static struct ip6protosw *g_tcp6_protosw = NULL; |
142 | static struct protosw *g_udp_protosw = NULL; |
143 | static struct ip6protosw *g_udp6_protosw = NULL; |
144 | |
145 | static errno_t |
146 | flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr, struct sockaddr **dup); |
147 | |
148 | static errno_t |
149 | flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket); |
150 | |
151 | static boolean_t |
152 | flow_divert_is_sockaddr_valid(struct sockaddr *addr); |
153 | |
154 | static int |
155 | flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr); |
156 | |
157 | struct sockaddr * |
158 | flow_divert_get_buffered_target_address(mbuf_t buffer); |
159 | |
160 | static boolean_t |
161 | flow_divert_has_pcb_local_address(const struct inpcb *inp); |
162 | |
163 | static void |
164 | flow_divert_disconnect_socket(struct socket *so); |
165 | |
166 | static inline uint8_t |
167 | flow_divert_syslog_type_to_oslog_type(int syslog_type) |
168 | { |
169 | switch (syslog_type) { |
170 | case LOG_ERR: return OS_LOG_TYPE_ERROR; |
171 | case LOG_INFO: return OS_LOG_TYPE_INFO; |
172 | case LOG_DEBUG: return OS_LOG_TYPE_DEBUG; |
173 | default: return OS_LOG_TYPE_DEFAULT; |
174 | } |
175 | } |
176 | |
177 | static inline int |
178 | flow_divert_pcb_cmp(const struct flow_divert_pcb *pcb_a, const struct flow_divert_pcb *pcb_b) |
179 | { |
180 | return memcmp(&pcb_a->hash, &pcb_b->hash, sizeof(pcb_a->hash)); |
181 | } |
182 | |
183 | RB_PROTOTYPE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp); |
184 | RB_GENERATE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp); |
185 | |
186 | static const char * |
187 | flow_divert_packet_type2str(uint8_t packet_type) |
188 | { |
189 | switch (packet_type) { |
190 | case FLOW_DIVERT_PKT_CONNECT: |
191 | return "connect" ; |
192 | case FLOW_DIVERT_PKT_CONNECT_RESULT: |
193 | return "connect result" ; |
194 | case FLOW_DIVERT_PKT_DATA: |
195 | return "data" ; |
196 | case FLOW_DIVERT_PKT_CLOSE: |
197 | return "close" ; |
198 | case FLOW_DIVERT_PKT_READ_NOTIFY: |
199 | return "read notification" ; |
200 | case FLOW_DIVERT_PKT_PROPERTIES_UPDATE: |
201 | return "properties update" ; |
202 | case FLOW_DIVERT_PKT_APP_MAP_CREATE: |
203 | return "app map create" ; |
204 | default: |
205 | return "unknown" ; |
206 | } |
207 | } |
208 | |
209 | static struct flow_divert_pcb * |
210 | flow_divert_pcb_lookup(uint32_t hash, struct flow_divert_group *group) |
211 | { |
212 | struct flow_divert_pcb key_item; |
213 | struct flow_divert_pcb *fd_cb = NULL; |
214 | |
215 | key_item.hash = hash; |
216 | |
217 | lck_rw_lock_shared(&group->lck); |
218 | fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item); |
219 | FDRETAIN(fd_cb); |
220 | lck_rw_done(&group->lck); |
221 | |
222 | return fd_cb; |
223 | } |
224 | |
225 | static errno_t |
226 | flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit) |
227 | { |
228 | errno_t error = 0; |
229 | struct flow_divert_pcb *exist = NULL; |
230 | struct flow_divert_group *group; |
231 | static uint32_t g_nextkey = 1; |
232 | static uint32_t g_hash_seed = 0; |
233 | int try_count = 0; |
234 | |
235 | if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) { |
236 | return EINVAL; |
237 | } |
238 | |
239 | socket_unlock(fd_cb->so, 0); |
240 | lck_rw_lock_shared(&g_flow_divert_group_lck); |
241 | |
242 | if (g_flow_divert_groups == NULL || g_active_group_count == 0) { |
243 | FDLOG0(LOG_ERR, &nil_pcb, "No active groups, flow divert cannot be used for this socket" ); |
244 | error = ENETUNREACH; |
245 | goto done; |
246 | } |
247 | |
248 | group = g_flow_divert_groups[ctl_unit]; |
249 | if (group == NULL) { |
250 | FDLOG(LOG_ERR, &nil_pcb, "Group for control unit %u is NULL, flow divert cannot be used for this socket" , ctl_unit); |
251 | error = ENETUNREACH; |
252 | goto done; |
253 | } |
254 | |
255 | socket_lock(fd_cb->so, 0); |
256 | |
257 | do { |
258 | uint32_t key[2]; |
259 | uint32_t idx; |
260 | |
261 | key[0] = g_nextkey++; |
262 | key[1] = RandomULong(); |
263 | |
264 | if (g_hash_seed == 0) { |
265 | g_hash_seed = RandomULong(); |
266 | } |
267 | |
268 | fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed); |
269 | |
270 | for (idx = 1; idx < GROUP_COUNT_MAX; idx++) { |
271 | struct flow_divert_group *curr_group = g_flow_divert_groups[idx]; |
272 | if (curr_group != NULL && curr_group != group) { |
273 | lck_rw_lock_shared(&curr_group->lck); |
274 | exist = RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb); |
275 | lck_rw_done(&curr_group->lck); |
276 | if (exist != NULL) { |
277 | break; |
278 | } |
279 | } |
280 | } |
281 | |
282 | if (exist == NULL) { |
283 | lck_rw_lock_exclusive(&group->lck); |
284 | exist = RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb); |
285 | lck_rw_done(&group->lck); |
286 | } |
287 | } while (exist != NULL && try_count++ < 3); |
288 | |
289 | if (exist == NULL) { |
290 | fd_cb->group = group; |
291 | FDRETAIN(fd_cb); /* The group now has a reference */ |
292 | } else { |
293 | fd_cb->hash = 0; |
294 | error = EEXIST; |
295 | } |
296 | |
297 | socket_unlock(fd_cb->so, 0); |
298 | |
299 | done: |
300 | lck_rw_done(&g_flow_divert_group_lck); |
301 | socket_lock(fd_cb->so, 0); |
302 | |
303 | return error; |
304 | } |
305 | |
306 | static struct flow_divert_pcb * |
307 | flow_divert_pcb_create(socket_t so) |
308 | { |
309 | struct flow_divert_pcb *new_pcb = NULL; |
310 | |
311 | MALLOC_ZONE(new_pcb, struct flow_divert_pcb *, sizeof(*new_pcb), M_FLOW_DIVERT_PCB, M_WAITOK); |
312 | if (new_pcb == NULL) { |
313 | FDLOG0(LOG_ERR, &nil_pcb, "failed to allocate a pcb" ); |
314 | return NULL; |
315 | } |
316 | |
317 | memset(new_pcb, 0, sizeof(*new_pcb)); |
318 | |
319 | lck_mtx_init(&new_pcb->mtx, flow_divert_mtx_grp, flow_divert_mtx_attr); |
320 | new_pcb->so = so; |
321 | new_pcb->log_level = nil_pcb.log_level; |
322 | |
323 | FDRETAIN(new_pcb); /* Represents the socket's reference */ |
324 | |
325 | return new_pcb; |
326 | } |
327 | |
328 | static void |
329 | flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb) |
330 | { |
331 | FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %u, app rx %u, tunnel tx %u, tunnel rx %u" , |
332 | fd_cb->bytes_written_by_app, fd_cb->bytes_read_by_app, fd_cb->bytes_sent, fd_cb->bytes_received); |
333 | |
334 | if (fd_cb->local_address != NULL) { |
335 | FREE(fd_cb->local_address, M_SONAME); |
336 | } |
337 | if (fd_cb->remote_address != NULL) { |
338 | FREE(fd_cb->remote_address, M_SONAME); |
339 | } |
340 | if (fd_cb->connect_token != NULL) { |
341 | mbuf_freem(fd_cb->connect_token); |
342 | } |
343 | if (fd_cb->connect_packet != NULL) { |
344 | mbuf_freem(fd_cb->connect_packet); |
345 | } |
346 | if (fd_cb->app_data != NULL) { |
347 | FREE(fd_cb->app_data, M_TEMP); |
348 | } |
349 | FREE_ZONE(fd_cb, sizeof(*fd_cb), M_FLOW_DIVERT_PCB); |
350 | } |
351 | |
352 | static void |
353 | flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb) |
354 | { |
355 | if (fd_cb->group != NULL) { |
356 | struct flow_divert_group *group = fd_cb->group; |
357 | lck_rw_lock_exclusive(&group->lck); |
358 | FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d" , group->ctl_unit, fd_cb->ref_count); |
359 | RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb); |
360 | fd_cb->group = NULL; |
361 | FDRELEASE(fd_cb); /* Release the group's reference */ |
362 | lck_rw_done(&group->lck); |
363 | } |
364 | } |
365 | |
366 | static int |
367 | flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_t *packet) |
368 | { |
369 | struct flow_divert_packet_header hdr; |
370 | int error = 0; |
371 | |
372 | error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, packet); |
373 | if (error) { |
374 | FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d" , error); |
375 | return error; |
376 | } |
377 | |
378 | hdr.packet_type = packet_type; |
379 | hdr.conn_id = htonl(fd_cb->hash); |
380 | |
381 | /* Lay down the header */ |
382 | error = mbuf_copyback(*packet, 0, sizeof(hdr), &hdr, MBUF_DONTWAIT); |
383 | if (error) { |
384 | FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d" , error); |
385 | mbuf_freem(*packet); |
386 | *packet = NULL; |
387 | return error; |
388 | } |
389 | |
390 | return 0; |
391 | } |
392 | |
393 | static int |
394 | flow_divert_packet_append_tlv(mbuf_t packet, uint8_t type, uint32_t length, const void *value) |
395 | { |
396 | uint32_t net_length = htonl(length); |
397 | int error = 0; |
398 | |
399 | error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(type), &type, MBUF_DONTWAIT); |
400 | if (error) { |
401 | FDLOG(LOG_ERR, &nil_pcb, "failed to append the type (%d)" , type); |
402 | return error; |
403 | } |
404 | |
405 | error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(net_length), &net_length, MBUF_DONTWAIT); |
406 | if (error) { |
407 | FDLOG(LOG_ERR, &nil_pcb, "failed to append the length (%u)" , length); |
408 | return error; |
409 | } |
410 | |
411 | error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), length, value, MBUF_DONTWAIT); |
412 | if (error) { |
413 | FDLOG0(LOG_ERR, &nil_pcb, "failed to append the value" ); |
414 | return error; |
415 | } |
416 | |
417 | return error; |
418 | } |
419 | |
420 | static int |
421 | flow_divert_packet_find_tlv(mbuf_t packet, int offset, uint8_t type, int *err, int next) |
422 | { |
423 | size_t cursor = offset; |
424 | int error = 0; |
425 | uint32_t curr_length; |
426 | uint8_t curr_type; |
427 | |
428 | *err = 0; |
429 | |
430 | do { |
431 | if (!next) { |
432 | error = mbuf_copydata(packet, cursor, sizeof(curr_type), &curr_type); |
433 | if (error) { |
434 | *err = ENOENT; |
435 | return -1; |
436 | } |
437 | } else { |
438 | next = 0; |
439 | curr_type = FLOW_DIVERT_TLV_NIL; |
440 | } |
441 | |
442 | if (curr_type != type) { |
443 | cursor += sizeof(curr_type); |
444 | error = mbuf_copydata(packet, cursor, sizeof(curr_length), &curr_length); |
445 | if (error) { |
446 | *err = error; |
447 | return -1; |
448 | } |
449 | |
450 | cursor += (sizeof(curr_length) + ntohl(curr_length)); |
451 | } |
452 | } while (curr_type != type); |
453 | |
454 | return cursor; |
455 | } |
456 | |
457 | static int |
458 | flow_divert_packet_get_tlv(mbuf_t packet, int offset, uint8_t type, size_t buff_len, void *buff, uint32_t *val_size) |
459 | { |
460 | int error = 0; |
461 | uint32_t length; |
462 | int tlv_offset; |
463 | |
464 | tlv_offset = flow_divert_packet_find_tlv(packet, offset, type, &error, 0); |
465 | if (tlv_offset < 0) { |
466 | return error; |
467 | } |
468 | |
469 | error = mbuf_copydata(packet, tlv_offset + sizeof(type), sizeof(length), &length); |
470 | if (error) { |
471 | return error; |
472 | } |
473 | |
474 | length = ntohl(length); |
475 | |
476 | if (val_size != NULL) { |
477 | *val_size = length; |
478 | } |
479 | |
480 | if (buff != NULL && buff_len > 0) { |
481 | size_t to_copy = (length < buff_len) ? length : buff_len; |
482 | error = mbuf_copydata(packet, tlv_offset + sizeof(type) + sizeof(length), to_copy, buff); |
483 | if (error) { |
484 | return error; |
485 | } |
486 | } |
487 | |
488 | return 0; |
489 | } |
490 | |
491 | static int |
492 | flow_divert_packet_compute_hmac(mbuf_t packet, struct flow_divert_group *group, uint8_t *hmac) |
493 | { |
494 | mbuf_t curr_mbuf = packet; |
495 | |
496 | if (g_crypto_funcs == NULL || group->token_key == NULL) { |
497 | return ENOPROTOOPT; |
498 | } |
499 | |
500 | cchmac_di_decl(g_crypto_funcs->ccsha1_di, hmac_ctx); |
501 | g_crypto_funcs->cchmac_init_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, group->token_key_size, group->token_key); |
502 | |
503 | while (curr_mbuf != NULL) { |
504 | g_crypto_funcs->cchmac_update_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, mbuf_len(curr_mbuf), mbuf_data(curr_mbuf)); |
505 | curr_mbuf = mbuf_next(curr_mbuf); |
506 | } |
507 | |
508 | g_crypto_funcs->cchmac_final_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, hmac); |
509 | |
510 | return 0; |
511 | } |
512 | |
513 | static int |
514 | flow_divert_packet_verify_hmac(mbuf_t packet, uint32_t ctl_unit) |
515 | { |
516 | int error = 0; |
517 | struct flow_divert_group *group = NULL; |
518 | int hmac_offset; |
519 | uint8_t packet_hmac[SHA_DIGEST_LENGTH]; |
520 | uint8_t computed_hmac[SHA_DIGEST_LENGTH]; |
521 | mbuf_t tail; |
522 | |
523 | lck_rw_lock_shared(&g_flow_divert_group_lck); |
524 | |
525 | if (g_flow_divert_groups != NULL && g_active_group_count > 0) { |
526 | group = g_flow_divert_groups[ctl_unit]; |
527 | } |
528 | |
529 | if (group == NULL) { |
530 | lck_rw_done(&g_flow_divert_group_lck); |
531 | return ENOPROTOOPT; |
532 | } |
533 | |
534 | lck_rw_lock_shared(&group->lck); |
535 | |
536 | if (group->token_key == NULL) { |
537 | error = ENOPROTOOPT; |
538 | goto done; |
539 | } |
540 | |
541 | hmac_offset = flow_divert_packet_find_tlv(packet, 0, FLOW_DIVERT_TLV_HMAC, &error, 0); |
542 | if (hmac_offset < 0) { |
543 | goto done; |
544 | } |
545 | |
546 | error = flow_divert_packet_get_tlv(packet, hmac_offset, FLOW_DIVERT_TLV_HMAC, sizeof(packet_hmac), packet_hmac, NULL); |
547 | if (error) { |
548 | goto done; |
549 | } |
550 | |
551 | /* Chop off the HMAC TLV */ |
552 | error = mbuf_split(packet, hmac_offset, MBUF_WAITOK, &tail); |
553 | if (error) { |
554 | goto done; |
555 | } |
556 | |
557 | mbuf_free(tail); |
558 | |
559 | error = flow_divert_packet_compute_hmac(packet, group, computed_hmac); |
560 | if (error) { |
561 | goto done; |
562 | } |
563 | |
564 | if (memcmp(packet_hmac, computed_hmac, sizeof(packet_hmac))) { |
565 | FDLOG0(LOG_WARNING, &nil_pcb, "HMAC in token does not match computed HMAC" ); |
566 | error = EINVAL; |
567 | goto done; |
568 | } |
569 | |
570 | done: |
571 | lck_rw_done(&group->lck); |
572 | lck_rw_done(&g_flow_divert_group_lck); |
573 | return error; |
574 | } |
575 | |
576 | static void |
577 | flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, int data_len, Boolean send) |
578 | { |
579 | struct inpcb *inp = NULL; |
580 | struct ifnet *ifp = NULL; |
581 | Boolean cell = FALSE; |
582 | Boolean wifi = FALSE; |
583 | Boolean wired = FALSE; |
584 | |
585 | inp = sotoinpcb(fd_cb->so); |
586 | if (inp == NULL) { |
587 | return; |
588 | } |
589 | |
590 | ifp = inp->inp_last_outifp; |
591 | if (ifp != NULL) { |
592 | cell = IFNET_IS_CELLULAR(ifp); |
593 | wifi = (!cell && IFNET_IS_WIFI(ifp)); |
594 | wired = (!wifi && IFNET_IS_WIRED(ifp)); |
595 | } |
596 | |
597 | if (send) { |
598 | INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1); |
599 | INP_ADD_STAT(inp, cell, wifi, wired, txbytes, data_len); |
600 | } else { |
601 | INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1); |
602 | INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len); |
603 | } |
604 | inp_set_activity_bitmap(inp); |
605 | } |
606 | |
607 | static errno_t |
608 | flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb) |
609 | { |
610 | struct inpcb *inp = NULL; |
611 | |
612 | inp = sotoinpcb(fd_cb->so); |
613 | if (inp && INP_NO_CELLULAR(inp) && inp->inp_last_outifp && |
614 | IFNET_IS_CELLULAR(inp->inp_last_outifp)) |
615 | return EHOSTUNREACH; |
616 | |
617 | return 0; |
618 | } |
619 | |
620 | static errno_t |
621 | flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb) |
622 | { |
623 | struct inpcb *inp = NULL; |
624 | |
625 | inp = sotoinpcb(fd_cb->so); |
626 | if (inp && INP_NO_EXPENSIVE(inp) && inp->inp_last_outifp && |
627 | IFNET_IS_EXPENSIVE(inp->inp_last_outifp)) |
628 | return EHOSTUNREACH; |
629 | |
630 | return 0; |
631 | } |
632 | |
633 | static void |
634 | flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, Boolean tunnel) |
635 | { |
636 | if (how != SHUT_RD) { |
637 | fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED; |
638 | if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) { |
639 | fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED; |
640 | /* If the tunnel is not accepting writes any more, then flush the send buffer */ |
641 | sbflush(&fd_cb->so->so_snd); |
642 | } |
643 | } |
644 | if (how != SHUT_WR) { |
645 | fd_cb->flags |= FLOW_DIVERT_READ_CLOSED; |
646 | if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) { |
647 | fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED; |
648 | } |
649 | } |
650 | } |
651 | |
652 | static uint16_t |
653 | trie_node_alloc(struct flow_divert_trie *trie) |
654 | { |
655 | if (trie->nodes_free_next < trie->nodes_count) { |
656 | uint16_t node_idx = trie->nodes_free_next++; |
657 | TRIE_NODE(trie, node_idx).child_map = NULL_TRIE_IDX; |
658 | return node_idx; |
659 | } else { |
660 | return NULL_TRIE_IDX; |
661 | } |
662 | } |
663 | |
664 | static uint16_t |
665 | trie_child_map_alloc(struct flow_divert_trie *trie) |
666 | { |
667 | if (trie->child_maps_free_next < trie->child_maps_count) { |
668 | return trie->child_maps_free_next++; |
669 | } else { |
670 | return NULL_TRIE_IDX; |
671 | } |
672 | } |
673 | |
674 | static uint16_t |
675 | trie_bytes_move(struct flow_divert_trie *trie, uint16_t bytes_idx, size_t bytes_size) |
676 | { |
677 | uint16_t start = trie->bytes_free_next; |
678 | if (start + bytes_size <= trie->bytes_count) { |
679 | if (start != bytes_idx) { |
680 | memmove(&TRIE_BYTE(trie, start), &TRIE_BYTE(trie, bytes_idx), bytes_size); |
681 | } |
682 | trie->bytes_free_next += bytes_size; |
683 | return start; |
684 | } else { |
685 | return NULL_TRIE_IDX; |
686 | } |
687 | } |
688 | |
689 | static uint16_t |
690 | flow_divert_trie_insert(struct flow_divert_trie *trie, uint16_t string_start, size_t string_len) |
691 | { |
692 | uint16_t current = trie->root; |
693 | uint16_t child = trie->root; |
694 | uint16_t string_end = string_start + string_len; |
695 | uint16_t string_idx = string_start; |
696 | uint16_t string_remainder = string_len; |
697 | |
698 | while (child != NULL_TRIE_IDX) { |
699 | uint16_t parent = current; |
700 | uint16_t node_idx; |
701 | uint16_t current_end; |
702 | |
703 | current = child; |
704 | child = NULL_TRIE_IDX; |
705 | |
706 | current_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length; |
707 | |
708 | for (node_idx = TRIE_NODE(trie, current).start; |
709 | node_idx < current_end && |
710 | string_idx < string_end && |
711 | TRIE_BYTE(trie, node_idx) == TRIE_BYTE(trie, string_idx); |
712 | node_idx++, string_idx++); |
713 | |
714 | string_remainder = string_end - string_idx; |
715 | |
716 | if (node_idx < (TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length)) { |
717 | /* |
718 | * We did not reach the end of the current node's string. |
719 | * We need to split the current node into two: |
720 | * 1. A new node that contains the prefix of the node that matches |
721 | * the prefix of the string being inserted. |
722 | * 2. The current node modified to point to the remainder |
723 | * of the current node's string. |
724 | */ |
725 | uint16_t prefix = trie_node_alloc(trie); |
726 | if (prefix == NULL_TRIE_IDX) { |
727 | FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while splitting an existing node" ); |
728 | return NULL_TRIE_IDX; |
729 | } |
730 | |
731 | /* |
732 | * Prefix points to the portion of the current nodes's string that has matched |
733 | * the input string thus far. |
734 | */ |
735 | TRIE_NODE(trie, prefix).start = TRIE_NODE(trie, current).start; |
736 | TRIE_NODE(trie, prefix).length = (node_idx - TRIE_NODE(trie, current).start); |
737 | |
738 | /* |
739 | * Prefix has the current node as the child corresponding to the first byte |
740 | * after the split. |
741 | */ |
742 | TRIE_NODE(trie, prefix).child_map = trie_child_map_alloc(trie); |
743 | if (TRIE_NODE(trie, prefix).child_map == NULL_TRIE_IDX) { |
744 | FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while splitting an existing node" ); |
745 | return NULL_TRIE_IDX; |
746 | } |
747 | TRIE_CHILD(trie, prefix, TRIE_BYTE(trie, node_idx)) = current; |
748 | |
749 | /* Parent has the prefix as the child correspoding to the first byte in the prefix */ |
750 | TRIE_CHILD(trie, parent, TRIE_BYTE(trie, TRIE_NODE(trie, prefix).start)) = prefix; |
751 | |
752 | /* Current node is adjusted to point to the remainder */ |
753 | TRIE_NODE(trie, current).start = node_idx; |
754 | TRIE_NODE(trie, current).length -= TRIE_NODE(trie, prefix).length; |
755 | |
756 | /* We want to insert the new leaf (if any) as a child of the prefix */ |
757 | current = prefix; |
758 | } |
759 | |
760 | if (string_remainder > 0) { |
761 | /* |
762 | * We still have bytes in the string that have not been matched yet. |
763 | * If the current node has children, iterate to the child corresponding |
764 | * to the next byte in the string. |
765 | */ |
766 | if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) { |
767 | child = TRIE_CHILD(trie, current, TRIE_BYTE(trie, string_idx)); |
768 | } |
769 | } |
770 | } /* while (child != NULL_TRIE_IDX) */ |
771 | |
772 | if (string_remainder > 0) { |
773 | /* Add a new leaf containing the remainder of the string */ |
774 | uint16_t leaf = trie_node_alloc(trie); |
775 | if (leaf == NULL_TRIE_IDX) { |
776 | FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while inserting a new leaf" ); |
777 | return NULL_TRIE_IDX; |
778 | } |
779 | |
780 | TRIE_NODE(trie, leaf).start = trie_bytes_move(trie, string_idx, string_remainder); |
781 | if (TRIE_NODE(trie, leaf).start == NULL_TRIE_IDX) { |
782 | FDLOG0(LOG_ERR, &nil_pcb, "Ran out of bytes while inserting a new leaf" ); |
783 | return NULL_TRIE_IDX; |
784 | } |
785 | TRIE_NODE(trie, leaf).length = string_remainder; |
786 | |
787 | /* Set the new leaf as the child of the current node */ |
788 | if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) { |
789 | TRIE_NODE(trie, current).child_map = trie_child_map_alloc(trie); |
790 | if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) { |
791 | FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while inserting a new leaf" ); |
792 | return NULL_TRIE_IDX; |
793 | } |
794 | } |
795 | TRIE_CHILD(trie, current, TRIE_BYTE(trie, TRIE_NODE(trie, leaf).start)) = leaf; |
796 | current = leaf; |
797 | } /* else duplicate or this string is a prefix of one of the existing strings */ |
798 | |
799 | return current; |
800 | } |
801 | |
802 | #define APPLE_WEBCLIP_ID_PREFIX "com.apple.webapp" |
803 | static uint16_t |
804 | flow_divert_trie_search(struct flow_divert_trie *trie, uint8_t *string_bytes) |
805 | { |
806 | uint16_t current = trie->root; |
807 | uint16_t string_idx = 0; |
808 | |
809 | while (current != NULL_TRIE_IDX) { |
810 | uint16_t next = NULL_TRIE_IDX; |
811 | uint16_t node_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length; |
812 | uint16_t node_idx; |
813 | |
814 | for (node_idx = TRIE_NODE(trie, current).start; |
815 | node_idx < node_end && string_bytes[string_idx] != '\0' && string_bytes[string_idx] == TRIE_BYTE(trie, node_idx); |
816 | node_idx++, string_idx++); |
817 | |
818 | if (node_idx == node_end) { |
819 | if (string_bytes[string_idx] == '\0') { |
820 | return current; /* Got an exact match */ |
821 | } else if (string_idx == strlen(APPLE_WEBCLIP_ID_PREFIX) && |
822 | 0 == strncmp((const char *)string_bytes, APPLE_WEBCLIP_ID_PREFIX, string_idx)) { |
823 | string_bytes[string_idx] = '\0'; |
824 | return current; /* Got an apple webclip id prefix match */ |
825 | } else if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) { |
826 | next = TRIE_CHILD(trie, current, string_bytes[string_idx]); |
827 | } |
828 | } |
829 | current = next; |
830 | } |
831 | |
832 | return NULL_TRIE_IDX; |
833 | } |
834 | |
835 | struct uuid_search_info { |
836 | uuid_t target_uuid; |
837 | char *found_signing_id; |
838 | boolean_t found_multiple_signing_ids; |
839 | proc_t found_proc; |
840 | }; |
841 | |
842 | static int |
843 | flow_divert_find_proc_by_uuid_callout(proc_t p, void *arg) |
844 | { |
845 | struct uuid_search_info *info = (struct uuid_search_info *)arg; |
846 | int result = PROC_RETURNED_DONE; /* By default, we didn't find the process */ |
847 | |
848 | if (info->found_signing_id != NULL) { |
849 | if (!info->found_multiple_signing_ids) { |
850 | /* All processes that were found had the same signing identifier, so just claim this first one and be done. */ |
851 | info->found_proc = p; |
852 | result = PROC_CLAIMED_DONE; |
853 | } else { |
854 | uuid_string_t uuid_str; |
855 | uuid_unparse(info->target_uuid, uuid_str); |
856 | FDLOG(LOG_WARNING, &nil_pcb, "Found multiple processes with UUID %s with different signing identifiers" , uuid_str); |
857 | } |
858 | FREE(info->found_signing_id, M_TEMP); |
859 | info->found_signing_id = NULL; |
860 | } |
861 | |
862 | if (result == PROC_RETURNED_DONE) { |
863 | uuid_string_t uuid_str; |
864 | uuid_unparse(info->target_uuid, uuid_str); |
865 | FDLOG(LOG_WARNING, &nil_pcb, "Failed to find a process with UUID %s" , uuid_str); |
866 | } |
867 | |
868 | return result; |
869 | } |
870 | |
871 | static int |
872 | flow_divert_find_proc_by_uuid_filter(proc_t p, void *arg) |
873 | { |
874 | struct uuid_search_info *info = (struct uuid_search_info *)arg; |
875 | int include = 0; |
876 | |
877 | if (info->found_multiple_signing_ids) { |
878 | return include; |
879 | } |
880 | |
881 | include = (uuid_compare(p->p_uuid, info->target_uuid) == 0); |
882 | if (include) { |
883 | const char *signing_id = cs_identity_get(p); |
884 | if (signing_id != NULL) { |
885 | FDLOG(LOG_INFO, &nil_pcb, "Found process %d with signing identifier %s" , p->p_pid, signing_id); |
886 | size_t signing_id_size = strlen(signing_id) + 1; |
887 | if (info->found_signing_id == NULL) { |
888 | MALLOC(info->found_signing_id, char *, signing_id_size, M_TEMP, M_WAITOK); |
889 | memcpy(info->found_signing_id, signing_id, signing_id_size); |
890 | } else if (memcmp(signing_id, info->found_signing_id, signing_id_size)) { |
891 | info->found_multiple_signing_ids = TRUE; |
892 | } |
893 | } else { |
894 | info->found_multiple_signing_ids = TRUE; |
895 | } |
896 | include = !info->found_multiple_signing_ids; |
897 | } |
898 | |
899 | return include; |
900 | } |
901 | |
902 | static proc_t |
903 | flow_divert_find_proc_by_uuid(uuid_t uuid) |
904 | { |
905 | struct uuid_search_info info; |
906 | |
907 | if (LOG_INFO <= nil_pcb.log_level) { |
908 | uuid_string_t uuid_str; |
909 | uuid_unparse(uuid, uuid_str); |
910 | FDLOG(LOG_INFO, &nil_pcb, "Looking for process with UUID %s" , uuid_str); |
911 | } |
912 | |
913 | memset(&info, 0, sizeof(info)); |
914 | info.found_proc = PROC_NULL; |
915 | uuid_copy(info.target_uuid, uuid); |
916 | |
917 | proc_iterate(PROC_ALLPROCLIST, flow_divert_find_proc_by_uuid_callout, &info, flow_divert_find_proc_by_uuid_filter, &info); |
918 | |
919 | return info.found_proc; |
920 | } |
921 | |
922 | static int |
923 | flow_divert_get_src_proc(struct socket *so, proc_t *proc) |
924 | { |
925 | int release = 0; |
926 | |
927 | if (so->so_flags & SOF_DELEGATED) { |
928 | if ((*proc)->p_pid != so->e_pid) { |
929 | *proc = proc_find(so->e_pid); |
930 | release = 1; |
931 | } else if (uuid_compare((*proc)->p_uuid, so->e_uuid)) { |
932 | *proc = flow_divert_find_proc_by_uuid(so->e_uuid); |
933 | release = 1; |
934 | } |
935 | } else if (*proc == PROC_NULL) { |
936 | *proc = current_proc(); |
937 | } |
938 | |
939 | if (*proc != PROC_NULL) { |
940 | if ((*proc)->p_pid == 0) { |
941 | if (release) { |
942 | proc_rele(*proc); |
943 | } |
944 | release = 0; |
945 | *proc = PROC_NULL; |
946 | } |
947 | } |
948 | |
949 | return release; |
950 | } |
951 | |
952 | static int |
953 | flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_t packet, Boolean enqueue) |
954 | { |
955 | int error; |
956 | |
957 | if (fd_cb->group == NULL) { |
958 | fd_cb->so->so_error = ECONNABORTED; |
959 | flow_divert_disconnect_socket(fd_cb->so); |
960 | return ECONNABORTED; |
961 | } |
962 | |
963 | lck_rw_lock_shared(&fd_cb->group->lck); |
964 | |
965 | if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) { |
966 | error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR); |
967 | } else { |
968 | error = ENOBUFS; |
969 | } |
970 | |
971 | if (error == ENOBUFS) { |
972 | if (enqueue) { |
973 | if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) { |
974 | lck_rw_lock_exclusive(&fd_cb->group->lck); |
975 | } |
976 | MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet); |
977 | error = 0; |
978 | } |
979 | OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits); |
980 | } |
981 | |
982 | lck_rw_done(&fd_cb->group->lck); |
983 | |
984 | return error; |
985 | } |
986 | |
987 | static int |
988 | flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr *to, struct socket *so, proc_t p, mbuf_t *out_connect_packet) |
989 | { |
990 | int error = 0; |
991 | int flow_type = 0; |
992 | char *signing_id = NULL; |
993 | int free_signing_id = 0; |
994 | mbuf_t connect_packet = NULL; |
995 | proc_t src_proc = p; |
996 | int release_proc = 0; |
997 | |
998 | error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet); |
999 | if (error) { |
1000 | goto done; |
1001 | } |
1002 | |
1003 | error = EPERM; |
1004 | |
1005 | if (fd_cb->connect_token != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_HMAC)) { |
1006 | uint32_t sid_size = 0; |
1007 | int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size); |
1008 | if (find_error == 0 && sid_size > 0) { |
1009 | MALLOC(signing_id, char *, sid_size + 1, M_TEMP, M_WAITOK | M_ZERO); |
1010 | if (signing_id != NULL) { |
1011 | flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_id, NULL); |
1012 | FDLOG(LOG_INFO, fd_cb, "Got %s from token" , signing_id); |
1013 | free_signing_id = 1; |
1014 | } |
1015 | } |
1016 | } |
1017 | |
1018 | socket_unlock(so, 0); |
1019 | |
1020 | if (signing_id == NULL) { |
1021 | release_proc = flow_divert_get_src_proc(so, &src_proc); |
1022 | if (src_proc != PROC_NULL) { |
1023 | proc_lock(src_proc); |
1024 | if (src_proc->p_csflags & (CS_VALID|CS_DEBUGGED)) { |
1025 | const char * cs_id; |
1026 | cs_id = cs_identity_get(src_proc); |
1027 | signing_id = __DECONST(char *, cs_id); |
1028 | } else { |
1029 | FDLOG0(LOG_WARNING, fd_cb, "Signature is invalid" ); |
1030 | } |
1031 | } else { |
1032 | FDLOG0(LOG_WARNING, fd_cb, "Failed to determine the current proc" ); |
1033 | } |
1034 | } else { |
1035 | src_proc = PROC_NULL; |
1036 | } |
1037 | |
1038 | if (signing_id != NULL) { |
1039 | uint16_t result = NULL_TRIE_IDX; |
1040 | lck_rw_lock_shared(&fd_cb->group->lck); |
1041 | if (fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP) { |
1042 | result = 1; |
1043 | } else { |
1044 | result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (uint8_t *)signing_id); |
1045 | } |
1046 | lck_rw_done(&fd_cb->group->lck); |
1047 | if (result != NULL_TRIE_IDX) { |
1048 | error = 0; |
1049 | FDLOG(LOG_INFO, fd_cb, "%s matched" , signing_id); |
1050 | |
1051 | error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_SIGNING_ID, strlen(signing_id), signing_id); |
1052 | if (error == 0) { |
1053 | if (src_proc != PROC_NULL) { |
1054 | unsigned char cdhash[SHA1_RESULTLEN]; |
1055 | error = proc_getcdhash(src_proc, cdhash); |
1056 | if (error == 0) { |
1057 | error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CDHASH, sizeof(cdhash), cdhash); |
1058 | if (error) { |
1059 | FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d" , error); |
1060 | } |
1061 | } else { |
1062 | FDLOG(LOG_ERR, fd_cb, "failed to get the cdhash: %d" , error); |
1063 | } |
1064 | } |
1065 | } else { |
1066 | FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d" , error); |
1067 | } |
1068 | } else { |
1069 | FDLOG(LOG_WARNING, fd_cb, "%s did not match" , signing_id); |
1070 | } |
1071 | } else { |
1072 | FDLOG0(LOG_WARNING, fd_cb, "Failed to get the code signing identity" ); |
1073 | if (fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP) { |
1074 | error = 0; |
1075 | } |
1076 | } |
1077 | |
1078 | if (src_proc != PROC_NULL) { |
1079 | proc_unlock(src_proc); |
1080 | if (release_proc) { |
1081 | proc_rele(src_proc); |
1082 | } |
1083 | } |
1084 | socket_lock(so, 0); |
1085 | |
1086 | if (free_signing_id) { |
1087 | FREE(signing_id, M_TEMP); |
1088 | } |
1089 | |
1090 | if (error) { |
1091 | goto done; |
1092 | } |
1093 | |
1094 | error = flow_divert_packet_append_tlv(connect_packet, |
1095 | FLOW_DIVERT_TLV_TRAFFIC_CLASS, |
1096 | sizeof(fd_cb->so->so_traffic_class), |
1097 | &fd_cb->so->so_traffic_class); |
1098 | if (error) { |
1099 | goto done; |
1100 | } |
1101 | |
1102 | if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) { |
1103 | flow_type = FLOW_DIVERT_FLOW_TYPE_TCP; |
1104 | } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) { |
1105 | flow_type = FLOW_DIVERT_FLOW_TYPE_UDP; |
1106 | } else { |
1107 | error = EINVAL; |
1108 | goto done; |
1109 | } |
1110 | error = flow_divert_packet_append_tlv(connect_packet, |
1111 | FLOW_DIVERT_TLV_FLOW_TYPE, |
1112 | sizeof(flow_type), |
1113 | &flow_type); |
1114 | |
1115 | if (error) { |
1116 | goto done; |
1117 | } |
1118 | |
1119 | if (fd_cb->so->so_flags & SOF_DELEGATED) { |
1120 | error = flow_divert_packet_append_tlv(connect_packet, |
1121 | FLOW_DIVERT_TLV_PID, |
1122 | sizeof(fd_cb->so->e_pid), |
1123 | &fd_cb->so->e_pid); |
1124 | if (error) { |
1125 | goto done; |
1126 | } |
1127 | |
1128 | error = flow_divert_packet_append_tlv(connect_packet, |
1129 | FLOW_DIVERT_TLV_UUID, |
1130 | sizeof(fd_cb->so->e_uuid), |
1131 | &fd_cb->so->e_uuid); |
1132 | if (error) { |
1133 | goto done; |
1134 | } |
1135 | } else { |
1136 | error = flow_divert_packet_append_tlv(connect_packet, |
1137 | FLOW_DIVERT_TLV_PID, |
1138 | sizeof(fd_cb->so->e_pid), |
1139 | &fd_cb->so->last_pid); |
1140 | if (error) { |
1141 | goto done; |
1142 | } |
1143 | |
1144 | error = flow_divert_packet_append_tlv(connect_packet, |
1145 | FLOW_DIVERT_TLV_UUID, |
1146 | sizeof(fd_cb->so->e_uuid), |
1147 | &fd_cb->so->last_uuid); |
1148 | if (error) { |
1149 | goto done; |
1150 | } |
1151 | } |
1152 | |
1153 | if (fd_cb->connect_token != NULL) { |
1154 | unsigned int token_len = m_length(fd_cb->connect_token); |
1155 | mbuf_concatenate(connect_packet, fd_cb->connect_token); |
1156 | mbuf_pkthdr_adjustlen(connect_packet, token_len); |
1157 | fd_cb->connect_token = NULL; |
1158 | } else { |
1159 | uint32_t ctl_unit = htonl(fd_cb->control_group_unit); |
1160 | |
1161 | error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit); |
1162 | if (error) { |
1163 | goto done; |
1164 | } |
1165 | |
1166 | error = flow_divert_append_target_endpoint_tlv(connect_packet, to); |
1167 | if (error) { |
1168 | goto done; |
1169 | } |
1170 | } |
1171 | |
1172 | if (fd_cb->local_address != NULL) { |
1173 | error = EALREADY; |
1174 | goto done; |
1175 | } else { |
1176 | struct inpcb *inp = sotoinpcb(so); |
1177 | if (flow_divert_has_pcb_local_address(inp)) { |
1178 | error = flow_divert_inp_to_sockaddr(inp, &fd_cb->local_address); |
1179 | if (error) { |
1180 | FDLOG0(LOG_ERR, fd_cb, "failed to get the local socket address." ); |
1181 | goto done; |
1182 | } |
1183 | } |
1184 | } |
1185 | |
1186 | if (fd_cb->local_address != NULL) { |
1187 | /* socket is bound. */ |
1188 | error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_LOCAL_ADDR, |
1189 | sizeof(struct sockaddr_storage), fd_cb->local_address); |
1190 | if (error) { |
1191 | goto done; |
1192 | } |
1193 | } |
1194 | |
1195 | if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) { |
1196 | uint32_t flags = FLOW_DIVERT_TOKEN_FLAG_TFO; |
1197 | error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags); |
1198 | if (error) { |
1199 | goto done; |
1200 | } |
1201 | } |
1202 | |
1203 | done: |
1204 | if (!error) { |
1205 | *out_connect_packet = connect_packet; |
1206 | } else if (connect_packet != NULL) { |
1207 | mbuf_freem(connect_packet); |
1208 | } |
1209 | |
1210 | return error; |
1211 | } |
1212 | |
1213 | static int |
1214 | flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb) |
1215 | { |
1216 | int error = 0; |
1217 | mbuf_t packet = NULL; |
1218 | int rbuff_space = 0; |
1219 | |
1220 | error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet); |
1221 | if (error) { |
1222 | FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d" , error); |
1223 | goto done; |
1224 | } |
1225 | |
1226 | rbuff_space = fd_cb->so->so_rcv.sb_hiwat; |
1227 | if (rbuff_space < 0) { |
1228 | rbuff_space = 0; |
1229 | } |
1230 | rbuff_space = htonl(rbuff_space); |
1231 | error = flow_divert_packet_append_tlv(packet, |
1232 | FLOW_DIVERT_TLV_SPACE_AVAILABLE, |
1233 | sizeof(rbuff_space), |
1234 | &rbuff_space); |
1235 | if (error) { |
1236 | goto done; |
1237 | } |
1238 | |
1239 | error = flow_divert_send_packet(fd_cb, packet, TRUE); |
1240 | if (error) { |
1241 | goto done; |
1242 | } |
1243 | |
1244 | done: |
1245 | if (error && packet != NULL) { |
1246 | mbuf_freem(packet); |
1247 | } |
1248 | |
1249 | return error; |
1250 | } |
1251 | |
1252 | static int |
1253 | flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how) |
1254 | { |
1255 | int error = 0; |
1256 | mbuf_t packet = NULL; |
1257 | uint32_t zero = 0; |
1258 | |
1259 | error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet); |
1260 | if (error) { |
1261 | FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d" , error); |
1262 | goto done; |
1263 | } |
1264 | |
1265 | error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(zero), &zero); |
1266 | if (error) { |
1267 | FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d" , error); |
1268 | goto done; |
1269 | } |
1270 | |
1271 | how = htonl(how); |
1272 | error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_HOW, sizeof(how), &how); |
1273 | if (error) { |
1274 | FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d" , error); |
1275 | goto done; |
1276 | } |
1277 | |
1278 | error = flow_divert_send_packet(fd_cb, packet, TRUE); |
1279 | if (error) { |
1280 | goto done; |
1281 | } |
1282 | |
1283 | done: |
1284 | if (error && packet != NULL) { |
1285 | mbuf_free(packet); |
1286 | } |
1287 | |
1288 | return error; |
1289 | } |
1290 | |
1291 | static int |
1292 | flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb) |
1293 | { |
1294 | if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) == |
1295 | (FLOW_DIVERT_TUNNEL_RD_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) |
1296 | { |
1297 | return SHUT_RDWR; |
1298 | } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) { |
1299 | return SHUT_RD; |
1300 | } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) { |
1301 | return SHUT_WR; |
1302 | } |
1303 | |
1304 | return -1; |
1305 | } |
1306 | |
1307 | /* |
1308 | * Determine what close messages if any need to be sent to the tunnel. Returns TRUE if the tunnel is closed for both reads and |
1309 | * writes. Returns FALSE otherwise. |
1310 | */ |
1311 | static void |
1312 | flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb) |
1313 | { |
1314 | int how = -1; |
1315 | |
1316 | /* Do not send any close messages if there is still data in the send buffer */ |
1317 | if (fd_cb->so->so_snd.sb_cc == 0) { |
1318 | if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED|FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_CLOSED) { |
1319 | /* Socket closed reads, but tunnel did not. Tell tunnel to close reads */ |
1320 | how = SHUT_RD; |
1321 | } |
1322 | if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRITE_CLOSED) { |
1323 | /* Socket closed writes, but tunnel did not. Tell tunnel to close writes */ |
1324 | if (how == SHUT_RD) { |
1325 | how = SHUT_RDWR; |
1326 | } else { |
1327 | how = SHUT_WR; |
1328 | } |
1329 | } |
1330 | } |
1331 | |
1332 | if (how != -1) { |
1333 | FDLOG(LOG_INFO, fd_cb, "sending close, how = %d" , how); |
1334 | if (flow_divert_send_close(fd_cb, how) != ENOBUFS) { |
1335 | /* Successfully sent the close packet. Record the ways in which the tunnel has been closed */ |
1336 | if (how != SHUT_RD) { |
1337 | fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED; |
1338 | } |
1339 | if (how != SHUT_WR) { |
1340 | fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED; |
1341 | } |
1342 | } |
1343 | } |
1344 | |
1345 | if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) { |
1346 | flow_divert_disconnect_socket(fd_cb->so); |
1347 | } |
1348 | } |
1349 | |
1350 | static errno_t |
1351 | flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, struct sockaddr *toaddr, Boolean force) |
1352 | { |
1353 | mbuf_t packet; |
1354 | mbuf_t last; |
1355 | int error = 0; |
1356 | |
1357 | error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet); |
1358 | if (error) { |
1359 | FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d" , error); |
1360 | return error; |
1361 | } |
1362 | |
1363 | if (toaddr != NULL) { |
1364 | error = flow_divert_append_target_endpoint_tlv(packet, toaddr); |
1365 | if (error) { |
1366 | FDLOG(LOG_ERR, fd_cb, "flow_divert_append_target_endpoint_tlv() failed: %d" , error); |
1367 | return error; |
1368 | } |
1369 | } |
1370 | |
1371 | if (data_len > 0 && data != NULL) { |
1372 | last = m_last(packet); |
1373 | mbuf_setnext(last, data); |
1374 | mbuf_pkthdr_adjustlen(packet, data_len); |
1375 | } |
1376 | error = flow_divert_send_packet(fd_cb, packet, force); |
1377 | |
1378 | if (error) { |
1379 | mbuf_setnext(last, NULL); |
1380 | mbuf_freem(packet); |
1381 | } else { |
1382 | fd_cb->bytes_sent += data_len; |
1383 | flow_divert_add_data_statistics(fd_cb, data_len, TRUE); |
1384 | } |
1385 | |
1386 | return error; |
1387 | } |
1388 | |
1389 | static void |
1390 | flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force) |
1391 | { |
1392 | size_t to_send; |
1393 | size_t sent = 0; |
1394 | int error = 0; |
1395 | mbuf_t buffer; |
1396 | |
1397 | to_send = fd_cb->so->so_snd.sb_cc; |
1398 | buffer = fd_cb->so->so_snd.sb_mb; |
1399 | |
1400 | if (buffer == NULL && to_send > 0) { |
1401 | FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu" , to_send); |
1402 | return; |
1403 | } |
1404 | |
1405 | /* Ignore the send window if force is enabled */ |
1406 | if (!force && (to_send > fd_cb->send_window)) { |
1407 | to_send = fd_cb->send_window; |
1408 | } |
1409 | |
1410 | if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) { |
1411 | while (sent < to_send) { |
1412 | mbuf_t data; |
1413 | size_t data_len; |
1414 | |
1415 | data_len = to_send - sent; |
1416 | if (data_len > FLOW_DIVERT_CHUNK_SIZE) { |
1417 | data_len = FLOW_DIVERT_CHUNK_SIZE; |
1418 | } |
1419 | |
1420 | error = mbuf_copym(buffer, sent, data_len, MBUF_DONTWAIT, &data); |
1421 | if (error) { |
1422 | FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d" , error); |
1423 | break; |
1424 | } |
1425 | |
1426 | error = flow_divert_send_data_packet(fd_cb, data, data_len, NULL, force); |
1427 | if (error) { |
1428 | mbuf_freem(data); |
1429 | break; |
1430 | } |
1431 | |
1432 | sent += data_len; |
1433 | } |
1434 | sbdrop(&fd_cb->so->so_snd, sent); |
1435 | sowwakeup(fd_cb->so); |
1436 | } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) { |
1437 | mbuf_t data; |
1438 | mbuf_t m; |
1439 | size_t data_len; |
1440 | |
1441 | while(buffer) { |
1442 | struct sockaddr *toaddr = flow_divert_get_buffered_target_address(buffer); |
1443 | |
1444 | m = buffer; |
1445 | if (toaddr != NULL) { |
1446 | /* look for data in the chain */ |
1447 | do { |
1448 | m = m->m_next; |
1449 | if (m != NULL && m->m_type == MT_DATA) { |
1450 | break; |
1451 | } |
1452 | } while(m); |
1453 | if (m == NULL) { |
1454 | /* unexpected */ |
1455 | FDLOG0(LOG_ERR, fd_cb, "failed to find type MT_DATA in the mbuf chain." ); |
1456 | goto move_on; |
1457 | } |
1458 | } |
1459 | data_len = mbuf_pkthdr_len(m); |
1460 | if (data_len > 0) { |
1461 | FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu" , data_len); |
1462 | error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data); |
1463 | if (error) { |
1464 | FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d" , error); |
1465 | break; |
1466 | } |
1467 | } else { |
1468 | data = NULL; |
1469 | } |
1470 | error = flow_divert_send_data_packet(fd_cb, data, data_len, toaddr, force); |
1471 | if (error) { |
1472 | mbuf_freem(data); |
1473 | break; |
1474 | } |
1475 | sent += data_len; |
1476 | move_on: |
1477 | buffer = buffer->m_nextpkt; |
1478 | (void) sbdroprecord(&(fd_cb->so->so_snd)); |
1479 | } |
1480 | } |
1481 | |
1482 | if (sent > 0) { |
1483 | FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data" , sent); |
1484 | if (fd_cb->send_window >= sent) { |
1485 | fd_cb->send_window -= sent; |
1486 | } else { |
1487 | fd_cb->send_window = 0; |
1488 | } |
1489 | } |
1490 | } |
1491 | |
1492 | static int |
1493 | flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data, struct sockaddr *toaddr) |
1494 | { |
1495 | size_t to_send = mbuf_pkthdr_len(data); |
1496 | int error = 0; |
1497 | |
1498 | if (to_send > fd_cb->send_window) { |
1499 | to_send = fd_cb->send_window; |
1500 | } |
1501 | |
1502 | if (fd_cb->so->so_snd.sb_cc > 0) { |
1503 | to_send = 0; /* If the send buffer is non-empty, then we can't send anything */ |
1504 | } |
1505 | |
1506 | if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) { |
1507 | size_t sent = 0; |
1508 | mbuf_t remaining_data = data; |
1509 | mbuf_t pkt_data = NULL; |
1510 | while (sent < to_send && remaining_data != NULL) { |
1511 | size_t pkt_data_len; |
1512 | |
1513 | pkt_data = remaining_data; |
1514 | |
1515 | if ((to_send - sent) > FLOW_DIVERT_CHUNK_SIZE) { |
1516 | pkt_data_len = FLOW_DIVERT_CHUNK_SIZE; |
1517 | } else { |
1518 | pkt_data_len = to_send - sent; |
1519 | } |
1520 | |
1521 | if (pkt_data_len < mbuf_pkthdr_len(pkt_data)) { |
1522 | error = mbuf_split(pkt_data, pkt_data_len, MBUF_DONTWAIT, &remaining_data); |
1523 | if (error) { |
1524 | FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d" , error); |
1525 | pkt_data = NULL; |
1526 | break; |
1527 | } |
1528 | } else { |
1529 | remaining_data = NULL; |
1530 | } |
1531 | |
1532 | error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len, NULL, FALSE); |
1533 | |
1534 | if (error) { |
1535 | break; |
1536 | } |
1537 | |
1538 | pkt_data = NULL; |
1539 | sent += pkt_data_len; |
1540 | } |
1541 | |
1542 | fd_cb->send_window -= sent; |
1543 | |
1544 | error = 0; |
1545 | |
1546 | if (pkt_data != NULL) { |
1547 | if (sbspace(&fd_cb->so->so_snd) > 0) { |
1548 | if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) { |
1549 | FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %u\n" , |
1550 | fd_cb->so->so_snd.sb_cc, fd_cb->send_window); |
1551 | } |
1552 | } else { |
1553 | error = ENOBUFS; |
1554 | } |
1555 | } |
1556 | |
1557 | if (remaining_data != NULL) { |
1558 | if (sbspace(&fd_cb->so->so_snd) > 0) { |
1559 | if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) { |
1560 | FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_window = %u\n" , |
1561 | fd_cb->so->so_snd.sb_cc, fd_cb->send_window); |
1562 | } |
1563 | } else { |
1564 | error = ENOBUFS; |
1565 | } |
1566 | } |
1567 | } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) { |
1568 | if (to_send || mbuf_pkthdr_len(data) == 0) { |
1569 | error = flow_divert_send_data_packet(fd_cb, data, to_send, toaddr, FALSE); |
1570 | if (error) { |
1571 | FDLOG(LOG_ERR, fd_cb, "flow_divert_send_data_packet failed. send data size = %lu" , to_send); |
1572 | } else { |
1573 | fd_cb->send_window -= to_send; |
1574 | } |
1575 | } else { |
1576 | /* buffer it */ |
1577 | if (sbspace(&fd_cb->so->so_snd) >= (int)mbuf_pkthdr_len(data)) { |
1578 | if (toaddr != NULL) { |
1579 | if (!sbappendaddr(&fd_cb->so->so_snd, toaddr, data, NULL, &error)) { |
1580 | FDLOG(LOG_ERR, fd_cb, |
1581 | "sbappendaddr failed. send buffer size = %u, send_window = %u, error = %d\n" , |
1582 | fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error); |
1583 | } |
1584 | } else { |
1585 | if (!sbappendrecord(&fd_cb->so->so_snd, data)) { |
1586 | FDLOG(LOG_ERR, fd_cb, |
1587 | "sbappendrecord failed. send buffer size = %u, send_window = %u, error = %d\n" , |
1588 | fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error); |
1589 | } |
1590 | } |
1591 | } else { |
1592 | error = ENOBUFS; |
1593 | } |
1594 | } |
1595 | } |
1596 | |
1597 | return error; |
1598 | } |
1599 | |
1600 | static int |
1601 | flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb, uint32_t read_count) |
1602 | { |
1603 | int error = 0; |
1604 | mbuf_t packet = NULL; |
1605 | uint32_t net_read_count = htonl(read_count); |
1606 | |
1607 | error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet); |
1608 | if (error) { |
1609 | FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d" , error); |
1610 | goto done; |
1611 | } |
1612 | |
1613 | error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_READ_COUNT, sizeof(net_read_count), &net_read_count); |
1614 | if (error) { |
1615 | FDLOG(LOG_ERR, fd_cb, "failed to add the read count: %d" , error); |
1616 | goto done; |
1617 | } |
1618 | |
1619 | error = flow_divert_send_packet(fd_cb, packet, TRUE); |
1620 | if (error) { |
1621 | goto done; |
1622 | } |
1623 | |
1624 | done: |
1625 | if (error && packet != NULL) { |
1626 | mbuf_free(packet); |
1627 | } |
1628 | |
1629 | return error; |
1630 | } |
1631 | |
1632 | static int |
1633 | flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class) |
1634 | { |
1635 | int error = 0; |
1636 | mbuf_t packet = NULL; |
1637 | |
1638 | error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet); |
1639 | if (error) { |
1640 | FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d" , error); |
1641 | goto done; |
1642 | } |
1643 | |
1644 | error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_TRAFFIC_CLASS, sizeof(traffic_class), &traffic_class); |
1645 | if (error) { |
1646 | FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d" , error); |
1647 | goto done; |
1648 | } |
1649 | |
1650 | error = flow_divert_send_packet(fd_cb, packet, TRUE); |
1651 | if (error) { |
1652 | goto done; |
1653 | } |
1654 | |
1655 | done: |
1656 | if (error && packet != NULL) { |
1657 | mbuf_free(packet); |
1658 | } |
1659 | |
1660 | return error; |
1661 | } |
1662 | |
1663 | static void |
1664 | flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset) |
1665 | { |
1666 | uint32_t connect_error; |
1667 | uint32_t ctl_unit = 0; |
1668 | int error = 0; |
1669 | struct flow_divert_group *grp = NULL; |
1670 | struct sockaddr_storage local_address; |
1671 | int out_if_index = 0; |
1672 | struct sockaddr_storage remote_address; |
1673 | uint32_t send_window; |
1674 | uint32_t app_data_length = 0; |
1675 | |
1676 | memset(&local_address, 0, sizeof(local_address)); |
1677 | memset(&remote_address, 0, sizeof(remote_address)); |
1678 | |
1679 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(connect_error), &connect_error, NULL); |
1680 | if (error) { |
1681 | FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d" , error); |
1682 | return; |
1683 | } |
1684 | |
1685 | FDLOG(LOG_INFO, fd_cb, "received connect result %u" , connect_error); |
1686 | |
1687 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_SPACE_AVAILABLE, sizeof(send_window), &send_window, NULL); |
1688 | if (error) { |
1689 | FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d" , error); |
1690 | return; |
1691 | } |
1692 | |
1693 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit, NULL); |
1694 | if (error) { |
1695 | FDLOG0(LOG_INFO, fd_cb, "No control unit provided in the connect result" ); |
1696 | } |
1697 | |
1698 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL); |
1699 | if (error) { |
1700 | FDLOG0(LOG_INFO, fd_cb, "No local address provided" ); |
1701 | } |
1702 | |
1703 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL); |
1704 | if (error) { |
1705 | FDLOG0(LOG_INFO, fd_cb, "No remote address provided" ); |
1706 | } |
1707 | |
1708 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL); |
1709 | if (error) { |
1710 | FDLOG0(LOG_INFO, fd_cb, "No output if index provided" ); |
1711 | } |
1712 | |
1713 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length); |
1714 | if (error) { |
1715 | FDLOG0(LOG_INFO, fd_cb, "No application data provided in connect result" ); |
1716 | } |
1717 | |
1718 | error = 0; |
1719 | connect_error = ntohl(connect_error); |
1720 | ctl_unit = ntohl(ctl_unit); |
1721 | |
1722 | lck_rw_lock_shared(&g_flow_divert_group_lck); |
1723 | |
1724 | if (connect_error == 0 && ctl_unit > 0) { |
1725 | if (ctl_unit >= GROUP_COUNT_MAX) { |
1726 | FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u" , ctl_unit); |
1727 | error = EINVAL; |
1728 | } else if (g_flow_divert_groups == NULL || g_active_group_count == 0) { |
1729 | FDLOG0(LOG_ERR, fd_cb, "No active groups, dropping connection" ); |
1730 | error = EINVAL; |
1731 | } else { |
1732 | grp = g_flow_divert_groups[ctl_unit]; |
1733 | if (grp == NULL) { |
1734 | error = ECONNRESET; |
1735 | } |
1736 | } |
1737 | } |
1738 | |
1739 | FDLOCK(fd_cb); |
1740 | if (fd_cb->so != NULL) { |
1741 | struct inpcb *inp = NULL; |
1742 | struct ifnet *ifp = NULL; |
1743 | struct flow_divert_group *old_group; |
1744 | |
1745 | socket_lock(fd_cb->so, 0); |
1746 | |
1747 | if (!(fd_cb->so->so_state & SS_ISCONNECTING)) { |
1748 | goto done; |
1749 | } |
1750 | |
1751 | inp = sotoinpcb(fd_cb->so); |
1752 | |
1753 | if (connect_error || error) { |
1754 | goto set_socket_state; |
1755 | } |
1756 | |
1757 | if (local_address.ss_family == 0 && fd_cb->local_address == NULL) { |
1758 | error = EINVAL; |
1759 | goto set_socket_state; |
1760 | } |
1761 | if (local_address.ss_family != 0 && fd_cb->local_address == NULL) { |
1762 | if (local_address.ss_len > sizeof(local_address)) { |
1763 | local_address.ss_len = sizeof(local_address); |
1764 | } |
1765 | fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1); |
1766 | } |
1767 | |
1768 | if (remote_address.ss_family != 0) { |
1769 | if (remote_address.ss_len > sizeof(remote_address)) { |
1770 | remote_address.ss_len = sizeof(remote_address); |
1771 | } |
1772 | fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1); |
1773 | } else { |
1774 | error = EINVAL; |
1775 | goto set_socket_state; |
1776 | } |
1777 | |
1778 | if (app_data_length > 0) { |
1779 | uint8_t *app_data = NULL; |
1780 | MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK); |
1781 | if (app_data != NULL) { |
1782 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL); |
1783 | if (error == 0) { |
1784 | FDLOG(LOG_INFO, fd_cb, "Got %u bytes of app data from the connect result" , app_data_length); |
1785 | if (fd_cb->app_data != NULL) { |
1786 | FREE(fd_cb->app_data, M_TEMP); |
1787 | } |
1788 | fd_cb->app_data = app_data; |
1789 | fd_cb->app_data_length = app_data_length; |
1790 | } else { |
1791 | FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the connect result packet" , app_data_length); |
1792 | FREE(app_data, M_TEMP); |
1793 | } |
1794 | } else { |
1795 | FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the connect result" , app_data_length); |
1796 | } |
1797 | } |
1798 | |
1799 | ifnet_head_lock_shared(); |
1800 | if (out_if_index > 0 && out_if_index <= if_index) { |
1801 | ifp = ifindex2ifnet[out_if_index]; |
1802 | } |
1803 | |
1804 | if (ifp != NULL) { |
1805 | inp->inp_last_outifp = ifp; |
1806 | } else { |
1807 | error = EINVAL; |
1808 | } |
1809 | ifnet_head_done(); |
1810 | |
1811 | if (error) { |
1812 | goto set_socket_state; |
1813 | } |
1814 | |
1815 | if (fd_cb->group == NULL) { |
1816 | error = EINVAL; |
1817 | goto set_socket_state; |
1818 | } |
1819 | |
1820 | if (grp != NULL) { |
1821 | old_group = fd_cb->group; |
1822 | |
1823 | lck_rw_lock_exclusive(&old_group->lck); |
1824 | lck_rw_lock_exclusive(&grp->lck); |
1825 | |
1826 | RB_REMOVE(fd_pcb_tree, &old_group->pcb_tree, fd_cb); |
1827 | if (RB_INSERT(fd_pcb_tree, &grp->pcb_tree, fd_cb) != NULL) { |
1828 | panic("group with unit %u already contains a connection with hash %u" , grp->ctl_unit, fd_cb->hash); |
1829 | } |
1830 | |
1831 | fd_cb->group = grp; |
1832 | |
1833 | lck_rw_done(&grp->lck); |
1834 | lck_rw_done(&old_group->lck); |
1835 | } |
1836 | |
1837 | fd_cb->send_window = ntohl(send_window); |
1838 | |
1839 | set_socket_state: |
1840 | if (!connect_error && !error) { |
1841 | FDLOG0(LOG_INFO, fd_cb, "sending connect result" ); |
1842 | error = flow_divert_send_connect_result(fd_cb); |
1843 | } |
1844 | |
1845 | if (connect_error || error) { |
1846 | if (!connect_error) { |
1847 | flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE); |
1848 | fd_cb->so->so_error = error; |
1849 | flow_divert_send_close_if_needed(fd_cb); |
1850 | } else { |
1851 | flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE); |
1852 | fd_cb->so->so_error = connect_error; |
1853 | } |
1854 | flow_divert_disconnect_socket(fd_cb->so); |
1855 | } else { |
1856 | flow_divert_send_buffered_data(fd_cb, FALSE); |
1857 | soisconnected(fd_cb->so); |
1858 | } |
1859 | |
1860 | done: |
1861 | socket_unlock(fd_cb->so, 0); |
1862 | } |
1863 | FDUNLOCK(fd_cb); |
1864 | |
1865 | lck_rw_done(&g_flow_divert_group_lck); |
1866 | } |
1867 | |
1868 | static void |
1869 | flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset) |
1870 | { |
1871 | uint32_t close_error; |
1872 | int error = 0; |
1873 | int how; |
1874 | |
1875 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(close_error), &close_error, NULL); |
1876 | if (error) { |
1877 | FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d" , error); |
1878 | return; |
1879 | } |
1880 | |
1881 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_HOW, sizeof(how), &how, NULL); |
1882 | if (error) { |
1883 | FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d" , error); |
1884 | return; |
1885 | } |
1886 | |
1887 | how = ntohl(how); |
1888 | |
1889 | FDLOG(LOG_INFO, fd_cb, "close received, how = %d" , how); |
1890 | |
1891 | FDLOCK(fd_cb); |
1892 | if (fd_cb->so != NULL) { |
1893 | socket_lock(fd_cb->so, 0); |
1894 | |
1895 | fd_cb->so->so_error = ntohl(close_error); |
1896 | |
1897 | flow_divert_update_closed_state(fd_cb, how, TRUE); |
1898 | |
1899 | how = flow_divert_tunnel_how_closed(fd_cb); |
1900 | if (how == SHUT_RDWR) { |
1901 | flow_divert_disconnect_socket(fd_cb->so); |
1902 | } else if (how == SHUT_RD) { |
1903 | socantrcvmore(fd_cb->so); |
1904 | } else if (how == SHUT_WR) { |
1905 | socantsendmore(fd_cb->so); |
1906 | } |
1907 | |
1908 | socket_unlock(fd_cb->so, 0); |
1909 | } |
1910 | FDUNLOCK(fd_cb); |
1911 | } |
1912 | |
1913 | static mbuf_t |
1914 | flow_divert_get_control_mbuf(struct flow_divert_pcb *fd_cb) |
1915 | { |
1916 | struct inpcb *inp = sotoinpcb(fd_cb->so); |
1917 | if (inp->inp_vflag & INP_IPV4 && inp->inp_flags & INP_RECVDSTADDR) { |
1918 | struct sockaddr_in *sin = (struct sockaddr_in *)(void *)fd_cb->local_address; |
1919 | |
1920 | return sbcreatecontrol((caddr_t) &sin->sin_addr, sizeof(struct in_addr), IP_RECVDSTADDR, IPPROTO_IP); |
1921 | } else if (inp->inp_vflag & INP_IPV6 && (inp->inp_flags & IN6P_PKTINFO) != 0) { |
1922 | struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)(void *)fd_cb->local_address; |
1923 | struct in6_pktinfo pi6; |
1924 | |
1925 | bcopy(&sin6->sin6_addr, &pi6.ipi6_addr, sizeof (struct in6_addr)); |
1926 | pi6.ipi6_ifindex = 0; |
1927 | return sbcreatecontrol((caddr_t)&pi6, sizeof (struct in6_pktinfo), IPV6_PKTINFO, IPPROTO_IPV6); |
1928 | } |
1929 | return (NULL); |
1930 | } |
1931 | |
1932 | static void |
1933 | flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_t packet, size_t offset) |
1934 | { |
1935 | FDLOCK(fd_cb); |
1936 | if (fd_cb->so != NULL) { |
1937 | int error = 0; |
1938 | mbuf_t data = NULL; |
1939 | size_t data_size; |
1940 | struct sockaddr_storage remote_address; |
1941 | boolean_t got_remote_sa = FALSE; |
1942 | |
1943 | socket_lock(fd_cb->so, 0); |
1944 | |
1945 | if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) { |
1946 | uint32_t val_size = 0; |
1947 | |
1948 | /* check if we got remote address with data */ |
1949 | memset(&remote_address, 0, sizeof(remote_address)); |
1950 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, &val_size); |
1951 | if (error || val_size > sizeof(remote_address)) { |
1952 | FDLOG0(LOG_INFO, fd_cb, "No remote address provided" ); |
1953 | error = 0; |
1954 | } else { |
1955 | /* validate the address */ |
1956 | if (flow_divert_is_sockaddr_valid((struct sockaddr *)&remote_address)) { |
1957 | got_remote_sa = TRUE; |
1958 | } |
1959 | offset += (sizeof(uint8_t) + sizeof(uint32_t) + val_size); |
1960 | } |
1961 | } |
1962 | |
1963 | data_size = (mbuf_pkthdr_len(packet) - offset); |
1964 | |
1965 | FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data" , data_size); |
1966 | |
1967 | error = mbuf_split(packet, offset, MBUF_DONTWAIT, &data); |
1968 | if (error || data == NULL) { |
1969 | FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d" , error); |
1970 | } else { |
1971 | if (flow_divert_check_no_cellular(fd_cb) || |
1972 | flow_divert_check_no_expensive(fd_cb)) |
1973 | { |
1974 | flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE); |
1975 | flow_divert_send_close(fd_cb, SHUT_RDWR); |
1976 | flow_divert_disconnect_socket(fd_cb->so); |
1977 | } else if (!(fd_cb->so->so_state & SS_CANTRCVMORE)) { |
1978 | if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) { |
1979 | if (sbappendstream(&fd_cb->so->so_rcv, data)) { |
1980 | fd_cb->bytes_received += data_size; |
1981 | flow_divert_add_data_statistics(fd_cb, data_size, FALSE); |
1982 | fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc; |
1983 | sorwakeup(fd_cb->so); |
1984 | data = NULL; |
1985 | } else { |
1986 | FDLOG0(LOG_ERR, fd_cb, "received data, but appendstream failed" ); |
1987 | } |
1988 | } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) { |
1989 | struct sockaddr *append_sa; |
1990 | mbuf_t mctl; |
1991 | |
1992 | if (got_remote_sa == TRUE) { |
1993 | error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family, |
1994 | (struct sockaddr *)&remote_address, &append_sa); |
1995 | } else { |
1996 | error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family, |
1997 | fd_cb->remote_address, &append_sa); |
1998 | } |
1999 | if (error) { |
2000 | FDLOG0(LOG_ERR, fd_cb, "failed to dup the socket address." ); |
2001 | } |
2002 | |
2003 | mctl = flow_divert_get_control_mbuf(fd_cb); |
2004 | if (sbappendaddr(&fd_cb->so->so_rcv, append_sa, data, mctl, NULL)) { |
2005 | fd_cb->bytes_received += data_size; |
2006 | flow_divert_add_data_statistics(fd_cb, data_size, FALSE); |
2007 | fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc; |
2008 | sorwakeup(fd_cb->so); |
2009 | data = NULL; |
2010 | } else { |
2011 | FDLOG0(LOG_ERR, fd_cb, "received data, but sbappendaddr failed" ); |
2012 | } |
2013 | if (!error) { |
2014 | FREE(append_sa, M_TEMP); |
2015 | } |
2016 | } |
2017 | } |
2018 | } |
2019 | socket_unlock(fd_cb->so, 0); |
2020 | |
2021 | if (data != NULL) { |
2022 | mbuf_freem(data); |
2023 | } |
2024 | } |
2025 | FDUNLOCK(fd_cb); |
2026 | } |
2027 | |
2028 | static void |
2029 | flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset) |
2030 | { |
2031 | uint32_t read_count; |
2032 | int error = 0; |
2033 | |
2034 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_READ_COUNT, sizeof(read_count), &read_count, NULL); |
2035 | if (error) { |
2036 | FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d" , error); |
2037 | return; |
2038 | } |
2039 | |
2040 | FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes" , ntohl(read_count)); |
2041 | |
2042 | FDLOCK(fd_cb); |
2043 | if (fd_cb->so != NULL) { |
2044 | socket_lock(fd_cb->so, 0); |
2045 | fd_cb->send_window += ntohl(read_count); |
2046 | flow_divert_send_buffered_data(fd_cb, FALSE); |
2047 | socket_unlock(fd_cb->so, 0); |
2048 | } |
2049 | FDUNLOCK(fd_cb); |
2050 | } |
2051 | |
2052 | static void |
2053 | flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, int offset) |
2054 | { |
2055 | int error = 0; |
2056 | uint32_t key_size = 0; |
2057 | int log_level; |
2058 | uint32_t flags = 0; |
2059 | |
2060 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size); |
2061 | if (error) { |
2062 | FDLOG(LOG_ERR, &nil_pcb, "failed to get the key size: %d" , error); |
2063 | return; |
2064 | } |
2065 | |
2066 | if (key_size == 0 || key_size > FLOW_DIVERT_MAX_KEY_SIZE) { |
2067 | FDLOG(LOG_ERR, &nil_pcb, "Invalid key size: %u" , key_size); |
2068 | return; |
2069 | } |
2070 | |
2071 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL); |
2072 | if (!error) { |
2073 | nil_pcb.log_level = log_level; |
2074 | } |
2075 | |
2076 | lck_rw_lock_exclusive(&group->lck); |
2077 | |
2078 | MALLOC(group->token_key, uint8_t *, key_size, M_TEMP, M_WAITOK); |
2079 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, key_size, group->token_key, NULL); |
2080 | if (error) { |
2081 | FDLOG(LOG_ERR, &nil_pcb, "failed to get the token key: %d" , error); |
2082 | FREE(group->token_key, M_TEMP); |
2083 | group->token_key = NULL; |
2084 | lck_rw_done(&group->lck); |
2085 | return; |
2086 | } |
2087 | |
2088 | group->token_key_size = key_size; |
2089 | |
2090 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags, NULL); |
2091 | if (!error) { |
2092 | group->flags = flags; |
2093 | } |
2094 | |
2095 | lck_rw_done(&group->lck); |
2096 | } |
2097 | |
2098 | static void |
2099 | flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset) |
2100 | { |
2101 | int error = 0; |
2102 | struct sockaddr_storage local_address; |
2103 | int out_if_index = 0; |
2104 | struct sockaddr_storage remote_address; |
2105 | uint32_t app_data_length = 0; |
2106 | |
2107 | FDLOG0(LOG_INFO, fd_cb, "received a properties update" ); |
2108 | |
2109 | memset(&local_address, 0, sizeof(local_address)); |
2110 | memset(&remote_address, 0, sizeof(remote_address)); |
2111 | |
2112 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL); |
2113 | if (error) { |
2114 | FDLOG0(LOG_INFO, fd_cb, "No local address provided in properties update" ); |
2115 | } |
2116 | |
2117 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL); |
2118 | if (error) { |
2119 | FDLOG0(LOG_INFO, fd_cb, "No remote address provided in properties update" ); |
2120 | } |
2121 | |
2122 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL); |
2123 | if (error) { |
2124 | FDLOG0(LOG_INFO, fd_cb, "No output if index provided in properties update" ); |
2125 | } |
2126 | |
2127 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length); |
2128 | if (error) { |
2129 | FDLOG0(LOG_INFO, fd_cb, "No application data provided in properties update" ); |
2130 | } |
2131 | |
2132 | FDLOCK(fd_cb); |
2133 | if (fd_cb->so != NULL) { |
2134 | socket_lock(fd_cb->so, 0); |
2135 | |
2136 | if (local_address.ss_family != 0) { |
2137 | if (local_address.ss_len > sizeof(local_address)) { |
2138 | local_address.ss_len = sizeof(local_address); |
2139 | } |
2140 | if (fd_cb->local_address != NULL) { |
2141 | FREE(fd_cb->local_address, M_SONAME); |
2142 | fd_cb->local_address = NULL; |
2143 | } |
2144 | fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1); |
2145 | } |
2146 | |
2147 | if (remote_address.ss_family != 0) { |
2148 | if (remote_address.ss_len > sizeof(remote_address)) { |
2149 | remote_address.ss_len = sizeof(remote_address); |
2150 | } |
2151 | if (fd_cb->remote_address != NULL) { |
2152 | FREE(fd_cb->remote_address, M_SONAME); |
2153 | fd_cb->remote_address = NULL; |
2154 | } |
2155 | fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1); |
2156 | } |
2157 | |
2158 | if (out_if_index > 0) { |
2159 | struct inpcb *inp = NULL; |
2160 | struct ifnet *ifp = NULL; |
2161 | |
2162 | inp = sotoinpcb(fd_cb->so); |
2163 | |
2164 | ifnet_head_lock_shared(); |
2165 | if (out_if_index <= if_index) { |
2166 | ifp = ifindex2ifnet[out_if_index]; |
2167 | } |
2168 | |
2169 | if (ifp != NULL) { |
2170 | inp->inp_last_outifp = ifp; |
2171 | } |
2172 | ifnet_head_done(); |
2173 | } |
2174 | |
2175 | if (app_data_length > 0) { |
2176 | uint8_t *app_data = NULL; |
2177 | MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK); |
2178 | if (app_data != NULL) { |
2179 | error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL); |
2180 | if (error == 0) { |
2181 | if (fd_cb->app_data != NULL) { |
2182 | FREE(fd_cb->app_data, M_TEMP); |
2183 | } |
2184 | fd_cb->app_data = app_data; |
2185 | fd_cb->app_data_length = app_data_length; |
2186 | } else { |
2187 | FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the properties update packet" , app_data_length); |
2188 | FREE(app_data, M_TEMP); |
2189 | } |
2190 | } else { |
2191 | FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the properties update" , app_data_length); |
2192 | } |
2193 | } |
2194 | |
2195 | socket_unlock(fd_cb->so, 0); |
2196 | } |
2197 | FDUNLOCK(fd_cb); |
2198 | } |
2199 | |
2200 | static void |
2201 | flow_divert_handle_app_map_create(struct flow_divert_group *group, mbuf_t packet, int offset) |
2202 | { |
2203 | size_t bytes_mem_size; |
2204 | size_t child_maps_mem_size; |
2205 | int cursor; |
2206 | int error = 0; |
2207 | struct flow_divert_trie new_trie; |
2208 | int insert_error = 0; |
2209 | size_t nodes_mem_size; |
2210 | int prefix_count = 0; |
2211 | int signing_id_count = 0; |
2212 | size_t trie_memory_size = 0; |
2213 | |
2214 | lck_rw_lock_exclusive(&group->lck); |
2215 | |
2216 | /* Re-set the current trie */ |
2217 | if (group->signing_id_trie.memory != NULL) { |
2218 | FREE(group->signing_id_trie.memory, M_TEMP); |
2219 | } |
2220 | memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie)); |
2221 | group->signing_id_trie.root = NULL_TRIE_IDX; |
2222 | |
2223 | memset(&new_trie, 0, sizeof(new_trie)); |
2224 | |
2225 | /* Get the number of shared prefixes in the new set of signing ID strings */ |
2226 | flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_PREFIX_COUNT, sizeof(prefix_count), &prefix_count, NULL); |
2227 | |
2228 | if (prefix_count < 0) { |
2229 | lck_rw_done(&group->lck); |
2230 | return; |
2231 | } |
2232 | |
2233 | /* Compute the number of signing IDs and the total amount of bytes needed to store them */ |
2234 | for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0); |
2235 | cursor >= 0; |
2236 | cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) |
2237 | { |
2238 | uint32_t sid_size = 0; |
2239 | flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size); |
2240 | new_trie.bytes_count += sid_size; |
2241 | signing_id_count++; |
2242 | } |
2243 | |
2244 | if (signing_id_count == 0) { |
2245 | lck_rw_done(&group->lck); |
2246 | return; |
2247 | } |
2248 | |
2249 | new_trie.nodes_count = (prefix_count + signing_id_count + 1); /* + 1 for the root node */ |
2250 | new_trie.child_maps_count = (prefix_count + 1); /* + 1 for the root node */ |
2251 | |
2252 | FDLOG(LOG_INFO, &nil_pcb, "Nodes count = %lu, child maps count = %lu, bytes_count = %lu" , |
2253 | new_trie.nodes_count, new_trie.child_maps_count, new_trie.bytes_count); |
2254 | |
2255 | nodes_mem_size = (sizeof(*new_trie.nodes) * new_trie.nodes_count); |
2256 | child_maps_mem_size = (sizeof(*new_trie.child_maps) * CHILD_MAP_SIZE * new_trie.child_maps_count); |
2257 | bytes_mem_size = (sizeof(*new_trie.bytes) * new_trie.bytes_count); |
2258 | |
2259 | trie_memory_size = nodes_mem_size + child_maps_mem_size + bytes_mem_size; |
2260 | if (trie_memory_size > FLOW_DIVERT_MAX_TRIE_MEMORY) { |
2261 | FDLOG(LOG_ERR, &nil_pcb, "Trie memory size (%lu) is too big (maximum is %u)" , trie_memory_size, FLOW_DIVERT_MAX_TRIE_MEMORY); |
2262 | lck_rw_done(&group->lck); |
2263 | return; |
2264 | } |
2265 | |
2266 | MALLOC(new_trie.memory, void *, trie_memory_size, M_TEMP, M_WAITOK); |
2267 | if (new_trie.memory == NULL) { |
2268 | FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate %lu bytes of memory for the signing ID trie" , |
2269 | nodes_mem_size + child_maps_mem_size + bytes_mem_size); |
2270 | lck_rw_done(&group->lck); |
2271 | return; |
2272 | } |
2273 | |
2274 | /* Initialize the free lists */ |
2275 | new_trie.nodes = (struct flow_divert_trie_node *)new_trie.memory; |
2276 | new_trie.nodes_free_next = 0; |
2277 | memset(new_trie.nodes, 0, nodes_mem_size); |
2278 | |
2279 | new_trie.child_maps = (uint16_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size); |
2280 | new_trie.child_maps_free_next = 0; |
2281 | memset(new_trie.child_maps, 0xff, child_maps_mem_size); |
2282 | |
2283 | new_trie.bytes = (uint8_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size + child_maps_mem_size); |
2284 | new_trie.bytes_free_next = 0; |
2285 | |
2286 | /* The root is an empty node */ |
2287 | new_trie.root = trie_node_alloc(&new_trie); |
2288 | |
2289 | /* Add each signing ID to the trie */ |
2290 | for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0); |
2291 | cursor >= 0; |
2292 | cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) |
2293 | { |
2294 | uint32_t sid_size = 0; |
2295 | flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size); |
2296 | if (new_trie.bytes_free_next + sid_size <= new_trie.bytes_count) { |
2297 | uint16_t new_node_idx; |
2298 | flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, &TRIE_BYTE(&new_trie, new_trie.bytes_free_next), NULL); |
2299 | new_node_idx = flow_divert_trie_insert(&new_trie, new_trie.bytes_free_next, sid_size); |
2300 | if (new_node_idx == NULL_TRIE_IDX) { |
2301 | insert_error = EINVAL; |
2302 | break; |
2303 | } |
2304 | } else { |
2305 | FDLOG0(LOG_ERR, &nil_pcb, "No place to put signing ID for insertion" ); |
2306 | insert_error = ENOBUFS; |
2307 | break; |
2308 | } |
2309 | } |
2310 | |
2311 | if (!insert_error) { |
2312 | group->signing_id_trie = new_trie; |
2313 | } else { |
2314 | FREE(new_trie.memory, M_TEMP); |
2315 | } |
2316 | |
2317 | lck_rw_done(&group->lck); |
2318 | } |
2319 | |
2320 | static int |
2321 | flow_divert_input(mbuf_t packet, struct flow_divert_group *group) |
2322 | { |
2323 | struct flow_divert_packet_header hdr; |
2324 | int error = 0; |
2325 | struct flow_divert_pcb *fd_cb; |
2326 | |
2327 | if (mbuf_pkthdr_len(packet) < sizeof(hdr)) { |
2328 | FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) < sizeof hdr (%lu)" , mbuf_pkthdr_len(packet), sizeof(hdr)); |
2329 | error = EINVAL; |
2330 | goto done; |
2331 | } |
2332 | |
2333 | if (mbuf_pkthdr_len(packet) > FD_CTL_RCVBUFF_SIZE) { |
2334 | FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) > %d" , mbuf_pkthdr_len(packet), FD_CTL_RCVBUFF_SIZE); |
2335 | error = EINVAL; |
2336 | goto done; |
2337 | } |
2338 | |
2339 | error = mbuf_copydata(packet, 0, sizeof(hdr), &hdr); |
2340 | if (error) { |
2341 | FDLOG(LOG_ERR, &nil_pcb, "mbuf_copydata failed for the header: %d" , error); |
2342 | error = ENOBUFS; |
2343 | goto done; |
2344 | } |
2345 | |
2346 | hdr.conn_id = ntohl(hdr.conn_id); |
2347 | |
2348 | if (hdr.conn_id == 0) { |
2349 | switch (hdr.packet_type) { |
2350 | case FLOW_DIVERT_PKT_GROUP_INIT: |
2351 | flow_divert_handle_group_init(group, packet, sizeof(hdr)); |
2352 | break; |
2353 | case FLOW_DIVERT_PKT_APP_MAP_CREATE: |
2354 | flow_divert_handle_app_map_create(group, packet, sizeof(hdr)); |
2355 | break; |
2356 | default: |
2357 | FDLOG(LOG_WARNING, &nil_pcb, "got an unknown message type: %d" , hdr.packet_type); |
2358 | break; |
2359 | } |
2360 | goto done; |
2361 | } |
2362 | |
2363 | fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */ |
2364 | if (fd_cb == NULL) { |
2365 | if (hdr.packet_type != FLOW_DIVERT_PKT_CLOSE && hdr.packet_type != FLOW_DIVERT_PKT_READ_NOTIFY) { |
2366 | FDLOG(LOG_NOTICE, &nil_pcb, "got a %s message from group %d for an unknown pcb: %u" , flow_divert_packet_type2str(hdr.packet_type), group->ctl_unit, hdr.conn_id); |
2367 | } |
2368 | goto done; |
2369 | } |
2370 | |
2371 | switch (hdr.packet_type) { |
2372 | case FLOW_DIVERT_PKT_CONNECT_RESULT: |
2373 | flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr)); |
2374 | break; |
2375 | case FLOW_DIVERT_PKT_CLOSE: |
2376 | flow_divert_handle_close(fd_cb, packet, sizeof(hdr)); |
2377 | break; |
2378 | case FLOW_DIVERT_PKT_DATA: |
2379 | flow_divert_handle_data(fd_cb, packet, sizeof(hdr)); |
2380 | break; |
2381 | case FLOW_DIVERT_PKT_READ_NOTIFY: |
2382 | flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr)); |
2383 | break; |
2384 | case FLOW_DIVERT_PKT_PROPERTIES_UPDATE: |
2385 | flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr)); |
2386 | break; |
2387 | default: |
2388 | FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d" , hdr.packet_type); |
2389 | break; |
2390 | } |
2391 | |
2392 | FDRELEASE(fd_cb); |
2393 | |
2394 | done: |
2395 | mbuf_freem(packet); |
2396 | return error; |
2397 | } |
2398 | |
2399 | static void |
2400 | flow_divert_close_all(struct flow_divert_group *group) |
2401 | { |
2402 | struct flow_divert_pcb *fd_cb; |
2403 | SLIST_HEAD(, flow_divert_pcb) tmp_list; |
2404 | |
2405 | SLIST_INIT(&tmp_list); |
2406 | |
2407 | lck_rw_lock_exclusive(&group->lck); |
2408 | |
2409 | MBUFQ_DRAIN(&group->send_queue); |
2410 | |
2411 | RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) { |
2412 | FDRETAIN(fd_cb); |
2413 | SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry); |
2414 | } |
2415 | |
2416 | lck_rw_done(&group->lck); |
2417 | |
2418 | while (!SLIST_EMPTY(&tmp_list)) { |
2419 | fd_cb = SLIST_FIRST(&tmp_list); |
2420 | FDLOCK(fd_cb); |
2421 | SLIST_REMOVE_HEAD(&tmp_list, tmp_list_entry); |
2422 | if (fd_cb->so != NULL) { |
2423 | socket_lock(fd_cb->so, 0); |
2424 | flow_divert_pcb_remove(fd_cb); |
2425 | flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE); |
2426 | fd_cb->so->so_error = ECONNABORTED; |
2427 | flow_divert_disconnect_socket(fd_cb->so); |
2428 | socket_unlock(fd_cb->so, 0); |
2429 | } |
2430 | FDUNLOCK(fd_cb); |
2431 | FDRELEASE(fd_cb); |
2432 | } |
2433 | } |
2434 | |
2435 | void |
2436 | flow_divert_detach(struct socket *so) |
2437 | { |
2438 | struct flow_divert_pcb *fd_cb = so->so_fd_pcb; |
2439 | |
2440 | VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); |
2441 | |
2442 | so->so_flags &= ~SOF_FLOW_DIVERT; |
2443 | so->so_fd_pcb = NULL; |
2444 | |
2445 | FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d" , fd_cb->ref_count); |
2446 | |
2447 | if (fd_cb->group != NULL) { |
2448 | /* Last-ditch effort to send any buffered data */ |
2449 | flow_divert_send_buffered_data(fd_cb, TRUE); |
2450 | |
2451 | flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE); |
2452 | flow_divert_send_close_if_needed(fd_cb); |
2453 | /* Remove from the group */ |
2454 | flow_divert_pcb_remove(fd_cb); |
2455 | } |
2456 | |
2457 | socket_unlock(so, 0); |
2458 | FDLOCK(fd_cb); |
2459 | fd_cb->so = NULL; |
2460 | FDUNLOCK(fd_cb); |
2461 | socket_lock(so, 0); |
2462 | |
2463 | FDRELEASE(fd_cb); /* Release the socket's reference */ |
2464 | } |
2465 | |
2466 | static int |
2467 | flow_divert_close(struct socket *so) |
2468 | { |
2469 | struct flow_divert_pcb *fd_cb = so->so_fd_pcb; |
2470 | |
2471 | VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); |
2472 | |
2473 | FDLOG0(LOG_INFO, fd_cb, "Closing" ); |
2474 | |
2475 | if (SOCK_TYPE(so) == SOCK_STREAM) { |
2476 | soisdisconnecting(so); |
2477 | sbflush(&so->so_rcv); |
2478 | } |
2479 | |
2480 | flow_divert_send_buffered_data(fd_cb, TRUE); |
2481 | flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE); |
2482 | flow_divert_send_close_if_needed(fd_cb); |
2483 | |
2484 | /* Remove from the group */ |
2485 | flow_divert_pcb_remove(fd_cb); |
2486 | |
2487 | return 0; |
2488 | } |
2489 | |
2490 | static int |
2491 | flow_divert_disconnectx(struct socket *so, sae_associd_t aid, |
2492 | sae_connid_t cid __unused) |
2493 | { |
2494 | if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) { |
2495 | return (EINVAL); |
2496 | } |
2497 | |
2498 | return (flow_divert_close(so)); |
2499 | } |
2500 | |
2501 | static int |
2502 | flow_divert_shutdown(struct socket *so) |
2503 | { |
2504 | struct flow_divert_pcb *fd_cb = so->so_fd_pcb; |
2505 | |
2506 | VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); |
2507 | |
2508 | FDLOG0(LOG_INFO, fd_cb, "Can't send more" ); |
2509 | |
2510 | socantsendmore(so); |
2511 | |
2512 | flow_divert_update_closed_state(fd_cb, SHUT_WR, FALSE); |
2513 | flow_divert_send_close_if_needed(fd_cb); |
2514 | |
2515 | return 0; |
2516 | } |
2517 | |
2518 | static int |
2519 | flow_divert_rcvd(struct socket *so, int flags __unused) |
2520 | { |
2521 | struct flow_divert_pcb *fd_cb = so->so_fd_pcb; |
2522 | uint32_t latest_sb_size; |
2523 | uint32_t read_count; |
2524 | |
2525 | VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); |
2526 | |
2527 | latest_sb_size = fd_cb->so->so_rcv.sb_cc; |
2528 | |
2529 | if (fd_cb->sb_size < latest_sb_size) { |
2530 | panic("flow divert rcvd event handler (%u): saved rcv buffer size (%u) is less than latest rcv buffer size (%u)" , |
2531 | fd_cb->hash, fd_cb->sb_size, latest_sb_size); |
2532 | } |
2533 | |
2534 | read_count = fd_cb->sb_size - latest_sb_size; |
2535 | |
2536 | FDLOG(LOG_DEBUG, fd_cb, "app read %u bytes" , read_count); |
2537 | |
2538 | if (read_count > 0 && flow_divert_send_read_notification(fd_cb, read_count) == 0) { |
2539 | fd_cb->bytes_read_by_app += read_count; |
2540 | fd_cb->sb_size = latest_sb_size; |
2541 | } |
2542 | |
2543 | return 0; |
2544 | } |
2545 | |
2546 | static int |
2547 | flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr) |
2548 | { |
2549 | int error = 0; |
2550 | int port = 0; |
2551 | |
2552 | error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_ADDRESS, toaddr->sa_len, toaddr); |
2553 | if (error) { |
2554 | goto done; |
2555 | } |
2556 | |
2557 | if (toaddr->sa_family == AF_INET) { |
2558 | port = ntohs((satosin(toaddr))->sin_port); |
2559 | } |
2560 | #if INET6 |
2561 | else { |
2562 | port = ntohs((satosin6(toaddr))->sin6_port); |
2563 | } |
2564 | #endif |
2565 | |
2566 | error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_PORT, sizeof(port), &port); |
2567 | if (error) { |
2568 | goto done; |
2569 | } |
2570 | |
2571 | done: |
2572 | return error; |
2573 | } |
2574 | |
2575 | struct sockaddr * |
2576 | flow_divert_get_buffered_target_address(mbuf_t buffer) |
2577 | { |
2578 | if (buffer != NULL && buffer->m_type == MT_SONAME) { |
2579 | struct sockaddr *toaddr = mtod(buffer, struct sockaddr *); |
2580 | if (toaddr != NULL && flow_divert_is_sockaddr_valid(toaddr)) { |
2581 | return toaddr; |
2582 | } |
2583 | } |
2584 | return NULL; |
2585 | } |
2586 | |
2587 | static boolean_t |
2588 | flow_divert_is_sockaddr_valid(struct sockaddr *addr) |
2589 | { |
2590 | switch(addr->sa_family) |
2591 | { |
2592 | case AF_INET: |
2593 | if (addr->sa_len != sizeof(struct sockaddr_in)) { |
2594 | return FALSE; |
2595 | } |
2596 | break; |
2597 | #if INET6 |
2598 | case AF_INET6: |
2599 | if (addr->sa_len != sizeof(struct sockaddr_in6)) { |
2600 | return FALSE; |
2601 | } |
2602 | break; |
2603 | #endif /* INET6 */ |
2604 | default: |
2605 | return FALSE; |
2606 | } |
2607 | return TRUE; |
2608 | } |
2609 | |
2610 | static errno_t |
2611 | flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket) |
2612 | { |
2613 | int error = 0; |
2614 | union sockaddr_in_4_6 sin46; |
2615 | |
2616 | bzero(&sin46, sizeof(sin46)); |
2617 | if (inp->inp_vflag & INP_IPV4) { |
2618 | struct sockaddr_in *sin = &sin46.sin; |
2619 | |
2620 | sin->sin_family = AF_INET; |
2621 | sin->sin_len = sizeof(*sin); |
2622 | sin->sin_port = inp->inp_lport; |
2623 | sin->sin_addr = inp->inp_laddr; |
2624 | } else if (inp->inp_vflag & INP_IPV6) { |
2625 | struct sockaddr_in6 *sin6 = &sin46.sin6; |
2626 | |
2627 | sin6->sin6_len = sizeof(*sin6); |
2628 | sin6->sin6_family = AF_INET6; |
2629 | sin6->sin6_port = inp->inp_lport; |
2630 | sin6->sin6_addr = inp->in6p_laddr; |
2631 | } |
2632 | *local_socket = dup_sockaddr((struct sockaddr *)&sin46, 1); |
2633 | if (*local_socket == NULL) { |
2634 | error = ENOBUFS; |
2635 | } |
2636 | return (error); |
2637 | } |
2638 | |
2639 | static boolean_t |
2640 | flow_divert_has_pcb_local_address(const struct inpcb *inp) |
2641 | { |
2642 | return (inp->inp_lport != 0 |
2643 | && (inp->inp_laddr.s_addr != INADDR_ANY || !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr))); |
2644 | } |
2645 | |
2646 | static errno_t |
2647 | flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr, |
2648 | struct sockaddr **dup) |
2649 | { |
2650 | int error = 0; |
2651 | struct sockaddr *result; |
2652 | struct sockaddr_storage ss; |
2653 | |
2654 | if (addr != NULL) { |
2655 | result = addr; |
2656 | } else { |
2657 | memset(&ss, 0, sizeof(ss)); |
2658 | ss.ss_family = family; |
2659 | if (ss.ss_family == AF_INET) { |
2660 | ss.ss_len = sizeof(struct sockaddr_in); |
2661 | } |
2662 | #if INET6 |
2663 | else if (ss.ss_family == AF_INET6) { |
2664 | ss.ss_len = sizeof(struct sockaddr_in6); |
2665 | } |
2666 | #endif /* INET6 */ |
2667 | else { |
2668 | error = EINVAL; |
2669 | } |
2670 | result = (struct sockaddr *)&ss; |
2671 | } |
2672 | |
2673 | if (!error) { |
2674 | *dup = dup_sockaddr(result, 1); |
2675 | if (*dup == NULL) { |
2676 | error = ENOBUFS; |
2677 | } |
2678 | } |
2679 | |
2680 | return error; |
2681 | } |
2682 | |
2683 | static void |
2684 | flow_divert_disconnect_socket(struct socket *so) |
2685 | { |
2686 | soisdisconnected(so); |
2687 | if (SOCK_TYPE(so) == SOCK_DGRAM) { |
2688 | struct inpcb *inp = NULL; |
2689 | |
2690 | inp = sotoinpcb(so); |
2691 | if (inp != NULL) { |
2692 | #if INET6 |
2693 | if (SOCK_CHECK_DOM(so, PF_INET6)) |
2694 | in6_pcbdetach(inp); |
2695 | else |
2696 | #endif /* INET6 */ |
2697 | in_pcbdetach(inp); |
2698 | } |
2699 | } |
2700 | } |
2701 | |
2702 | static errno_t |
2703 | flow_divert_getpeername(struct socket *so, struct sockaddr **sa) |
2704 | { |
2705 | struct flow_divert_pcb *fd_cb = so->so_fd_pcb; |
2706 | |
2707 | VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); |
2708 | |
2709 | return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family, |
2710 | fd_cb->remote_address, |
2711 | sa); |
2712 | } |
2713 | |
2714 | static errno_t |
2715 | flow_divert_getsockaddr(struct socket *so, struct sockaddr **sa) |
2716 | { |
2717 | struct flow_divert_pcb *fd_cb = so->so_fd_pcb; |
2718 | |
2719 | VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); |
2720 | |
2721 | return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family, |
2722 | fd_cb->local_address, |
2723 | sa); |
2724 | } |
2725 | |
2726 | static errno_t |
2727 | flow_divert_ctloutput(struct socket *so, struct sockopt *sopt) |
2728 | { |
2729 | struct flow_divert_pcb *fd_cb = so->so_fd_pcb; |
2730 | |
2731 | VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); |
2732 | |
2733 | if (sopt->sopt_name == SO_TRAFFIC_CLASS) { |
2734 | if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) { |
2735 | flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class); |
2736 | } |
2737 | } |
2738 | |
2739 | if (SOCK_DOM(so) == PF_INET) { |
2740 | return g_tcp_protosw->pr_ctloutput(so, sopt); |
2741 | } |
2742 | #if INET6 |
2743 | else if (SOCK_DOM(so) == PF_INET6) { |
2744 | return g_tcp6_protosw->pr_ctloutput(so, sopt); |
2745 | } |
2746 | #endif |
2747 | return 0; |
2748 | } |
2749 | |
2750 | errno_t |
2751 | flow_divert_connect_out(struct socket *so, struct sockaddr *to, proc_t p) |
2752 | { |
2753 | struct flow_divert_pcb *fd_cb = so->so_fd_pcb; |
2754 | int error = 0; |
2755 | struct inpcb *inp = sotoinpcb(so); |
2756 | struct sockaddr_in *sinp; |
2757 | mbuf_t connect_packet = NULL; |
2758 | int do_send = 1; |
2759 | |
2760 | VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); |
2761 | |
2762 | if (fd_cb->group == NULL) { |
2763 | error = ENETUNREACH; |
2764 | goto done; |
2765 | } |
2766 | |
2767 | if (inp == NULL) { |
2768 | error = EINVAL; |
2769 | goto done; |
2770 | } else if (inp->inp_state == INPCB_STATE_DEAD) { |
2771 | if (so->so_error) { |
2772 | error = so->so_error; |
2773 | so->so_error = 0; |
2774 | } else { |
2775 | error = EINVAL; |
2776 | } |
2777 | goto done; |
2778 | } |
2779 | |
2780 | if ((fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && !(fd_cb->flags & FLOW_DIVERT_TRANSFERRED)) { |
2781 | error = EALREADY; |
2782 | goto done; |
2783 | } |
2784 | |
2785 | if (fd_cb->flags & FLOW_DIVERT_TRANSFERRED) { |
2786 | FDLOG0(LOG_INFO, fd_cb, "fully transferred" ); |
2787 | fd_cb->flags &= ~FLOW_DIVERT_TRANSFERRED; |
2788 | if (fd_cb->remote_address != NULL) { |
2789 | soisconnected(fd_cb->so); |
2790 | goto done; |
2791 | } |
2792 | } |
2793 | |
2794 | FDLOG0(LOG_INFO, fd_cb, "Connecting" ); |
2795 | |
2796 | if (fd_cb->connect_packet == NULL) { |
2797 | if (to == NULL) { |
2798 | FDLOG0(LOG_ERR, fd_cb, "No destination address available when creating connect packet" ); |
2799 | error = EINVAL; |
2800 | goto done; |
2801 | } |
2802 | |
2803 | sinp = (struct sockaddr_in *)(void *)to; |
2804 | if (sinp->sin_family == AF_INET && IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) { |
2805 | error = EAFNOSUPPORT; |
2806 | goto done; |
2807 | } |
2808 | |
2809 | error = flow_divert_create_connect_packet(fd_cb, to, so, p, &connect_packet); |
2810 | if (error) { |
2811 | goto done; |
2812 | } |
2813 | |
2814 | if (so->so_flags1 & SOF1_PRECONNECT_DATA) { |
2815 | FDLOG0(LOG_INFO, fd_cb, "Delaying sending the connect packet until send or receive" ); |
2816 | do_send = 0; |
2817 | } |
2818 | } else { |
2819 | FDLOG0(LOG_INFO, fd_cb, "Sending saved connect packet" ); |
2820 | connect_packet = fd_cb->connect_packet; |
2821 | fd_cb->connect_packet = NULL; |
2822 | } |
2823 | |
2824 | if (do_send) { |
2825 | error = flow_divert_send_packet(fd_cb, connect_packet, TRUE); |
2826 | if (error) { |
2827 | goto done; |
2828 | } |
2829 | |
2830 | fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED; |
2831 | } else { |
2832 | fd_cb->connect_packet = connect_packet; |
2833 | connect_packet = NULL; |
2834 | } |
2835 | |
2836 | soisconnecting(so); |
2837 | |
2838 | done: |
2839 | if (error && connect_packet != NULL) { |
2840 | mbuf_freem(connect_packet); |
2841 | } |
2842 | return error; |
2843 | } |
2844 | |
2845 | static int |
2846 | flow_divert_connectx_out_common(struct socket *so, struct sockaddr *dst, |
2847 | struct proc *p, sae_connid_t *pcid, struct uio *auio, user_ssize_t *bytes_written) |
2848 | { |
2849 | struct inpcb *inp = sotoinpcb(so); |
2850 | int error; |
2851 | |
2852 | if (inp == NULL) { |
2853 | return (EINVAL); |
2854 | } |
2855 | |
2856 | VERIFY(dst != NULL); |
2857 | |
2858 | error = flow_divert_connect_out(so, dst, p); |
2859 | |
2860 | if (error != 0) { |
2861 | return error; |
2862 | } |
2863 | |
2864 | /* if there is data, send it */ |
2865 | if (auio != NULL) { |
2866 | user_ssize_t datalen = 0; |
2867 | |
2868 | socket_unlock(so, 0); |
2869 | |
2870 | VERIFY(bytes_written != NULL); |
2871 | |
2872 | datalen = uio_resid(auio); |
2873 | error = so->so_proto->pr_usrreqs->pru_sosend(so, NULL, (uio_t)auio, NULL, NULL, 0); |
2874 | socket_lock(so, 0); |
2875 | |
2876 | if (error == 0 || error == EWOULDBLOCK) { |
2877 | *bytes_written = datalen - uio_resid(auio); |
2878 | } |
2879 | |
2880 | /* |
2881 | * sosend returns EWOULDBLOCK if it's a non-blocking |
2882 | * socket or a timeout occured (this allows to return |
2883 | * the amount of queued data through sendit()). |
2884 | * |
2885 | * However, connectx() returns EINPROGRESS in case of a |
2886 | * blocking socket. So we change the return value here. |
2887 | */ |
2888 | if (error == EWOULDBLOCK) { |
2889 | error = EINPROGRESS; |
2890 | } |
2891 | } |
2892 | |
2893 | if (error == 0 && pcid != NULL) { |
2894 | *pcid = 1; /* there is only 1 connection for a TCP */ |
2895 | } |
2896 | |
2897 | return (error); |
2898 | } |
2899 | |
2900 | static int |
2901 | flow_divert_connectx_out(struct socket *so, struct sockaddr *src __unused, |
2902 | struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused, |
2903 | sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused, |
2904 | uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written) |
2905 | { |
2906 | return (flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written)); |
2907 | } |
2908 | |
2909 | #if INET6 |
2910 | static int |
2911 | flow_divert_connectx6_out(struct socket *so, struct sockaddr *src __unused, |
2912 | struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused, |
2913 | sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused, |
2914 | uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written) |
2915 | { |
2916 | return (flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written)); |
2917 | } |
2918 | #endif /* INET6 */ |
2919 | |
2920 | static int |
2921 | flow_divert_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags, |
2922 | uint32_t *ifindex, int32_t *soerror, user_addr_t src, socklen_t *src_len, |
2923 | user_addr_t dst, socklen_t *dst_len, uint32_t *aux_type, |
2924 | user_addr_t aux_data __unused, uint32_t *aux_len) |
2925 | { |
2926 | int error = 0; |
2927 | struct flow_divert_pcb *fd_cb = so->so_fd_pcb; |
2928 | struct ifnet *ifp = NULL; |
2929 | struct inpcb *inp = sotoinpcb(so); |
2930 | |
2931 | VERIFY((so->so_flags & SOF_FLOW_DIVERT)); |
2932 | |
2933 | if (so->so_fd_pcb == NULL || inp == NULL) { |
2934 | error = EINVAL; |
2935 | goto out; |
2936 | } |
2937 | |
2938 | if (cid != SAE_CONNID_ANY && cid != SAE_CONNID_ALL && cid != 1) { |
2939 | error = EINVAL; |
2940 | goto out; |
2941 | } |
2942 | |
2943 | ifp = inp->inp_last_outifp; |
2944 | *ifindex = ((ifp != NULL) ? ifp->if_index : 0); |
2945 | *soerror = so->so_error; |
2946 | *flags = 0; |
2947 | |
2948 | if (so->so_state & SS_ISCONNECTED) { |
2949 | *flags |= (CIF_CONNECTED | CIF_PREFERRED); |
2950 | } |
2951 | |
2952 | if (fd_cb->local_address == NULL) { |
2953 | struct sockaddr_in sin; |
2954 | bzero(&sin, sizeof(sin)); |
2955 | sin.sin_len = sizeof(sin); |
2956 | sin.sin_family = AF_INET; |
2957 | *src_len = sin.sin_len; |
2958 | if (src != USER_ADDR_NULL) { |
2959 | error = copyout(&sin, src, sin.sin_len); |
2960 | if (error != 0) { |
2961 | goto out; |
2962 | } |
2963 | } |
2964 | } else { |
2965 | *src_len = fd_cb->local_address->sa_len; |
2966 | if (src != USER_ADDR_NULL) { |
2967 | error = copyout(fd_cb->local_address, src, fd_cb->local_address->sa_len); |
2968 | if (error != 0) { |
2969 | goto out; |
2970 | } |
2971 | } |
2972 | } |
2973 | |
2974 | if (fd_cb->remote_address == NULL) { |
2975 | struct sockaddr_in sin; |
2976 | bzero(&sin, sizeof(sin)); |
2977 | sin.sin_len = sizeof(sin); |
2978 | sin.sin_family = AF_INET; |
2979 | *dst_len = sin.sin_len; |
2980 | if (dst != USER_ADDR_NULL) { |
2981 | error = copyout(&sin, dst, sin.sin_len); |
2982 | if (error != 0) { |
2983 | goto out; |
2984 | } |
2985 | } |
2986 | } else { |
2987 | *dst_len = fd_cb->remote_address->sa_len; |
2988 | if (dst != USER_ADDR_NULL) { |
2989 | error = copyout(fd_cb->remote_address, dst, fd_cb->remote_address->sa_len); |
2990 | if (error != 0) { |
2991 | goto out; |
2992 | } |
2993 | } |
2994 | } |
2995 | |
2996 | *aux_type = 0; |
2997 | *aux_len = 0; |
2998 | |
2999 | out: |
3000 | return error; |
3001 | } |
3002 | |
3003 | static int |
3004 | flow_divert_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp __unused, struct proc *p __unused) |
3005 | { |
3006 | int error = 0; |
3007 | |
3008 | switch (cmd) { |
3009 | case SIOCGCONNINFO32: { |
3010 | struct so_cinforeq32 cifr; |
3011 | bcopy(data, &cifr, sizeof (cifr)); |
3012 | error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags, |
3013 | &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src, |
3014 | &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len, |
3015 | &cifr.scir_aux_type, cifr.scir_aux_data, |
3016 | &cifr.scir_aux_len); |
3017 | if (error == 0) { |
3018 | bcopy(&cifr, data, sizeof (cifr)); |
3019 | } |
3020 | break; |
3021 | } |
3022 | |
3023 | case SIOCGCONNINFO64: { |
3024 | struct so_cinforeq64 cifr; |
3025 | bcopy(data, &cifr, sizeof (cifr)); |
3026 | error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags, |
3027 | &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src, |
3028 | &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len, |
3029 | &cifr.scir_aux_type, cifr.scir_aux_data, |
3030 | &cifr.scir_aux_len); |
3031 | if (error == 0) { |
3032 | bcopy(&cifr, data, sizeof (cifr)); |
3033 | } |
3034 | break; |
3035 | } |
3036 | |
3037 | default: |
3038 | error = EOPNOTSUPP; |
3039 | } |
3040 | |
3041 | return error; |
3042 | } |
3043 | |
3044 | static int |
3045 | flow_divert_in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p) |
3046 | { |
3047 | int error = flow_divert_control(so, cmd, data, ifp, p); |
3048 | |
3049 | if (error == EOPNOTSUPP) { |
3050 | error = in_control(so, cmd, data, ifp, p); |
3051 | } |
3052 | |
3053 | return error; |
3054 | } |
3055 | |
3056 | static int |
3057 | flow_divert_in6_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p) |
3058 | { |
3059 | int error = flow_divert_control(so, cmd, data, ifp, p); |
3060 | |
3061 | if (error == EOPNOTSUPP) { |
3062 | error = in6_control(so, cmd, data, ifp, p); |
3063 | } |
3064 | |
3065 | return error; |
3066 | } |
3067 | |
3068 | static errno_t |
3069 | flow_divert_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p) |
3070 | { |
3071 | struct flow_divert_pcb *fd_cb = so->so_fd_pcb; |
3072 | int error = 0; |
3073 | struct inpcb *inp; |
3074 | |
3075 | VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); |
3076 | |
3077 | inp = sotoinpcb(so); |
3078 | if (inp == NULL || inp->inp_state == INPCB_STATE_DEAD) { |
3079 | error = ECONNRESET; |
3080 | goto done; |
3081 | } |
3082 | |
3083 | if (control && mbuf_len(control) > 0) { |
3084 | error = EINVAL; |
3085 | goto done; |
3086 | } |
3087 | |
3088 | if (flags & MSG_OOB) { |
3089 | error = EINVAL; |
3090 | goto done; /* We don't support OOB data */ |
3091 | } |
3092 | |
3093 | error = flow_divert_check_no_cellular(fd_cb) || |
3094 | flow_divert_check_no_expensive(fd_cb); |
3095 | if (error) { |
3096 | goto done; |
3097 | } |
3098 | |
3099 | /* Implicit connect */ |
3100 | if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) { |
3101 | FDLOG0(LOG_INFO, fd_cb, "implicit connect" ); |
3102 | error = flow_divert_connect_out(so, to, p); |
3103 | if (error) { |
3104 | goto done; |
3105 | } |
3106 | |
3107 | if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) { |
3108 | /* Open up the send window so that the data will get sent right away */ |
3109 | fd_cb->send_window = mbuf_pkthdr_len(data); |
3110 | } |
3111 | } |
3112 | |
3113 | FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes" , mbuf_pkthdr_len(data)); |
3114 | |
3115 | fd_cb->bytes_written_by_app += mbuf_pkthdr_len(data); |
3116 | error = flow_divert_send_app_data(fd_cb, data, to); |
3117 | if (error) { |
3118 | goto done; |
3119 | } |
3120 | |
3121 | data = NULL; |
3122 | |
3123 | if (flags & PRUS_EOF) { |
3124 | flow_divert_shutdown(so); |
3125 | } |
3126 | |
3127 | done: |
3128 | if (data) { |
3129 | mbuf_freem(data); |
3130 | } |
3131 | if (control) { |
3132 | mbuf_free(control); |
3133 | } |
3134 | return error; |
3135 | } |
3136 | |
3137 | static int |
3138 | flow_divert_preconnect(struct socket *so) |
3139 | { |
3140 | struct flow_divert_pcb *fd_cb = so->so_fd_pcb; |
3141 | int error = 0; |
3142 | |
3143 | if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && fd_cb->connect_packet != NULL) { |
3144 | FDLOG0(LOG_INFO, fd_cb, "Pre-connect read: sending saved connect packet" ); |
3145 | mbuf_t connect_packet = fd_cb->connect_packet; |
3146 | fd_cb->connect_packet = NULL; |
3147 | |
3148 | error = flow_divert_send_packet(fd_cb, connect_packet, TRUE); |
3149 | if (error) { |
3150 | mbuf_freem(connect_packet); |
3151 | } |
3152 | |
3153 | fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED; |
3154 | } |
3155 | |
3156 | soclearfastopen(so); |
3157 | |
3158 | return error; |
3159 | } |
3160 | |
3161 | static void |
3162 | flow_divert_set_protosw(struct socket *so) |
3163 | { |
3164 | so->so_flags |= SOF_FLOW_DIVERT; |
3165 | if (SOCK_DOM(so) == PF_INET) { |
3166 | so->so_proto = &g_flow_divert_in_protosw; |
3167 | } |
3168 | #if INET6 |
3169 | else { |
3170 | so->so_proto = (struct protosw *)&g_flow_divert_in6_protosw; |
3171 | } |
3172 | #endif /* INET6 */ |
3173 | } |
3174 | |
3175 | static void |
3176 | flow_divert_set_udp_protosw(struct socket *so) |
3177 | { |
3178 | so->so_flags |= SOF_FLOW_DIVERT; |
3179 | if (SOCK_DOM(so) == PF_INET) { |
3180 | so->so_proto = &g_flow_divert_in_udp_protosw; |
3181 | } |
3182 | #if INET6 |
3183 | else { |
3184 | so->so_proto = (struct protosw *)&g_flow_divert_in6_udp_protosw; |
3185 | } |
3186 | #endif /* INET6 */ |
3187 | } |
3188 | |
3189 | static errno_t |
3190 | flow_divert_attach(struct socket *so, uint32_t flow_id, uint32_t ctl_unit) |
3191 | { |
3192 | int error = 0; |
3193 | struct flow_divert_pcb *fd_cb = NULL; |
3194 | struct ifnet *ifp = NULL; |
3195 | struct inpcb *inp = NULL; |
3196 | struct socket *old_so; |
3197 | mbuf_t recv_data = NULL; |
3198 | |
3199 | socket_unlock(so, 0); |
3200 | |
3201 | FDLOG(LOG_INFO, &nil_pcb, "Attaching socket to flow %u" , flow_id); |
3202 | |
3203 | /* Find the flow divert control block */ |
3204 | lck_rw_lock_shared(&g_flow_divert_group_lck); |
3205 | if (g_flow_divert_groups != NULL && g_active_group_count > 0) { |
3206 | struct flow_divert_group *group = g_flow_divert_groups[ctl_unit]; |
3207 | if (group != NULL) { |
3208 | fd_cb = flow_divert_pcb_lookup(flow_id, group); |
3209 | } |
3210 | } |
3211 | lck_rw_done(&g_flow_divert_group_lck); |
3212 | |
3213 | if (fd_cb == NULL) { |
3214 | error = ENOENT; |
3215 | goto done; |
3216 | } |
3217 | |
3218 | FDLOCK(fd_cb); |
3219 | |
3220 | /* Dis-associate the flow divert control block from its current socket */ |
3221 | old_so = fd_cb->so; |
3222 | |
3223 | inp = sotoinpcb(old_so); |
3224 | |
3225 | VERIFY(inp != NULL); |
3226 | |
3227 | socket_lock(old_so, 0); |
3228 | flow_divert_disconnect_socket(old_so); |
3229 | old_so->so_flags &= ~SOF_FLOW_DIVERT; |
3230 | old_so->so_fd_pcb = NULL; |
3231 | if (SOCK_TYPE(old_so) == SOCK_STREAM) { |
3232 | old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_TCP, SOCK_STREAM); |
3233 | } else if (SOCK_TYPE(old_so) == SOCK_DGRAM) { |
3234 | old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_UDP, SOCK_DGRAM); |
3235 | } |
3236 | fd_cb->so = NULL; |
3237 | /* Save the output interface */ |
3238 | ifp = inp->inp_last_outifp; |
3239 | if (old_so->so_rcv.sb_cc > 0) { |
3240 | error = mbuf_dup(old_so->so_rcv.sb_mb, MBUF_DONTWAIT, &recv_data); |
3241 | sbflush(&old_so->so_rcv); |
3242 | } |
3243 | socket_unlock(old_so, 0); |
3244 | |
3245 | /* Associate the new socket with the flow divert control block */ |
3246 | socket_lock(so, 0); |
3247 | so->so_fd_pcb = fd_cb; |
3248 | inp = sotoinpcb(so); |
3249 | inp->inp_last_outifp = ifp; |
3250 | if (recv_data != NULL) { |
3251 | if (sbappendstream(&so->so_rcv, recv_data)) { |
3252 | sorwakeup(so); |
3253 | } |
3254 | } |
3255 | flow_divert_set_protosw(so); |
3256 | socket_unlock(so, 0); |
3257 | |
3258 | fd_cb->so = so; |
3259 | fd_cb->flags |= FLOW_DIVERT_TRANSFERRED; |
3260 | |
3261 | FDUNLOCK(fd_cb); |
3262 | |
3263 | done: |
3264 | socket_lock(so, 0); |
3265 | |
3266 | if (fd_cb != NULL) { |
3267 | FDRELEASE(fd_cb); /* Release the reference obtained via flow_divert_pcb_lookup */ |
3268 | } |
3269 | |
3270 | return error; |
3271 | } |
3272 | |
3273 | errno_t |
3274 | flow_divert_implicit_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p) |
3275 | { |
3276 | struct flow_divert_pcb *fd_cb = so->so_fd_pcb; |
3277 | struct inpcb *inp; |
3278 | int error = 0; |
3279 | |
3280 | inp = sotoinpcb(so); |
3281 | if (inp == NULL) { |
3282 | return (EINVAL); |
3283 | } |
3284 | |
3285 | if (fd_cb == NULL) { |
3286 | uint32_t fd_ctl_unit = necp_socket_get_flow_divert_control_unit(inp); |
3287 | if (fd_ctl_unit > 0) { |
3288 | error = flow_divert_pcb_init(so, fd_ctl_unit); |
3289 | fd_cb = so->so_fd_pcb; |
3290 | if (error != 0 || fd_cb == NULL) { |
3291 | goto done; |
3292 | } |
3293 | } else { |
3294 | error = ENETDOWN; |
3295 | goto done; |
3296 | } |
3297 | } |
3298 | return flow_divert_data_out(so, flags, data, to, control, p); |
3299 | |
3300 | done: |
3301 | if (data) { |
3302 | mbuf_freem(data); |
3303 | } |
3304 | if (control) { |
3305 | mbuf_free(control); |
3306 | } |
3307 | |
3308 | return error; |
3309 | } |
3310 | |
3311 | errno_t |
3312 | flow_divert_pcb_init(struct socket *so, uint32_t ctl_unit) |
3313 | { |
3314 | errno_t error = 0; |
3315 | struct flow_divert_pcb *fd_cb; |
3316 | |
3317 | if (so->so_flags & SOF_FLOW_DIVERT) { |
3318 | return EALREADY; |
3319 | } |
3320 | |
3321 | fd_cb = flow_divert_pcb_create(so); |
3322 | if (fd_cb != NULL) { |
3323 | error = flow_divert_pcb_insert(fd_cb, ctl_unit); |
3324 | if (error) { |
3325 | FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d" , error); |
3326 | FDRELEASE(fd_cb); |
3327 | } else { |
3328 | fd_cb->control_group_unit = ctl_unit; |
3329 | so->so_fd_pcb = fd_cb; |
3330 | |
3331 | if (SOCK_TYPE(so) == SOCK_STREAM) { |
3332 | flow_divert_set_protosw(so); |
3333 | } else if (SOCK_TYPE(so) == SOCK_DGRAM) { |
3334 | flow_divert_set_udp_protosw(so); |
3335 | } |
3336 | |
3337 | FDLOG0(LOG_INFO, fd_cb, "Created" ); |
3338 | } |
3339 | } else { |
3340 | error = ENOMEM; |
3341 | } |
3342 | |
3343 | return error; |
3344 | } |
3345 | |
3346 | errno_t |
3347 | flow_divert_token_set(struct socket *so, struct sockopt *sopt) |
3348 | { |
3349 | uint32_t ctl_unit = 0; |
3350 | uint32_t key_unit = 0; |
3351 | uint32_t flow_id = 0; |
3352 | int error = 0; |
3353 | int hmac_error = 0; |
3354 | mbuf_t token = NULL; |
3355 | |
3356 | if (so->so_flags & SOF_FLOW_DIVERT) { |
3357 | error = EALREADY; |
3358 | goto done; |
3359 | } |
3360 | |
3361 | if (g_init_result) { |
3362 | FDLOG(LOG_ERR, &nil_pcb, "flow_divert_init failed (%d), cannot use flow divert" , g_init_result); |
3363 | error = ENOPROTOOPT; |
3364 | goto done; |
3365 | } |
3366 | |
3367 | if ((SOCK_TYPE(so) != SOCK_STREAM && SOCK_TYPE(so) != SOCK_DGRAM) || |
3368 | (SOCK_PROTO(so) != IPPROTO_TCP && SOCK_PROTO(so) != IPPROTO_UDP) || |
3369 | (SOCK_DOM(so) != PF_INET |
3370 | #if INET6 |
3371 | && SOCK_DOM(so) != PF_INET6 |
3372 | #endif |
3373 | )) |
3374 | { |
3375 | error = EINVAL; |
3376 | goto done; |
3377 | } else { |
3378 | if (SOCK_TYPE(so) == SOCK_STREAM && SOCK_PROTO(so) == IPPROTO_TCP) { |
3379 | struct tcpcb *tp = sototcpcb(so); |
3380 | if (tp == NULL || tp->t_state != TCPS_CLOSED) { |
3381 | error = EINVAL; |
3382 | goto done; |
3383 | } |
3384 | } |
3385 | } |
3386 | |
3387 | error = soopt_getm(sopt, &token); |
3388 | if (error) { |
3389 | token = NULL; |
3390 | goto done; |
3391 | } |
3392 | |
3393 | error = soopt_mcopyin(sopt, token); |
3394 | if (error) { |
3395 | token = NULL; |
3396 | goto done; |
3397 | } |
3398 | |
3399 | error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(key_unit), (void *)&key_unit, NULL); |
3400 | if (!error) { |
3401 | key_unit = ntohl(key_unit); |
3402 | if (key_unit >= GROUP_COUNT_MAX) { |
3403 | key_unit = 0; |
3404 | } |
3405 | } else if (error != ENOENT) { |
3406 | FDLOG(LOG_ERR, &nil_pcb, "Failed to get the key unit from the token: %d" , error); |
3407 | goto done; |
3408 | } else { |
3409 | key_unit = 0; |
3410 | } |
3411 | |
3412 | error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), (void *)&ctl_unit, NULL); |
3413 | if (error) { |
3414 | FDLOG(LOG_ERR, &nil_pcb, "Failed to get the control socket unit from the token: %d" , error); |
3415 | goto done; |
3416 | } |
3417 | |
3418 | /* A valid kernel control unit is required */ |
3419 | ctl_unit = ntohl(ctl_unit); |
3420 | if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) { |
3421 | FDLOG(LOG_ERR, &nil_pcb, "Got an invalid control socket unit: %u" , ctl_unit); |
3422 | error = EINVAL; |
3423 | goto done; |
3424 | } |
3425 | |
3426 | socket_unlock(so, 0); |
3427 | hmac_error = flow_divert_packet_verify_hmac(token, (key_unit != 0 ? key_unit : ctl_unit)); |
3428 | socket_lock(so, 0); |
3429 | |
3430 | if (hmac_error && hmac_error != ENOENT) { |
3431 | FDLOG(LOG_ERR, &nil_pcb, "HMAC verfication failed: %d" , hmac_error); |
3432 | error = hmac_error; |
3433 | goto done; |
3434 | } |
3435 | |
3436 | error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_FLOW_ID, sizeof(flow_id), (void *)&flow_id, NULL); |
3437 | if (error && error != ENOENT) { |
3438 | FDLOG(LOG_ERR, &nil_pcb, "Failed to get the flow ID from the token: %d" , error); |
3439 | goto done; |
3440 | } |
3441 | |
3442 | if (flow_id == 0) { |
3443 | error = flow_divert_pcb_init(so, ctl_unit); |
3444 | if (error == 0) { |
3445 | struct flow_divert_pcb *fd_cb = so->so_fd_pcb; |
3446 | int log_level = LOG_NOTICE; |
3447 | |
3448 | error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_LOG_LEVEL, |
3449 | sizeof(log_level), &log_level, NULL); |
3450 | if (error == 0) { |
3451 | fd_cb->log_level = log_level; |
3452 | } |
3453 | error = 0; |
3454 | |
3455 | fd_cb->connect_token = token; |
3456 | token = NULL; |
3457 | } |
3458 | } else { |
3459 | error = flow_divert_attach(so, flow_id, ctl_unit); |
3460 | } |
3461 | |
3462 | if (hmac_error == 0) { |
3463 | struct flow_divert_pcb *fd_cb = so->so_fd_pcb; |
3464 | if (fd_cb != NULL) { |
3465 | fd_cb->flags |= FLOW_DIVERT_HAS_HMAC; |
3466 | } |
3467 | } |
3468 | |
3469 | done: |
3470 | if (token != NULL) { |
3471 | mbuf_freem(token); |
3472 | } |
3473 | |
3474 | return error; |
3475 | } |
3476 | |
3477 | errno_t |
3478 | flow_divert_token_get(struct socket *so, struct sockopt *sopt) |
3479 | { |
3480 | uint32_t ctl_unit; |
3481 | int error = 0; |
3482 | uint8_t hmac[SHA_DIGEST_LENGTH]; |
3483 | struct flow_divert_pcb *fd_cb = so->so_fd_pcb; |
3484 | mbuf_t token = NULL; |
3485 | struct flow_divert_group *control_group = NULL; |
3486 | |
3487 | if (!(so->so_flags & SOF_FLOW_DIVERT)) { |
3488 | error = EINVAL; |
3489 | goto done; |
3490 | } |
3491 | |
3492 | VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); |
3493 | |
3494 | if (fd_cb->group == NULL) { |
3495 | error = EINVAL; |
3496 | goto done; |
3497 | } |
3498 | |
3499 | error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &token); |
3500 | if (error) { |
3501 | FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d" , error); |
3502 | goto done; |
3503 | } |
3504 | |
3505 | ctl_unit = htonl(fd_cb->group->ctl_unit); |
3506 | |
3507 | error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit); |
3508 | if (error) { |
3509 | goto done; |
3510 | } |
3511 | |
3512 | error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb->hash); |
3513 | if (error) { |
3514 | goto done; |
3515 | } |
3516 | |
3517 | if (fd_cb->app_data != NULL) { |
3518 | error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_APP_DATA, fd_cb->app_data_length, fd_cb->app_data); |
3519 | if (error) { |
3520 | goto done; |
3521 | } |
3522 | } |
3523 | |
3524 | socket_unlock(so, 0); |
3525 | lck_rw_lock_shared(&g_flow_divert_group_lck); |
3526 | |
3527 | if (g_flow_divert_groups != NULL && g_active_group_count > 0 && |
3528 | fd_cb->control_group_unit > 0 && fd_cb->control_group_unit < GROUP_COUNT_MAX) |
3529 | { |
3530 | control_group = g_flow_divert_groups[fd_cb->control_group_unit]; |
3531 | } |
3532 | |
3533 | if (control_group != NULL) { |
3534 | lck_rw_lock_shared(&control_group->lck); |
3535 | ctl_unit = htonl(control_group->ctl_unit); |
3536 | error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(ctl_unit), &ctl_unit); |
3537 | if (!error) { |
3538 | error = flow_divert_packet_compute_hmac(token, control_group, hmac); |
3539 | } |
3540 | lck_rw_done(&control_group->lck); |
3541 | } else { |
3542 | error = ENOPROTOOPT; |
3543 | } |
3544 | |
3545 | lck_rw_done(&g_flow_divert_group_lck); |
3546 | socket_lock(so, 0); |
3547 | |
3548 | if (error) { |
3549 | goto done; |
3550 | } |
3551 | |
3552 | error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_HMAC, sizeof(hmac), hmac); |
3553 | if (error) { |
3554 | goto done; |
3555 | } |
3556 | |
3557 | if (sopt->sopt_val == USER_ADDR_NULL) { |
3558 | /* If the caller passed NULL to getsockopt, just set the size of the token and return */ |
3559 | sopt->sopt_valsize = mbuf_pkthdr_len(token); |
3560 | goto done; |
3561 | } |
3562 | |
3563 | error = soopt_mcopyout(sopt, token); |
3564 | if (error) { |
3565 | token = NULL; /* For some reason, soopt_mcopyout() frees the mbuf if it fails */ |
3566 | goto done; |
3567 | } |
3568 | |
3569 | done: |
3570 | if (token != NULL) { |
3571 | mbuf_freem(token); |
3572 | } |
3573 | |
3574 | return error; |
3575 | } |
3576 | |
3577 | static errno_t |
3578 | flow_divert_kctl_connect(kern_ctl_ref kctlref __unused, struct sockaddr_ctl *sac, void **unitinfo) |
3579 | { |
3580 | struct flow_divert_group *new_group = NULL; |
3581 | int error = 0; |
3582 | |
3583 | if (sac->sc_unit >= GROUP_COUNT_MAX) { |
3584 | error = EINVAL; |
3585 | goto done; |
3586 | } |
3587 | |
3588 | *unitinfo = NULL; |
3589 | |
3590 | MALLOC_ZONE(new_group, struct flow_divert_group *, sizeof(*new_group), M_FLOW_DIVERT_GROUP, M_WAITOK); |
3591 | if (new_group == NULL) { |
3592 | error = ENOBUFS; |
3593 | goto done; |
3594 | } |
3595 | |
3596 | memset(new_group, 0, sizeof(*new_group)); |
3597 | |
3598 | lck_rw_init(&new_group->lck, flow_divert_mtx_grp, flow_divert_mtx_attr); |
3599 | RB_INIT(&new_group->pcb_tree); |
3600 | new_group->ctl_unit = sac->sc_unit; |
3601 | MBUFQ_INIT(&new_group->send_queue); |
3602 | new_group->signing_id_trie.root = NULL_TRIE_IDX; |
3603 | |
3604 | lck_rw_lock_exclusive(&g_flow_divert_group_lck); |
3605 | |
3606 | if (g_flow_divert_groups == NULL) { |
3607 | MALLOC(g_flow_divert_groups, |
3608 | struct flow_divert_group **, |
3609 | GROUP_COUNT_MAX * sizeof(struct flow_divert_group *), |
3610 | M_TEMP, |
3611 | M_WAITOK | M_ZERO); |
3612 | } |
3613 | |
3614 | if (g_flow_divert_groups == NULL) { |
3615 | error = ENOBUFS; |
3616 | } else if (g_flow_divert_groups[sac->sc_unit] != NULL) { |
3617 | error = EALREADY; |
3618 | } else { |
3619 | g_flow_divert_groups[sac->sc_unit] = new_group; |
3620 | g_active_group_count++; |
3621 | } |
3622 | |
3623 | lck_rw_done(&g_flow_divert_group_lck); |
3624 | |
3625 | *unitinfo = new_group; |
3626 | |
3627 | done: |
3628 | if (error != 0 && new_group != NULL) { |
3629 | FREE_ZONE(new_group, sizeof(*new_group), M_FLOW_DIVERT_GROUP); |
3630 | } |
3631 | return error; |
3632 | } |
3633 | |
3634 | static errno_t |
3635 | flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused, uint32_t unit, void *unitinfo) |
3636 | { |
3637 | struct flow_divert_group *group = NULL; |
3638 | errno_t error = 0; |
3639 | |
3640 | if (unit >= GROUP_COUNT_MAX) { |
3641 | return EINVAL; |
3642 | } |
3643 | |
3644 | FDLOG(LOG_INFO, &nil_pcb, "disconnecting group %d" , unit); |
3645 | |
3646 | lck_rw_lock_exclusive(&g_flow_divert_group_lck); |
3647 | |
3648 | if (g_flow_divert_groups == NULL || g_active_group_count == 0) { |
3649 | panic("flow divert group %u is disconnecting, but no groups are active (groups = %p, active count = %u" , unit, |
3650 | g_flow_divert_groups, g_active_group_count); |
3651 | } |
3652 | |
3653 | group = g_flow_divert_groups[unit]; |
3654 | |
3655 | if (group != (struct flow_divert_group *)unitinfo) { |
3656 | panic("group with unit %d (%p) != unit info (%p)" , unit, group, unitinfo); |
3657 | } |
3658 | |
3659 | if (group != NULL) { |
3660 | flow_divert_close_all(group); |
3661 | if (group->token_key != NULL) { |
3662 | memset(group->token_key, 0, group->token_key_size); |
3663 | FREE(group->token_key, M_TEMP); |
3664 | group->token_key = NULL; |
3665 | group->token_key_size = 0; |
3666 | } |
3667 | |
3668 | /* Re-set the current trie */ |
3669 | if (group->signing_id_trie.memory != NULL) { |
3670 | FREE(group->signing_id_trie.memory, M_TEMP); |
3671 | } |
3672 | memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie)); |
3673 | group->signing_id_trie.root = NULL_TRIE_IDX; |
3674 | |
3675 | FREE_ZONE(group, sizeof(*group), M_FLOW_DIVERT_GROUP); |
3676 | g_flow_divert_groups[unit] = NULL; |
3677 | g_active_group_count--; |
3678 | } else { |
3679 | error = EINVAL; |
3680 | } |
3681 | |
3682 | if (g_active_group_count == 0) { |
3683 | FREE(g_flow_divert_groups, M_TEMP); |
3684 | g_flow_divert_groups = NULL; |
3685 | } |
3686 | |
3687 | lck_rw_done(&g_flow_divert_group_lck); |
3688 | |
3689 | return error; |
3690 | } |
3691 | |
3692 | static errno_t |
3693 | flow_divert_kctl_send(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, mbuf_t m, int flags __unused) |
3694 | { |
3695 | return flow_divert_input(m, (struct flow_divert_group *)unitinfo); |
3696 | } |
3697 | |
3698 | static void |
3699 | flow_divert_kctl_rcvd(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, int flags __unused) |
3700 | { |
3701 | struct flow_divert_group *group = (struct flow_divert_group *)unitinfo; |
3702 | |
3703 | if (!OSTestAndClear(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits)) { |
3704 | struct flow_divert_pcb *fd_cb; |
3705 | SLIST_HEAD(, flow_divert_pcb) tmp_list; |
3706 | |
3707 | lck_rw_lock_shared(&g_flow_divert_group_lck); |
3708 | lck_rw_lock_exclusive(&group->lck); |
3709 | |
3710 | while (!MBUFQ_EMPTY(&group->send_queue)) { |
3711 | mbuf_t next_packet; |
3712 | FDLOG0(LOG_DEBUG, &nil_pcb, "trying ctl_enqueuembuf again" ); |
3713 | next_packet = MBUFQ_FIRST(&group->send_queue); |
3714 | int error = ctl_enqueuembuf(g_flow_divert_kctl_ref, group->ctl_unit, next_packet, CTL_DATA_EOR); |
3715 | if (error) { |
3716 | FDLOG(LOG_DEBUG, &nil_pcb, "ctl_enqueuembuf returned an error: %d" , error); |
3717 | OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits); |
3718 | lck_rw_done(&group->lck); |
3719 | lck_rw_done(&g_flow_divert_group_lck); |
3720 | return; |
3721 | } |
3722 | MBUFQ_DEQUEUE(&group->send_queue, next_packet); |
3723 | } |
3724 | |
3725 | SLIST_INIT(&tmp_list); |
3726 | |
3727 | RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) { |
3728 | FDRETAIN(fd_cb); |
3729 | SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry); |
3730 | } |
3731 | |
3732 | lck_rw_done(&group->lck); |
3733 | |
3734 | SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) { |
3735 | FDLOCK(fd_cb); |
3736 | if (fd_cb->so != NULL) { |
3737 | socket_lock(fd_cb->so, 0); |
3738 | if (fd_cb->group != NULL) { |
3739 | flow_divert_send_buffered_data(fd_cb, FALSE); |
3740 | } |
3741 | socket_unlock(fd_cb->so, 0); |
3742 | } |
3743 | FDUNLOCK(fd_cb); |
3744 | FDRELEASE(fd_cb); |
3745 | } |
3746 | |
3747 | lck_rw_done(&g_flow_divert_group_lck); |
3748 | } |
3749 | } |
3750 | |
3751 | static int |
3752 | flow_divert_kctl_init(void) |
3753 | { |
3754 | struct kern_ctl_reg ctl_reg; |
3755 | int result; |
3756 | |
3757 | memset(&ctl_reg, 0, sizeof(ctl_reg)); |
3758 | |
3759 | strlcpy(ctl_reg.ctl_name, FLOW_DIVERT_CONTROL_NAME, sizeof(ctl_reg.ctl_name)); |
3760 | ctl_reg.ctl_name[sizeof(ctl_reg.ctl_name)-1] = '\0'; |
3761 | ctl_reg.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED; |
3762 | ctl_reg.ctl_sendsize = FD_CTL_SENDBUFF_SIZE; |
3763 | ctl_reg.ctl_recvsize = FD_CTL_RCVBUFF_SIZE; |
3764 | |
3765 | ctl_reg.ctl_connect = flow_divert_kctl_connect; |
3766 | ctl_reg.ctl_disconnect = flow_divert_kctl_disconnect; |
3767 | ctl_reg.ctl_send = flow_divert_kctl_send; |
3768 | ctl_reg.ctl_rcvd = flow_divert_kctl_rcvd; |
3769 | |
3770 | result = ctl_register(&ctl_reg, &g_flow_divert_kctl_ref); |
3771 | |
3772 | if (result) { |
3773 | FDLOG(LOG_ERR, &nil_pcb, "flow_divert_kctl_init - ctl_register failed: %d\n" , result); |
3774 | return result; |
3775 | } |
3776 | |
3777 | return 0; |
3778 | } |
3779 | |
3780 | void |
3781 | flow_divert_init(void) |
3782 | { |
3783 | memset(&nil_pcb, 0, sizeof(nil_pcb)); |
3784 | nil_pcb.log_level = LOG_NOTICE; |
3785 | |
3786 | g_tcp_protosw = pffindproto(AF_INET, IPPROTO_TCP, SOCK_STREAM); |
3787 | |
3788 | VERIFY(g_tcp_protosw != NULL); |
3789 | |
3790 | memcpy(&g_flow_divert_in_protosw, g_tcp_protosw, sizeof(g_flow_divert_in_protosw)); |
3791 | memcpy(&g_flow_divert_in_usrreqs, g_tcp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_usrreqs)); |
3792 | |
3793 | g_flow_divert_in_usrreqs.pru_connect = flow_divert_connect_out; |
3794 | g_flow_divert_in_usrreqs.pru_connectx = flow_divert_connectx_out; |
3795 | g_flow_divert_in_usrreqs.pru_control = flow_divert_in_control; |
3796 | g_flow_divert_in_usrreqs.pru_disconnect = flow_divert_close; |
3797 | g_flow_divert_in_usrreqs.pru_disconnectx = flow_divert_disconnectx; |
3798 | g_flow_divert_in_usrreqs.pru_peeraddr = flow_divert_getpeername; |
3799 | g_flow_divert_in_usrreqs.pru_rcvd = flow_divert_rcvd; |
3800 | g_flow_divert_in_usrreqs.pru_send = flow_divert_data_out; |
3801 | g_flow_divert_in_usrreqs.pru_shutdown = flow_divert_shutdown; |
3802 | g_flow_divert_in_usrreqs.pru_sockaddr = flow_divert_getsockaddr; |
3803 | g_flow_divert_in_usrreqs.pru_preconnect = flow_divert_preconnect; |
3804 | |
3805 | g_flow_divert_in_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs; |
3806 | g_flow_divert_in_protosw.pr_ctloutput = flow_divert_ctloutput; |
3807 | |
3808 | /* |
3809 | * Socket filters shouldn't attach/detach to/from this protosw |
3810 | * since pr_protosw is to be used instead, which points to the |
3811 | * real protocol; if they do, it is a bug and we should panic. |
3812 | */ |
3813 | g_flow_divert_in_protosw.pr_filter_head.tqh_first = |
3814 | (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef; |
3815 | g_flow_divert_in_protosw.pr_filter_head.tqh_last = |
3816 | (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef; |
3817 | |
3818 | /* UDP */ |
3819 | g_udp_protosw = pffindproto(AF_INET, IPPROTO_UDP, SOCK_DGRAM); |
3820 | VERIFY(g_udp_protosw != NULL); |
3821 | |
3822 | memcpy(&g_flow_divert_in_udp_protosw, g_udp_protosw, sizeof(g_flow_divert_in_udp_protosw)); |
3823 | memcpy(&g_flow_divert_in_udp_usrreqs, g_udp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_udp_usrreqs)); |
3824 | |
3825 | g_flow_divert_in_udp_usrreqs.pru_connect = flow_divert_connect_out; |
3826 | g_flow_divert_in_udp_usrreqs.pru_connectx = flow_divert_connectx_out; |
3827 | g_flow_divert_in_udp_usrreqs.pru_control = flow_divert_in_control; |
3828 | g_flow_divert_in_udp_usrreqs.pru_disconnect = flow_divert_close; |
3829 | g_flow_divert_in_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx; |
3830 | g_flow_divert_in_udp_usrreqs.pru_peeraddr = flow_divert_getpeername; |
3831 | g_flow_divert_in_udp_usrreqs.pru_rcvd = flow_divert_rcvd; |
3832 | g_flow_divert_in_udp_usrreqs.pru_send = flow_divert_data_out; |
3833 | g_flow_divert_in_udp_usrreqs.pru_shutdown = flow_divert_shutdown; |
3834 | g_flow_divert_in_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr; |
3835 | g_flow_divert_in_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp; |
3836 | g_flow_divert_in_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp; |
3837 | g_flow_divert_in_udp_usrreqs.pru_preconnect = flow_divert_preconnect; |
3838 | |
3839 | g_flow_divert_in_udp_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs; |
3840 | g_flow_divert_in_udp_protosw.pr_ctloutput = flow_divert_ctloutput; |
3841 | |
3842 | /* |
3843 | * Socket filters shouldn't attach/detach to/from this protosw |
3844 | * since pr_protosw is to be used instead, which points to the |
3845 | * real protocol; if they do, it is a bug and we should panic. |
3846 | */ |
3847 | g_flow_divert_in_udp_protosw.pr_filter_head.tqh_first = |
3848 | (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef; |
3849 | g_flow_divert_in_udp_protosw.pr_filter_head.tqh_last = |
3850 | (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef; |
3851 | |
3852 | #if INET6 |
3853 | g_tcp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_TCP, SOCK_STREAM); |
3854 | |
3855 | VERIFY(g_tcp6_protosw != NULL); |
3856 | |
3857 | memcpy(&g_flow_divert_in6_protosw, g_tcp6_protosw, sizeof(g_flow_divert_in6_protosw)); |
3858 | memcpy(&g_flow_divert_in6_usrreqs, g_tcp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_usrreqs)); |
3859 | |
3860 | g_flow_divert_in6_usrreqs.pru_connect = flow_divert_connect_out; |
3861 | g_flow_divert_in6_usrreqs.pru_connectx = flow_divert_connectx6_out; |
3862 | g_flow_divert_in6_usrreqs.pru_control = flow_divert_in6_control; |
3863 | g_flow_divert_in6_usrreqs.pru_disconnect = flow_divert_close; |
3864 | g_flow_divert_in6_usrreqs.pru_disconnectx = flow_divert_disconnectx; |
3865 | g_flow_divert_in6_usrreqs.pru_peeraddr = flow_divert_getpeername; |
3866 | g_flow_divert_in6_usrreqs.pru_rcvd = flow_divert_rcvd; |
3867 | g_flow_divert_in6_usrreqs.pru_send = flow_divert_data_out; |
3868 | g_flow_divert_in6_usrreqs.pru_shutdown = flow_divert_shutdown; |
3869 | g_flow_divert_in6_usrreqs.pru_sockaddr = flow_divert_getsockaddr; |
3870 | g_flow_divert_in6_usrreqs.pru_preconnect = flow_divert_preconnect; |
3871 | |
3872 | g_flow_divert_in6_protosw.pr_usrreqs = &g_flow_divert_in6_usrreqs; |
3873 | g_flow_divert_in6_protosw.pr_ctloutput = flow_divert_ctloutput; |
3874 | /* |
3875 | * Socket filters shouldn't attach/detach to/from this protosw |
3876 | * since pr_protosw is to be used instead, which points to the |
3877 | * real protocol; if they do, it is a bug and we should panic. |
3878 | */ |
3879 | g_flow_divert_in6_protosw.pr_filter_head.tqh_first = |
3880 | (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef; |
3881 | g_flow_divert_in6_protosw.pr_filter_head.tqh_last = |
3882 | (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef; |
3883 | |
3884 | /* UDP6 */ |
3885 | g_udp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_UDP, SOCK_DGRAM); |
3886 | |
3887 | VERIFY(g_udp6_protosw != NULL); |
3888 | |
3889 | memcpy(&g_flow_divert_in6_udp_protosw, g_udp6_protosw, sizeof(g_flow_divert_in6_udp_protosw)); |
3890 | memcpy(&g_flow_divert_in6_udp_usrreqs, g_udp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_udp_usrreqs)); |
3891 | |
3892 | g_flow_divert_in6_udp_usrreqs.pru_connect = flow_divert_connect_out; |
3893 | g_flow_divert_in6_udp_usrreqs.pru_connectx = flow_divert_connectx6_out; |
3894 | g_flow_divert_in6_udp_usrreqs.pru_control = flow_divert_in6_control; |
3895 | g_flow_divert_in6_udp_usrreqs.pru_disconnect = flow_divert_close; |
3896 | g_flow_divert_in6_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx; |
3897 | g_flow_divert_in6_udp_usrreqs.pru_peeraddr = flow_divert_getpeername; |
3898 | g_flow_divert_in6_udp_usrreqs.pru_rcvd = flow_divert_rcvd; |
3899 | g_flow_divert_in6_udp_usrreqs.pru_send = flow_divert_data_out; |
3900 | g_flow_divert_in6_udp_usrreqs.pru_shutdown = flow_divert_shutdown; |
3901 | g_flow_divert_in6_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr; |
3902 | g_flow_divert_in6_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp; |
3903 | g_flow_divert_in6_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp; |
3904 | g_flow_divert_in6_udp_usrreqs.pru_preconnect = flow_divert_preconnect; |
3905 | |
3906 | g_flow_divert_in6_udp_protosw.pr_usrreqs = &g_flow_divert_in6_udp_usrreqs; |
3907 | g_flow_divert_in6_udp_protosw.pr_ctloutput = flow_divert_ctloutput; |
3908 | /* |
3909 | * Socket filters shouldn't attach/detach to/from this protosw |
3910 | * since pr_protosw is to be used instead, which points to the |
3911 | * real protocol; if they do, it is a bug and we should panic. |
3912 | */ |
3913 | g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_first = |
3914 | (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef; |
3915 | g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_last = |
3916 | (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef; |
3917 | #endif /* INET6 */ |
3918 | |
3919 | flow_divert_grp_attr = lck_grp_attr_alloc_init(); |
3920 | if (flow_divert_grp_attr == NULL) { |
3921 | FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_attr_alloc_init failed" ); |
3922 | g_init_result = ENOMEM; |
3923 | goto done; |
3924 | } |
3925 | |
3926 | flow_divert_mtx_grp = lck_grp_alloc_init(FLOW_DIVERT_CONTROL_NAME, flow_divert_grp_attr); |
3927 | if (flow_divert_mtx_grp == NULL) { |
3928 | FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_alloc_init failed" ); |
3929 | g_init_result = ENOMEM; |
3930 | goto done; |
3931 | } |
3932 | |
3933 | flow_divert_mtx_attr = lck_attr_alloc_init(); |
3934 | if (flow_divert_mtx_attr == NULL) { |
3935 | FDLOG0(LOG_ERR, &nil_pcb, "lck_attr_alloc_init failed" ); |
3936 | g_init_result = ENOMEM; |
3937 | goto done; |
3938 | } |
3939 | |
3940 | g_init_result = flow_divert_kctl_init(); |
3941 | if (g_init_result) { |
3942 | goto done; |
3943 | } |
3944 | |
3945 | lck_rw_init(&g_flow_divert_group_lck, flow_divert_mtx_grp, flow_divert_mtx_attr); |
3946 | |
3947 | done: |
3948 | if (g_init_result != 0) { |
3949 | if (flow_divert_mtx_attr != NULL) { |
3950 | lck_attr_free(flow_divert_mtx_attr); |
3951 | flow_divert_mtx_attr = NULL; |
3952 | } |
3953 | if (flow_divert_mtx_grp != NULL) { |
3954 | lck_grp_free(flow_divert_mtx_grp); |
3955 | flow_divert_mtx_grp = NULL; |
3956 | } |
3957 | if (flow_divert_grp_attr != NULL) { |
3958 | lck_grp_attr_free(flow_divert_grp_attr); |
3959 | flow_divert_grp_attr = NULL; |
3960 | } |
3961 | |
3962 | if (g_flow_divert_kctl_ref != NULL) { |
3963 | ctl_deregister(g_flow_divert_kctl_ref); |
3964 | g_flow_divert_kctl_ref = NULL; |
3965 | } |
3966 | } |
3967 | } |
3968 | |