<
1/*
2 * Copyright (c) 2013-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23
24/*
25 * THEORY OF OPERATION
26 *
27 * The socket content filter subsystem provides a way for user space agents to
28 * make filtering decisions based on the content of the data being sent and
29 * received by INET/INET6 sockets.
30 *
31 * A content filter user space agents gets a copy of the data and the data is
32 * also kept in kernel buffer until the user space agents makes a pass or drop
33 * decision. This unidirectional flow of content avoids unnecessary data copies
34 * back to the kernel.
35 *
36 * A user space filter agent opens a kernel control socket with the name
37 * CONTENT_FILTER_CONTROL_NAME to attach to the socket content filter subsystem.
38 * When connected, a "struct content_filter" is created and set as the
39 * "unitinfo" of the corresponding kernel control socket instance.
40 *
41 * The socket content filter subsystem exchanges messages with the user space
42 * filter agent until an ultimate pass or drop decision is made by the
43 * user space filter agent.
44 *
45 * It should be noted that messages about many INET/INET6 sockets can be multiplexed
46 * over a single kernel control socket.
47 *
48 * Notes:
49 * - The current implementation supports all INET/INET6 sockets (i.e. TCP,
50 * UDP, ICMP, etc).
51 * - The current implementation supports up to two simultaneous content filters
52 * for iOS devices and eight simultaneous content filters for OSX.
53 *
54 *
55 * NECP FILTER CONTROL UNIT
56 *
57 * A user space filter agent uses the Network Extension Control Policy (NECP)
58 * database to specify which INET/INET6 sockets need to be filtered. The NECP
59 * criteria may be based on a variety of properties like user ID or proc UUID.
60 *
61 * The NECP "filter control unit" is used by the socket content filter subsystem
62 * to deliver the relevant INET/INET6 content information to the appropriate
63 * user space filter agent via its kernel control socket instance.
64 * This works as follows:
65 *
66 * 1) The user space filter agent specifies an NECP filter control unit when
67 * in adds its filtering rules to the NECP database.
68 *
69 * 2) The user space filter agent also sets its NECP filter control unit on the
70 * content filter kernel control socket via the socket option
71 * CFIL_OPT_NECP_CONTROL_UNIT.
72 *
73 * 3) The NECP database is consulted to find out if a given INET/INET6 socket
74 * needs to be subjected to content filtering and returns the corresponding
75 * NECP filter control unit -- the NECP filter control unit is actually
76 * stored in the INET/INET6 socket structure so the NECP lookup is really simple.
77 *
78 * 4) The NECP filter control unit is then used to find the corresponding
79 * kernel control socket instance.
80 *
81 * Note: NECP currently supports a single filter control unit per INET/INET6 socket
82 * but this restriction may be soon lifted.
83 *
84 *
85 * THE MESSAGING PROTOCOL
86 *
87 * The socket content filter subsystem and a user space filter agent
88 * communicate over the kernel control socket via an asynchronous
89 * messaging protocol (this is not a request-response protocol).
90 * The socket content filter subsystem sends event messages to the user
91 * space filter agent about the INET/INET6 sockets it is interested to filter.
92 * The user space filter agent sends action messages to either allow
93 * data to pass or to disallow the data flow (and drop the connection).
94 *
95 * All messages over a content filter kernel control socket share the same
96 * common header of type "struct cfil_msg_hdr". The message type tells if
97 * it's a event message "CFM_TYPE_EVENT" or a action message "CFM_TYPE_ACTION".
98 * The message header field "cfm_sock_id" identifies a given INET/INET6 flow.
99 * For TCP, flows are per-socket. For UDP and other datagrame protocols, there
100 * could be multiple flows per socket.
101 *
102 * Note the message header length field may be padded for alignment and can
103 * be larger than the actual content of the message.
104 * The field "cfm_op" describe the kind of event or action.
105 *
106 * Here are the kinds of content filter events:
107 * - CFM_OP_SOCKET_ATTACHED: a new INET/INET6 socket is being filtered
108 * - CFM_OP_SOCKET_CLOSED: A INET/INET6 socket is closed
109 * - CFM_OP_DATA_OUT: A span of data is being sent on a INET/INET6 socket
110 * - CFM_OP_DATA_IN: A span of data is being or received on a INET/INET6 socket
111 *
112 *
113 * EVENT MESSAGES
114 *
115 * The CFM_OP_DATA_OUT and CFM_OP_DATA_IN event messages contains a span of
116 * data that is being sent or received. The position of this span of data
117 * in the data flow is described by a set of start and end offsets. These
118 * are absolute 64 bits offsets. The first byte sent (or received) starts
119 * at offset 0 and ends at offset 1. The length of the content data
120 * is given by the difference between the end offset and the start offset.
121 *
122 * After a CFM_OP_SOCKET_ATTACHED is delivered, CFM_OP_DATA_OUT and
123 * CFM_OP_DATA_OUT events are not delivered until a CFM_OP_DATA_UPDATE
124 * action message is sent by the user space filter agent.
125 *
126 * Note: absolute 64 bits offsets should be large enough for the foreseeable
127 * future. A 64-bits counter will wrap after 468 years at 10 Gbit/sec:
128 * 2E64 / ((10E9 / 8) * 60 * 60 * 24 * 365.25) = 467.63
129 *
130 * They are two kinds of primary content filter actions:
131 * - CFM_OP_DATA_UPDATE: to update pass or peek offsets for each direction.
132 * - CFM_OP_DROP: to shutdown socket and disallow further data flow
133 *
134 * There is also an action to mark a given client flow as already filtered
135 * at a higher level, CFM_OP_BLESS_CLIENT.
136 *
137 *
138 * ACTION MESSAGES
139 *
140 * The CFM_OP_DATA_UPDATE action messages let the user space filter
141 * agent allow data to flow up to the specified pass offset -- there
142 * is a pass offset for outgoing data and a pass offset for incoming data.
143 * When a new INET/INET6 socket is attached to the content filter and a flow is
144 * created, each pass offset is initially set to 0 so no data is allowed to pass by
145 * default. When the pass offset is set to CFM_MAX_OFFSET via a CFM_OP_DATA_UPDATE
146 * then the data flow becomes unrestricted.
147 *
148 * Note that pass offsets can only be incremented. A CFM_OP_DATA_UPDATE message
149 * with a pass offset smaller than the pass offset of a previous
150 * CFM_OP_DATA_UPDATE message is silently ignored.
151 *
152 * A user space filter agent also uses CFM_OP_DATA_UPDATE action messages
153 * to tell the kernel how much data it wants to see by using the peek offsets.
154 * Just like pass offsets, there is a peek offset for each direction.
155 * When a new INET/INET6 flow is created, each peek offset is initially set to 0
156 * so no CFM_OP_DATA_OUT and CFM_OP_DATA_IN event messages are dispatched by default
157 * until a CFM_OP_DATA_UPDATE action message with a greater than 0 peek offset is sent
158 * by the user space filter agent. When the peek offset is set to CFM_MAX_OFFSET via
159 * a CFM_OP_DATA_UPDATE then the flow of update data events becomes unrestricted.
160 *
161 * Note that peek offsets cannot be smaller than the corresponding pass offset.
162 * Also a peek offsets cannot be smaller than the corresponding end offset
163 * of the last CFM_OP_DATA_OUT/CFM_OP_DATA_IN message dispatched. Trying
164 * to set a too small peek value is silently ignored.
165 *
166 *
167 * PER FLOW "struct cfil_info"
168 *
169 * As soon as a INET/INET6 socket gets attached to a content filter, a
170 * "struct cfil_info" is created to hold the content filtering state for this
171 * socket. For UDP and other datagram protocols, as soon as traffic is seen for
172 * each new flow identified by its 4-tuple of source address/port and destination
173 * address/port, a "struct cfil_info" is created. Each datagram socket may
174 * have multiple flows maintained in a hash table of "struct cfil_info" entries.
175 *
176 * The content filtering state is made of the following information
177 * for each direction:
178 * - The current pass offset;
179 * - The first and last offsets of the data pending, waiting for a filtering
180 * decision;
181 * - The inject queue for data that passed the filters and that needs
182 * to be re-injected;
183 * - A content filter specific state in a set of "struct cfil_entry"
184 *
185 *
186 * CONTENT FILTER STATE "struct cfil_entry"
187 *
188 * The "struct cfil_entry" maintains the information most relevant to the
189 * message handling over a kernel control socket with a user space filter agent.
190 *
191 * The "struct cfil_entry" holds the NECP filter control unit that corresponds
192 * to the kernel control socket unit it corresponds to and also has a pointer
193 * to the corresponding "struct content_filter".
194 *
195 * For each direction, "struct cfil_entry" maintains the following information:
196 * - The pass offset
197 * - The peek offset
198 * - The offset of the last data peeked at by the filter
199 * - A queue of data that's waiting to be delivered to the user space filter
200 * agent on the kernel control socket
201 * - A queue of data for which event messages have been sent on the kernel
202 * control socket and are pending for a filtering decision.
203 *
204 *
205 * CONTENT FILTER QUEUES
206 *
207 * Data that is being filtered is steered away from the INET/INET6 socket buffer
208 * and instead will sit in one of three content filter queues until the data
209 * can be re-injected into the INET/INET6 socket buffer.
210 *
211 * A content filter queue is represented by "struct cfil_queue" that contains
212 * a list of mbufs and the start and end offset of the data span of
213 * the list of mbufs.
214 *
215 * The data moves into the three content filter queues according to this
216 * sequence:
217 * a) The "cfe_ctl_q" of "struct cfil_entry"
218 * b) The "cfe_pending_q" of "struct cfil_entry"
219 * c) The "cfi_inject_q" of "struct cfil_info"
220 *
221 * Note: The sequence (a),(b) may be repeated several times if there is more
222 * than one content filter attached to the INET/INET6 socket.
223 *
224 * The "cfe_ctl_q" queue holds data than cannot be delivered to the
225 * kernel conntrol socket for two reasons:
226 * - The peek offset is less that the end offset of the mbuf data
227 * - The kernel control socket is flow controlled
228 *
229 * The "cfe_pending_q" queue holds data for which CFM_OP_DATA_OUT or
230 * CFM_OP_DATA_IN have been successfully dispatched to the kernel control
231 * socket and are waiting for a pass action message fromn the user space
232 * filter agent. An mbuf length must be fully allowed to pass to be removed
233 * from the cfe_pending_q.
234 *
235 * The "cfi_inject_q" queue holds data that has been fully allowed to pass
236 * by the user space filter agent and that needs to be re-injected into the
237 * INET/INET6 socket.
238 *
239 *
240 * IMPACT ON FLOW CONTROL
241 *
242 * An essential aspect of the content filer subsystem is to minimize the
243 * impact on flow control of the INET/INET6 sockets being filtered.
244 *
245 * The processing overhead of the content filtering may have an effect on
246 * flow control by adding noticeable delays and cannot be eliminated --
247 * care must be taken by the user space filter agent to minimize the
248 * processing delays.
249 *
250 * The amount of data being filtered is kept in buffers while waiting for
251 * a decision by the user space filter agent. This amount of data pending
252 * needs to be subtracted from the amount of data available in the
253 * corresponding INET/INET6 socket buffer. This is done by modifying
254 * sbspace() and tcp_sbspace() to account for amount of data pending
255 * in the content filter.
256 *
257 *
258 * LOCKING STRATEGY
259 *
260 * The global state of content filter subsystem is protected by a single
261 * read-write lock "cfil_lck_rw". The data flow can be done with the
262 * cfil read-write lock held as shared so it can be re-entered from multiple
263 * threads.
264 *
265 * The per INET/INET6 socket content filterstate -- "struct cfil_info" -- is
266 * protected by the socket lock.
267 *
268 * A INET/INET6 socket lock cannot be taken while the cfil read-write lock
269 * is held. That's why we have some sequences where we drop the cfil read-write
270 * lock before taking the INET/INET6 lock.
271 *
272 * It is also important to lock the INET/INET6 socket buffer while the content
273 * filter is modifying the amount of pending data. Otherwise the calculations
274 * in sbspace() and tcp_sbspace() could be wrong.
275 *
276 * The "cfil_lck_rw" protects "struct content_filter" and also the fields
277 * "cfe_link" and "cfe_filter" of "struct cfil_entry".
278 *
279 * Actually "cfe_link" and "cfe_filter" are protected by both by
280 * "cfil_lck_rw" and the socket lock: they may be modified only when
281 * "cfil_lck_rw" is exclusive and the socket is locked.
282 *
283 * To read the other fields of "struct content_filter" we have to take
284 * "cfil_lck_rw" in shared mode.
285 *
286 * DATAGRAM SPECIFICS:
287 *
288 * The socket content filter supports all INET/INET6 protocols. However
289 * the treatments for TCP sockets and for datagram (UDP, ICMP, etc) sockets
290 * are slightly different.
291 *
292 * Each datagram socket may have multiple flows. Each flow is identified
293 * by the flow's source address/port and destination address/port tuple
294 * and is represented as a "struct cfil_info" entry. For each socket,
295 * a hash table is used to maintain the collection of flows under that socket.
296 *
297 * Each datagram flow is uniquely identified by it's "struct cfil_info" cfi_sock_id.
298 * The highest 32-bits of the cfi_sock_id contains the socket's so_gencnt. This portion
299 * of the cfi_sock_id is used locate the socket during socket lookup. The lowest 32-bits
300 * of the cfi_sock_id contains a hash of the flow's 4-tuple. This portion of the cfi_sock_id
301 * is used as the hash value for the flow hash table lookup within the parent socket.
302 *
303 * Since datagram sockets may not be connected, flow states may not be maintained in the
304 * socket structures and thus have to be saved for each packet. These saved states will be
305 * used for both outgoing and incoming reinjections. For outgoing packets, destination
306 * address/port as well as the current socket states will be saved. During reinjection,
307 * these saved states will be used instead. For incoming packets, control and address
308 * mbufs will be chained to the data. During reinjection, the whole chain will be queued
309 * onto the incoming socket buffer.
310 *
311 * LIMITATIONS
312 *
313 * - Support all INET/INET6 sockets, such as TCP, UDP, ICMP, etc
314 *
315 * - Does not support TCP unordered messages
316 */
317
318/*
319 * TO DO LIST
320 *
321 * Deal with OOB
322 *
323 */
324
325#include <sys/types.h>
326#include <sys/kern_control.h>
327#include <sys/queue.h>
328#include <sys/domain.h>
329#include <sys/protosw.h>
330#include <sys/syslog.h>
331#include <sys/systm.h>
332#include <sys/param.h>
333#include <sys/mbuf.h>
334
335#include <kern/locks.h>
336#include <kern/zalloc.h>
337#include <kern/debug.h>
338
339#include <net/ntstat.h>
340#include <net/content_filter.h>
341#include <net/content_filter_crypto.h>
342
343#define _IP_VHL
344#include <netinet/ip.h>
345#include <netinet/in_pcb.h>
346#include <netinet/tcp.h>
347#include <netinet/tcp_var.h>
348#include <netinet/udp.h>
349#include <netinet/udp_var.h>
350#include <kern/socket_flows.h>
351
352#include <string.h>
353#include <libkern/libkern.h>
354#include <kern/sched_prim.h>
355#include <kern/task.h>
356#include <mach/task_info.h>
357
358#include <net/sockaddr_utils.h>
359
360#define MAX_CONTENT_FILTER 8
361
362extern int tcp_msl;
363extern struct inpcbinfo ripcbinfo;
364struct cfil_entry;
365
366/*
367 * The structure content_filter represents a user space content filter
368 * It's created and associated with a kernel control socket instance
369 */
370struct content_filter {
371 kern_ctl_ref cf_kcref;
372 u_int32_t cf_kcunit;
373 u_int32_t cf_flags;
374
375 uint32_t cf_necp_control_unit;
376
377 uint32_t cf_sock_count;
378 TAILQ_HEAD(, cfil_entry) cf_sock_entries;
379
380 cfil_crypto_state_t cf_crypto_state;
381};
382
383#define CFF_ACTIVE 0x01
384#define CFF_DETACHING 0x02
385#define CFF_FLOW_CONTROLLED 0x04
386#define CFF_PRESERVE_CONNECTIONS 0x08
387
388struct content_filter *content_filters[MAX_CONTENT_FILTER];
389uint32_t cfil_active_count = 0; /* Number of active content filters */
390uint32_t cfil_sock_attached_count = 0; /* Number of sockets attachements */
391uint32_t cfil_sock_attached_stats_count = 0; /* Number of sockets requested periodic stats report */
392uint32_t cfil_close_wait_timeout = 1000; /* in milliseconds */
393
394static kern_ctl_ref cfil_kctlref = NULL;
395
396static LCK_GRP_DECLARE(cfil_lck_grp, "content filter");
397static LCK_RW_DECLARE(cfil_lck_rw, &cfil_lck_grp);
398
399#define CFIL_RW_LCK_MAX 8
400
401int cfil_rw_nxt_lck = 0;
402void* cfil_rw_lock_history[CFIL_RW_LCK_MAX];
403
404int cfil_rw_nxt_unlck = 0;
405void* cfil_rw_unlock_history[CFIL_RW_LCK_MAX];
406
407static KALLOC_TYPE_DEFINE(content_filter_zone, struct content_filter, NET_KT_DEFAULT);
408
409MBUFQ_HEAD(cfil_mqhead);
410
411struct cfil_queue {
412 uint64_t q_start; /* offset of first byte in queue */
413 uint64_t q_end; /* offset of last byte in queue */
414 struct cfil_mqhead q_mq;
415};
416
417/*
418 * struct cfil_entry
419 *
420 * The is one entry per content filter
421 */
422struct cfil_entry {
423 TAILQ_ENTRY(cfil_entry) cfe_link;
424 SLIST_ENTRY(cfil_entry) cfe_order_link;
425 struct content_filter *cfe_filter;
426
427 struct cfil_info *cfe_cfil_info;
428 uint32_t cfe_flags;
429 uint32_t cfe_necp_control_unit;
430 struct timeval cfe_last_event; /* To user space */
431 struct timeval cfe_last_action; /* From user space */
432 uint64_t cfe_byte_inbound_count_reported; /* stats already been reported */
433 uint64_t cfe_byte_outbound_count_reported; /* stats already been reported */
434 struct timeval cfe_stats_report_ts; /* Timestamp for last stats report */
435 uint32_t cfe_stats_report_frequency; /* Interval for stats report in msecs */
436 boolean_t cfe_laddr_sent;
437
438 struct cfe_buf {
439 /*
440 * cfe_pending_q holds data that has been delivered to
441 * the filter and for which we are waiting for an action
442 */
443 struct cfil_queue cfe_pending_q;
444 /*
445 * This queue is for data that has not be delivered to
446 * the content filter (new data, pass peek or flow control)
447 */
448 struct cfil_queue cfe_ctl_q;
449
450 uint64_t cfe_pass_offset;
451 uint64_t cfe_peek_offset;
452 uint64_t cfe_peeked;
453 } cfe_snd, cfe_rcv;
454};
455
456#define CFEF_CFIL_ATTACHED 0x0001 /* was attached to filter */
457#define CFEF_SENT_SOCK_ATTACHED 0x0002 /* sock attach event was sent */
458#define CFEF_DATA_START 0x0004 /* can send data event */
459#define CFEF_FLOW_CONTROLLED 0x0008 /* wait for flow control lift */
460#define CFEF_SENT_DISCONNECT_IN 0x0010 /* event was sent */
461#define CFEF_SENT_DISCONNECT_OUT 0x0020 /* event was sent */
462#define CFEF_SENT_SOCK_CLOSED 0x0040 /* closed event was sent */
463#define CFEF_CFIL_DETACHED 0x0080 /* filter was detached */
464
465
466#define CFI_ADD_TIME_LOG(cfil, t1, t0, op) \
467 struct timeval64 _tdiff; \
468 if ((cfil)->cfi_op_list_ctr < CFI_MAX_TIME_LOG_ENTRY) { \
469 timersub(t1, t0, &_tdiff); \
470 (cfil)->cfi_op_time[(cfil)->cfi_op_list_ctr] = (uint32_t)(_tdiff.tv_sec * 1000 + _tdiff.tv_usec / 1000);\
471 (cfil)->cfi_op_list[(cfil)->cfi_op_list_ctr] = (unsigned char)op; \
472 (cfil)->cfi_op_list_ctr ++; \
473 }
474
475/*
476 * struct cfil_info
477 *
478 * There is a struct cfil_info per socket
479 */
480struct cfil_info {
481 TAILQ_ENTRY(cfil_info) cfi_link;
482 TAILQ_ENTRY(cfil_info) cfi_link_stats;
483 struct socket *cfi_so;
484 uint64_t cfi_flags;
485 uint64_t cfi_sock_id;
486 struct timeval64 cfi_first_event;
487 uint32_t cfi_op_list_ctr;
488 uint32_t cfi_op_time[CFI_MAX_TIME_LOG_ENTRY]; /* time interval in microseconds since first event */
489 unsigned char cfi_op_list[CFI_MAX_TIME_LOG_ENTRY];
490 union sockaddr_in_4_6 cfi_so_attach_faddr; /* faddr at the time of attach */
491 union sockaddr_in_4_6 cfi_so_attach_laddr; /* laddr at the time of attach */
492
493 int cfi_dir;
494 uint64_t cfi_byte_inbound_count;
495 uint64_t cfi_byte_outbound_count;
496
497 boolean_t cfi_isSignatureLatest; /* Indicates if signature covers latest flow attributes */
498 u_int32_t cfi_filter_control_unit;
499 u_int32_t cfi_debug;
500 struct cfi_buf {
501 /*
502 * cfi_pending_first and cfi_pending_last describe the total
503 * amount of data outstanding for all the filters on
504 * this socket and data in the flow queue
505 * cfi_pending_mbcnt counts in sballoc() "chars of mbufs used"
506 */
507 uint64_t cfi_pending_first;
508 uint64_t cfi_pending_last;
509 uint32_t cfi_pending_mbcnt;
510 uint32_t cfi_pending_mbnum;
511 uint32_t cfi_tail_drop_cnt;
512 /*
513 * cfi_pass_offset is the minimum of all the filters
514 */
515 uint64_t cfi_pass_offset;
516 /*
517 * cfi_inject_q holds data that needs to be re-injected
518 * into the socket after filtering and that can
519 * be queued because of flow control
520 */
521 struct cfil_queue cfi_inject_q;
522 } cfi_snd, cfi_rcv;
523
524 struct cfil_entry cfi_entries[MAX_CONTENT_FILTER];
525 struct soflow_hash_entry *cfi_hash_entry;
526 SLIST_HEAD(, cfil_entry) cfi_ordered_entries;
527 os_refcnt_t cfi_ref_count;
528} __attribute__((aligned(8)));
529
530#define CFIF_DROP 0x0001 /* drop action applied */
531#define CFIF_CLOSE_WAIT 0x0002 /* waiting for filter to close */
532#define CFIF_SOCK_CLOSED 0x0004 /* socket is closed */
533#define CFIF_RETRY_INJECT_IN 0x0010 /* inject in failed */
534#define CFIF_RETRY_INJECT_OUT 0x0020 /* inject out failed */
535#define CFIF_SHUT_WR 0x0040 /* shutdown write */
536#define CFIF_SHUT_RD 0x0080 /* shutdown read */
537#define CFIF_SOCKET_CONNECTED 0x0100 /* socket is connected */
538#define CFIF_INITIAL_VERDICT 0x0200 /* received initial verdict */
539#define CFIF_NO_CLOSE_WAIT 0x0400 /* do not wait to close */
540#define CFIF_SO_DELAYED_DEAD 0x0800 /* Delayed socket DEAD marking */
541#define CFIF_SO_DELAYED_TCP_TIME_WAIT 0x1000 /* Delayed TCP FIN TIME WAIT */
542
543#define CFI_MASK_GENCNT 0xFFFFFFFF00000000 /* upper 32 bits */
544#define CFI_SHIFT_GENCNT 32
545#define CFI_MASK_FLOWHASH 0x00000000FFFFFFFF /* lower 32 bits */
546#define CFI_SHIFT_FLOWHASH 0
547
548#define CFI_ENTRY_KCUNIT(i, e) ((uint32_t)(((e) - &((i)->cfi_entries[0])) + 1))
549
550static KALLOC_TYPE_DEFINE(cfil_info_zone, struct cfil_info, NET_KT_DEFAULT);
551
552TAILQ_HEAD(cfil_sock_head, cfil_info) cfil_sock_head;
553TAILQ_HEAD(cfil_sock_head_stats, cfil_info) cfil_sock_head_stats;
554
555#define CFIL_QUEUE_VERIFY(x) if (cfil_debug) cfil_queue_verify(x)
556#define CFIL_INFO_VERIFY(x) if (cfil_debug) cfil_info_verify(x)
557
558/*
559 * UDP Socket Support
560 */
561#define IS_ICMP(so) (so && (SOCK_CHECK_TYPE(so, SOCK_RAW) || SOCK_CHECK_TYPE(so, SOCK_DGRAM)) && \
562 (SOCK_CHECK_PROTO(so, IPPROTO_ICMP) || SOCK_CHECK_PROTO(so, IPPROTO_ICMPV6)))
563#define IS_RAW(so) (so && SOCK_CHECK_TYPE(so, SOCK_RAW) && SOCK_CHECK_PROTO(so, IPPROTO_RAW))
564
565#define OPTIONAL_IP_HEADER(so) (!IS_TCP(so) && !IS_UDP(so))
566#define GET_SO_PROTOCOL(so) (so ? SOCK_PROTO(so) : IPPROTO_IP)
567#define GET_SO_INP_PROTOCOL(so) ((so && sotoinpcb(so)) ? sotoinpcb(so)->inp_ip_p : IPPROTO_IP)
568#define GET_SO_PROTO(so) ((GET_SO_PROTOCOL(so) != IPPROTO_IP) ? GET_SO_PROTOCOL(so) : GET_SO_INP_PROTOCOL(so))
569#define IS_INP_V6(inp) (inp && (inp->inp_vflag & INP_IPV6))
570
571#define UNCONNECTED(inp) (inp && (((inp->inp_vflag & INP_IPV4) && (inp->inp_faddr.s_addr == INADDR_ANY)) || \
572 ((inp->inp_vflag & INP_IPV6) && IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr))))
573#define IS_ENTRY_ATTACHED(cfil_info, kcunit) (cfil_info != NULL && (kcunit <= MAX_CONTENT_FILTER) && \
574 cfil_info->cfi_entries[kcunit - 1].cfe_filter != NULL)
575#define IS_DNS(local, remote) (check_port(local, 53) || check_port(remote, 53) || check_port(local, 5353) || check_port(remote, 5353))
576#define IS_INITIAL_TFO_DATA(so) (so && (so->so_flags1 & SOF1_PRECONNECT_DATA) && (so->so_state & SS_ISCONNECTING))
577#define NULLADDRESS(addr) ((addr.sa.sa_len == 0) || \
578 (addr.sa.sa_family == AF_INET && addr.sin.sin_addr.s_addr == 0) || \
579 (addr.sa.sa_family == AF_INET6 && IN6_IS_ADDR_UNSPECIFIED(&addr.sin6.sin6_addr)))
580
581#define SKIP_FILTER_FOR_TCP_SOCKET(so) \
582 (so == NULL || \
583 (!SOCK_CHECK_DOM(so, PF_INET) && !SOCK_CHECK_DOM(so, PF_INET6)) || \
584 !SOCK_CHECK_TYPE(so, SOCK_STREAM) || \
585 !SOCK_CHECK_PROTO(so, IPPROTO_TCP) || \
586 (so->so_flags & SOF_MP_SUBFLOW) != 0 || \
587 (so->so_flags1 & SOF1_CONTENT_FILTER_SKIP) != 0)
588
589/*
590 * Special handling for 0.0.0.0-faddr TCP flows. This flows will be changed to loopback addr by TCP and
591 * may result in an immediate TCP RESET and socket close. This leads to CFIL blocking the owner thread for
592 * 1 sec waiting for ack from user-space provider (ack recevied by CFIL but socket already removed from
593 * global socket list). To avoid this, identify these flows and do not perform the close-wait blocking.
594 * These flows are identified as destined to Loopback address and were disconnected shortly after connect
595 * (before initial-verdict received).
596 */
597#define IS_LOOPBACK_FADDR(inp) \
598 (inp && ((IS_INP_V6(inp) && IN6_IS_ADDR_LOOPBACK(&inp->in6p_faddr)) || (ntohl(inp->inp_faddr.s_addr) == INADDR_LOOPBACK)))
599
600#define SET_NO_CLOSE_WAIT(inp, cfil_info) \
601 if (inp && cfil_info && !(cfil_info->cfi_flags & CFIF_INITIAL_VERDICT) && IS_LOOPBACK_FADDR(inp)) { \
602 cfil_info->cfi_flags |= CFIF_NO_CLOSE_WAIT; \
603 }
604
605#define IS_NO_CLOSE_WAIT(cfil_info) (cfil_info && (cfil_info->cfi_flags & CFIF_NO_CLOSE_WAIT))
606
607os_refgrp_decl(static, cfil_refgrp, "CFILRefGroup", NULL);
608
609#define CFIL_INFO_FREE(cfil_info) \
610 if (cfil_info && (os_ref_release(&cfil_info->cfi_ref_count) == 0)) { \
611 cfil_info_free(cfil_info); \
612 }
613
614#define SOCKET_PID(so) ((so->so_flags & SOF_DELEGATED) ? so->e_pid : so->last_pid)
615#define MATCH_PID(so) (so && (cfil_log_pid == SOCKET_PID(so)))
616#define MATCH_PORT(inp, local, remote) \
617 ((inp && ntohs(inp->inp_lport) == cfil_log_port) || (inp && ntohs(inp->inp_fport) == cfil_log_port) || \
618 check_port(local, cfil_log_port) || check_port(remote, cfil_log_port))
619#define MATCH_PROTO(so) (GET_SO_PROTO(so) == cfil_log_proto)
620
621#define DEBUG_FLOW(inp, so, local, remote) \
622 ((cfil_log_port && MATCH_PORT(inp, local, remote)) || (cfil_log_pid && MATCH_PID(so)) || (cfil_log_proto && MATCH_PROTO(so)))
623
624#define SO_DELAYED_DEAD_SET(so, set) \
625 if (so->so_cfil) { \
626 if (set) { \
627 so->so_cfil->cfi_flags |= CFIF_SO_DELAYED_DEAD; \
628 } else { \
629 so->so_cfil->cfi_flags &= ~CFIF_SO_DELAYED_DEAD; \
630 } \
631 } else if (so->so_flow_db) { \
632 if (set) { \
633 so->so_flow_db->soflow_db_flags |= SOFLOWF_SO_DELAYED_DEAD; \
634 } else { \
635 so->so_flow_db->soflow_db_flags &= ~SOFLOWF_SO_DELAYED_DEAD; \
636 } \
637 }
638
639#define SO_DELAYED_DEAD_GET(so) \
640 (so->so_cfil ? (so->so_cfil->cfi_flags & CFIF_SO_DELAYED_DEAD) : \
641 (so->so_flow_db) ? (so->so_flow_db->soflow_db_flags & SOFLOWF_SO_DELAYED_DEAD) : false)
642
643#define SO_DELAYED_TCP_TIME_WAIT_SET(so, set) \
644 if (so->so_cfil) { \
645 if (set) { \
646 so->so_cfil->cfi_flags |= CFIF_SO_DELAYED_TCP_TIME_WAIT; \
647 } else { \
648 so->so_cfil->cfi_flags &= ~CFIF_SO_DELAYED_TCP_TIME_WAIT; \
649 } \
650 }
651
652#define SO_DELAYED_TCP_TIME_WAIT_GET(so) \
653 (so->so_cfil ? (so->so_cfil->cfi_flags & CFIF_SO_DELAYED_TCP_TIME_WAIT) : false)
654
655/*
656 * Periodic Statistics Report:
657 */
658static struct thread *cfil_stats_report_thread;
659#define CFIL_STATS_REPORT_INTERVAL_MIN_MSEC 500 // Highest report frequency
660#define CFIL_STATS_REPORT_RUN_INTERVAL_NSEC (CFIL_STATS_REPORT_INTERVAL_MIN_MSEC * NSEC_PER_MSEC)
661#define CFIL_STATS_REPORT_MAX_COUNT 50 // Max stats to be reported per run
662
663/* This buffer must have same layout as struct cfil_msg_stats_report */
664struct cfil_stats_report_buffer {
665 struct cfil_msg_hdr msghdr;
666 uint32_t count;
667 struct cfil_msg_sock_stats stats[CFIL_STATS_REPORT_MAX_COUNT];
668};
669static struct cfil_stats_report_buffer *global_cfil_stats_report_buffers[MAX_CONTENT_FILTER];
670static uint32_t global_cfil_stats_counts[MAX_CONTENT_FILTER];
671
672/*
673 * UDP Garbage Collection:
674 */
675#define UDP_FLOW_GC_ACTION_TO 10 // Flow Action Timeout (no action from user space) in seconds
676#define UDP_FLOW_GC_MAX_COUNT 100 // Max UDP flows to be handled per run
677
678/*
679 * UDP flow queue thresholds
680 */
681#define UDP_FLOW_GC_MBUF_CNT_MAX (2 << MBSHIFT) // Max mbuf byte count in flow queue (2MB)
682#define UDP_FLOW_GC_MBUF_NUM_MAX (UDP_FLOW_GC_MBUF_CNT_MAX >> MCLSHIFT) // Max mbuf count in flow queue (1K)
683#define UDP_FLOW_GC_MBUF_SHIFT 5 // Shift to get 1/32 of platform limits
684/*
685 * UDP flow queue threshold globals:
686 */
687static unsigned int cfil_udp_gc_mbuf_num_max = UDP_FLOW_GC_MBUF_NUM_MAX;
688static unsigned int cfil_udp_gc_mbuf_cnt_max = UDP_FLOW_GC_MBUF_CNT_MAX;
689
690/*
691 * CFIL specific mbuf tag:
692 * Save state of socket at the point of data entry into cfil.
693 * Use saved state for reinjection at protocol layer.
694 */
695struct cfil_tag {
696 union sockaddr_in_4_6 cfil_faddr;
697 uint32_t cfil_so_state_change_cnt;
698 uint32_t cfil_so_options;
699 int cfil_inp_flags;
700};
701
702/*
703 * Global behavior flags:
704 */
705#define CFIL_BEHAVIOR_FLAG_PRESERVE_CONNECTIONS 0x00000001
706static uint32_t cfil_behavior_flags = 0;
707
708#define DO_PRESERVE_CONNECTIONS (cfil_behavior_flags & CFIL_BEHAVIOR_FLAG_PRESERVE_CONNECTIONS)
709
710/*
711 * Statistics
712 */
713
714struct cfil_stats cfil_stats;
715
716/*
717 * For troubleshooting
718 */
719int cfil_log_level = LOG_ERR;
720int cfil_log_port = 0;
721int cfil_log_pid = 0;
722int cfil_log_proto = 0;
723int cfil_log_data = 0;
724int cfil_log_stats = 0;
725int cfil_debug = 1;
726
727/*
728 * Sysctls for logs and statistics
729 */
730static int sysctl_cfil_filter_list(struct sysctl_oid *, void *, int,
731 struct sysctl_req *);
732static int sysctl_cfil_sock_list(struct sysctl_oid *, void *, int,
733 struct sysctl_req *);
734
735SYSCTL_NODE(_net, OID_AUTO, cfil, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "cfil");
736
737SYSCTL_INT(_net_cfil, OID_AUTO, log, CTLFLAG_RW | CTLFLAG_LOCKED,
738 &cfil_log_level, 0, "");
739
740SYSCTL_INT(_net_cfil, OID_AUTO, log_port, CTLFLAG_RW | CTLFLAG_LOCKED,
741 &cfil_log_port, 0, "");
742
743SYSCTL_INT(_net_cfil, OID_AUTO, log_pid, CTLFLAG_RW | CTLFLAG_LOCKED,
744 &cfil_log_pid, 0, "");
745
746SYSCTL_INT(_net_cfil, OID_AUTO, log_proto, CTLFLAG_RW | CTLFLAG_LOCKED,
747 &cfil_log_proto, 0, "");
748
749SYSCTL_INT(_net_cfil, OID_AUTO, log_data, CTLFLAG_RW | CTLFLAG_LOCKED,
750 &cfil_log_data, 0, "");
751
752SYSCTL_INT(_net_cfil, OID_AUTO, log_stats, CTLFLAG_RW | CTLFLAG_LOCKED,
753 &cfil_log_stats, 0, "");
754
755SYSCTL_INT(_net_cfil, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_LOCKED,
756 &cfil_debug, 0, "");
757
758SYSCTL_UINT(_net_cfil, OID_AUTO, sock_attached_count, CTLFLAG_RD | CTLFLAG_LOCKED,
759 &cfil_sock_attached_count, 0, "");
760
761SYSCTL_UINT(_net_cfil, OID_AUTO, active_count, CTLFLAG_RD | CTLFLAG_LOCKED,
762 &cfil_active_count, 0, "");
763
764SYSCTL_UINT(_net_cfil, OID_AUTO, close_wait_timeout, CTLFLAG_RW | CTLFLAG_LOCKED,
765 &cfil_close_wait_timeout, 0, "");
766
767SYSCTL_UINT(_net_cfil, OID_AUTO, behavior_flags, CTLFLAG_RW | CTLFLAG_LOCKED,
768 &cfil_behavior_flags, 0, "");
769
770static int cfil_sbtrim = 1;
771SYSCTL_UINT(_net_cfil, OID_AUTO, sbtrim, CTLFLAG_RW | CTLFLAG_LOCKED,
772 &cfil_sbtrim, 0, "");
773
774SYSCTL_PROC(_net_cfil, OID_AUTO, filter_list, CTLFLAG_RD | CTLFLAG_LOCKED,
775 0, 0, sysctl_cfil_filter_list, "S,cfil_filter_stat", "");
776
777SYSCTL_PROC(_net_cfil, OID_AUTO, sock_list, CTLFLAG_RD | CTLFLAG_LOCKED,
778 0, 0, sysctl_cfil_sock_list, "S,cfil_sock_stat", "");
779
780SYSCTL_STRUCT(_net_cfil, OID_AUTO, stats, CTLFLAG_RD | CTLFLAG_LOCKED,
781 &cfil_stats, cfil_stats, "");
782
783/*
784 * Forward declaration to appease the compiler
785 */
786static int cfil_action_data_pass(struct socket *, struct cfil_info *, uint32_t, int,
787 uint64_t, uint64_t);
788static int cfil_action_drop(struct socket *, struct cfil_info *, uint32_t);
789static int cfil_action_bless_client(uint32_t, struct cfil_msg_hdr *);
790static int cfil_action_set_crypto_key(uint32_t, struct cfil_msg_hdr *);
791static int cfil_dispatch_closed_event(struct socket *, struct cfil_info *, int);
792static int cfil_data_common(struct socket *, struct cfil_info *, int, struct sockaddr *,
793 struct mbuf *, struct mbuf *, uint32_t);
794static int cfil_data_filter(struct socket *, struct cfil_info *, uint32_t, int,
795 struct mbuf *, uint32_t);
796static void fill_ip_sockaddr_4_6(union sockaddr_in_4_6 *,
797 struct in_addr, u_int16_t);
798static void fill_ip6_sockaddr_4_6(union sockaddr_in_4_6 *,
799 struct in6_addr *, u_int16_t, uint32_t);
800
801static int cfil_dispatch_attach_event(struct socket *, struct cfil_info *, uint32_t, int);
802static void cfil_info_free(struct cfil_info *);
803static struct cfil_info * cfil_info_alloc(struct socket *, struct soflow_hash_entry *);
804static int cfil_info_attach_unit(struct socket *, uint32_t, struct cfil_info *);
805static struct socket * cfil_socket_from_sock_id(cfil_sock_id_t, bool);
806static struct socket * cfil_socket_from_client_uuid(uuid_t, bool *);
807static int cfil_service_pending_queue(struct socket *, struct cfil_info *, uint32_t, int);
808static int cfil_data_service_ctl_q(struct socket *, struct cfil_info *, uint32_t, int);
809static void cfil_info_verify(struct cfil_info *);
810static int cfil_update_data_offsets(struct socket *, struct cfil_info *, uint32_t, int,
811 uint64_t, uint64_t);
812static int cfil_acquire_sockbuf(struct socket *, struct cfil_info *, int);
813static void cfil_release_sockbuf(struct socket *, int);
814static int cfil_filters_attached(struct socket *);
815
816static void cfil_rw_lock_exclusive(lck_rw_t *);
817static void cfil_rw_unlock_exclusive(lck_rw_t *);
818static void cfil_rw_lock_shared(lck_rw_t *);
819static void cfil_rw_unlock_shared(lck_rw_t *);
820static boolean_t cfil_rw_lock_shared_to_exclusive(lck_rw_t *);
821static void cfil_rw_lock_exclusive_to_shared(lck_rw_t *);
822
823static unsigned int cfil_data_length(struct mbuf *, int *, int *);
824static struct cfil_info *cfil_sock_udp_get_info(struct socket *, uint32_t, bool, struct soflow_hash_entry *, struct sockaddr *, struct sockaddr *);
825static errno_t cfil_sock_udp_handle_data(bool, struct socket *, struct sockaddr *, struct sockaddr *,
826 struct mbuf *, struct mbuf *, uint32_t, struct soflow_hash_entry *);
827static int32_t cfil_sock_udp_data_pending(struct sockbuf *, bool);
828static void cfil_sock_udp_is_closed(struct socket *);
829static int cfil_sock_udp_notify_shutdown(struct socket *, int, int, int);
830static int cfil_sock_udp_shutdown(struct socket *, int *);
831static void cfil_sock_udp_close_wait(struct socket *);
832static void cfil_sock_udp_buf_update(struct sockbuf *);
833static int cfil_filters_udp_attached(struct socket *, bool);
834static void cfil_get_flow_address_v6(struct soflow_hash_entry *, struct inpcb *,
835 struct in6_addr **, struct in6_addr **,
836 u_int16_t *, u_int16_t *);
837static void cfil_get_flow_address(struct soflow_hash_entry *, struct inpcb *,
838 struct in_addr *, struct in_addr *,
839 u_int16_t *, u_int16_t *);
840static void cfil_info_log(int, struct cfil_info *, const char *);
841void cfil_filter_show(u_int32_t);
842void cfil_info_show(void);
843bool cfil_info_action_timed_out(struct cfil_info *, int);
844bool cfil_info_buffer_threshold_exceeded(struct cfil_info *);
845struct m_tag *cfil_dgram_save_socket_state(struct cfil_info *, struct mbuf *);
846boolean_t cfil_dgram_peek_socket_state(struct mbuf *m, int *inp_flags);
847static void cfil_sock_received_verdict(struct socket *so);
848static void cfil_fill_event_msg_addresses(struct soflow_hash_entry *, struct inpcb *,
849 union sockaddr_in_4_6 *, union sockaddr_in_4_6 *,
850 boolean_t, boolean_t);
851static void cfil_stats_report_thread_func(void *, wait_result_t);
852static void cfil_stats_report(void *v, wait_result_t w);
853static bool cfil_dgram_gc_needed(struct socket *, struct soflow_hash_entry *, u_int64_t);
854static bool cfil_dgram_gc_perform(struct socket *, struct soflow_hash_entry *);
855static bool cfil_dgram_detach_entry(struct socket *, struct soflow_hash_entry *);
856static bool cfil_dgram_detach_db(struct socket *, struct soflow_db *);
857bool check_port(struct sockaddr *, u_short);
858
859/*
860 * Content filter global read write lock
861 */
862
863static void
864cfil_rw_lock_exclusive(lck_rw_t *lck)
865{
866 void *lr_saved;
867
868 lr_saved = __builtin_return_address(0);
869
870 lck_rw_lock_exclusive(lck);
871
872 cfil_rw_lock_history[cfil_rw_nxt_lck] = lr_saved;
873 cfil_rw_nxt_lck = (cfil_rw_nxt_lck + 1) % CFIL_RW_LCK_MAX;
874}
875
876static void
877cfil_rw_unlock_exclusive(lck_rw_t *lck)
878{
879 void *lr_saved;
880
881 lr_saved = __builtin_return_address(0);
882
883 lck_rw_unlock_exclusive(lck);
884
885 cfil_rw_unlock_history[cfil_rw_nxt_unlck] = lr_saved;
886 cfil_rw_nxt_unlck = (cfil_rw_nxt_unlck + 1) % CFIL_RW_LCK_MAX;
887}
888
889static void
890cfil_rw_lock_shared(lck_rw_t *lck)
891{
892 void *lr_saved;
893
894 lr_saved = __builtin_return_address(0);
895
896 lck_rw_lock_shared(lck);
897
898 cfil_rw_lock_history[cfil_rw_nxt_lck] = lr_saved;
899 cfil_rw_nxt_lck = (cfil_rw_nxt_lck + 1) % CFIL_RW_LCK_MAX;
900}
901
902static void
903cfil_rw_unlock_shared(lck_rw_t *lck)
904{
905 void *lr_saved;
906
907 lr_saved = __builtin_return_address(0);
908
909 lck_rw_unlock_shared(lck);
910
911 cfil_rw_unlock_history[cfil_rw_nxt_unlck] = lr_saved;
912 cfil_rw_nxt_unlck = (cfil_rw_nxt_unlck + 1) % CFIL_RW_LCK_MAX;
913}
914
915static boolean_t
916cfil_rw_lock_shared_to_exclusive(lck_rw_t *lck)
917{
918 void *lr_saved;
919 boolean_t upgraded;
920
921 lr_saved = __builtin_return_address(0);
922
923 upgraded = lck_rw_lock_shared_to_exclusive(lck);
924 if (upgraded) {
925 cfil_rw_unlock_history[cfil_rw_nxt_unlck] = lr_saved;
926 cfil_rw_nxt_unlck = (cfil_rw_nxt_unlck + 1) % CFIL_RW_LCK_MAX;
927 }
928 return upgraded;
929}
930
931static void
932cfil_rw_lock_exclusive_to_shared(lck_rw_t *lck)
933{
934 void *lr_saved;
935
936 lr_saved = __builtin_return_address(0);
937
938 lck_rw_lock_exclusive_to_shared(lck);
939
940 cfil_rw_lock_history[cfil_rw_nxt_lck] = lr_saved;
941 cfil_rw_nxt_lck = (cfil_rw_nxt_lck + 1) % CFIL_RW_LCK_MAX;
942}
943
944static void
945cfil_rw_lock_assert_held(lck_rw_t *lck, int exclusive)
946{
947#if !MACH_ASSERT
948#pragma unused(lck, exclusive)
949#endif
950 LCK_RW_ASSERT(lck,
951 exclusive ? LCK_RW_ASSERT_EXCLUSIVE : LCK_RW_ASSERT_HELD);
952}
953
954/*
955 * Return the number of bytes in the mbuf chain using the same
956 * method as m_length() or sballoc()
957 *
958 * Returns data len - starting from PKT start
959 * - retmbcnt - optional param to get total mbuf bytes in chain
960 * - retmbnum - optional param to get number of mbufs in chain
961 */
962static unsigned int
963cfil_data_length(struct mbuf *m, int *retmbcnt, int *retmbnum)
964{
965 struct mbuf *m0;
966 unsigned int pktlen = 0;
967 int mbcnt;
968 int mbnum;
969
970 // Locate M_PKTHDR and mark as start of data if present
971 for (m0 = m; m0 != NULL; m0 = m0->m_next) {
972 if (m0->m_flags & M_PKTHDR) {
973 m = m0;
974 break;
975 }
976 }
977
978 if (retmbcnt == NULL && retmbnum == NULL) {
979 return m_length(m);
980 }
981
982 pktlen = 0;
983 mbcnt = 0;
984 mbnum = 0;
985 for (m0 = m; m0 != NULL; m0 = m0->m_next) {
986 pktlen += m0->m_len;
987 mbnum++;
988 mbcnt += _MSIZE;
989 if (m0->m_flags & M_EXT) {
990 mbcnt += m0->m_ext.ext_size;
991 }
992 }
993 if (retmbcnt) {
994 *retmbcnt = mbcnt;
995 }
996 if (retmbnum) {
997 *retmbnum = mbnum;
998 }
999 return pktlen;
1000}
1001
1002static struct mbuf *
1003cfil_data_start(struct mbuf *m)
1004{
1005 struct mbuf *m0;
1006
1007 // Locate M_PKTHDR and use it as start of data if present
1008 for (m0 = m; m0 != NULL; m0 = m0->m_next) {
1009 if (m0->m_flags & M_PKTHDR) {
1010 return m0;
1011 }
1012 }
1013 return m;
1014}
1015
1016/*
1017 * Common mbuf queue utilities
1018 */
1019
1020static inline void
1021cfil_queue_init(struct cfil_queue *cfq)
1022{
1023 cfq->q_start = 0;
1024 cfq->q_end = 0;
1025 MBUFQ_INIT(&cfq->q_mq);
1026}
1027
1028static inline uint64_t
1029cfil_queue_drain(struct cfil_queue *cfq)
1030{
1031 uint64_t drained = cfq->q_start - cfq->q_end;
1032 cfq->q_start = 0;
1033 cfq->q_end = 0;
1034 MBUFQ_DRAIN(&cfq->q_mq);
1035
1036 return drained;
1037}
1038
1039/* Return 1 when empty, 0 otherwise */
1040static inline int
1041cfil_queue_empty(struct cfil_queue *cfq)
1042{
1043 return MBUFQ_EMPTY(&cfq->q_mq);
1044}
1045
1046static inline uint64_t
1047cfil_queue_offset_first(struct cfil_queue *cfq)
1048{
1049 return cfq->q_start;
1050}
1051
1052static inline uint64_t
1053cfil_queue_offset_last(struct cfil_queue *cfq)
1054{
1055 return cfq->q_end;
1056}
1057
1058static inline uint64_t
1059cfil_queue_len(struct cfil_queue *cfq)
1060{
1061 return cfq->q_end - cfq->q_start;
1062}
1063
1064/*
1065 * Routines to verify some fundamental assumptions
1066 */
1067
1068static void
1069cfil_queue_verify(struct cfil_queue *cfq)
1070{
1071 mbuf_t chain;
1072 mbuf_t m;
1073 mbuf_t n;
1074 uint64_t queuesize = 0;
1075
1076 /* Verify offset are ordered */
1077 VERIFY(cfq->q_start <= cfq->q_end);
1078
1079 /*
1080 * When queue is empty, the offsets are equal otherwise the offsets
1081 * are different
1082 */
1083 VERIFY((MBUFQ_EMPTY(&cfq->q_mq) && cfq->q_start == cfq->q_end) ||
1084 (!MBUFQ_EMPTY(&cfq->q_mq) &&
1085 cfq->q_start != cfq->q_end));
1086
1087 MBUFQ_FOREACH(chain, &cfq->q_mq) {
1088 size_t chainsize = 0;
1089 m = chain;
1090 unsigned int mlen = cfil_data_length(m, NULL, NULL);
1091 // skip the addr and control stuff if present
1092 m = cfil_data_start(m);
1093
1094 if (m == NULL ||
1095 m == (void *)M_TAG_FREE_PATTERN ||
1096 m->m_next == (void *)M_TAG_FREE_PATTERN ||
1097 m->m_nextpkt == (void *)M_TAG_FREE_PATTERN) {
1098 panic("%s - mq %p is free at %p", __func__,
1099 &cfq->q_mq, m);
1100 }
1101 for (n = m; n != NULL; n = n->m_next) {
1102 if (!m_has_mtype(m: n, mtype_flags: MTF_DATA | MTF_HEADER | MTF_OOBDATA)) {
1103 panic("%s - %p unsupported type %u", __func__,
1104 n, n->m_type);
1105 }
1106 chainsize += n->m_len;
1107 }
1108 if (mlen != chainsize) {
1109 panic("%s - %p m_length() %u != chainsize %lu",
1110 __func__, m, mlen, chainsize);
1111 }
1112 queuesize += chainsize;
1113 }
1114 OS_ANALYZER_SUPPRESS("81031590") if (queuesize != cfq->q_end - cfq->q_start) {
1115 panic("%s - %p queuesize %llu != offsetdiffs %llu", __func__,
1116 m, queuesize, cfq->q_end - cfq->q_start);
1117 }
1118}
1119
1120static void
1121cfil_queue_enqueue(struct cfil_queue *cfq, mbuf_t m, size_t len)
1122{
1123 CFIL_QUEUE_VERIFY(cfq);
1124
1125 MBUFQ_ENQUEUE(&cfq->q_mq, m);
1126 cfq->q_end += len;
1127
1128 CFIL_QUEUE_VERIFY(cfq);
1129}
1130
1131static void
1132cfil_queue_remove(struct cfil_queue *cfq, mbuf_t m, size_t len)
1133{
1134 CFIL_QUEUE_VERIFY(cfq);
1135
1136 VERIFY(cfil_data_length(m, NULL, NULL) == len);
1137
1138 MBUFQ_REMOVE(&cfq->q_mq, m);
1139 MBUFQ_NEXT(m) = NULL;
1140 cfq->q_start += len;
1141
1142 CFIL_QUEUE_VERIFY(cfq);
1143}
1144
1145static mbuf_t
1146cfil_queue_first(struct cfil_queue *cfq)
1147{
1148 return MBUFQ_FIRST(&cfq->q_mq);
1149}
1150
1151static mbuf_t
1152cfil_queue_next(struct cfil_queue *cfq, mbuf_t m)
1153{
1154#pragma unused(cfq)
1155 return MBUFQ_NEXT(m);
1156}
1157
1158static void
1159cfil_entry_buf_verify(struct cfe_buf *cfe_buf)
1160{
1161 CFIL_QUEUE_VERIFY(&cfe_buf->cfe_ctl_q);
1162 CFIL_QUEUE_VERIFY(&cfe_buf->cfe_pending_q);
1163
1164 /* Verify the queues are ordered so that pending is before ctl */
1165 VERIFY(cfe_buf->cfe_ctl_q.q_start >= cfe_buf->cfe_pending_q.q_end);
1166
1167 /* The peek offset cannot be less than the pass offset */
1168 VERIFY(cfe_buf->cfe_peek_offset >= cfe_buf->cfe_pass_offset);
1169
1170 /* Make sure we've updated the offset we peeked at */
1171 VERIFY(cfe_buf->cfe_ctl_q.q_start <= cfe_buf->cfe_peeked);
1172}
1173
1174static void
1175cfil_entry_verify(struct cfil_entry *entry)
1176{
1177 cfil_entry_buf_verify(cfe_buf: &entry->cfe_snd);
1178 cfil_entry_buf_verify(cfe_buf: &entry->cfe_rcv);
1179}
1180
1181static void
1182cfil_info_buf_verify(struct cfi_buf *cfi_buf)
1183{
1184 CFIL_QUEUE_VERIFY(&cfi_buf->cfi_inject_q);
1185
1186 VERIFY(cfi_buf->cfi_pending_first <= cfi_buf->cfi_pending_last);
1187}
1188
1189static void
1190cfil_info_verify(struct cfil_info *cfil_info)
1191{
1192 int i;
1193
1194 if (cfil_info == NULL) {
1195 return;
1196 }
1197
1198 cfil_info_buf_verify(cfi_buf: &cfil_info->cfi_snd);
1199 cfil_info_buf_verify(cfi_buf: &cfil_info->cfi_rcv);
1200
1201 for (i = 0; i < MAX_CONTENT_FILTER; i++) {
1202 cfil_entry_verify(entry: &cfil_info->cfi_entries[i]);
1203 }
1204}
1205
1206static void
1207verify_content_filter(struct content_filter *cfc)
1208{
1209 struct cfil_entry *entry;
1210 uint32_t count = 0;
1211
1212 VERIFY(cfc->cf_sock_count >= 0);
1213
1214 TAILQ_FOREACH(entry, &cfc->cf_sock_entries, cfe_link) {
1215 count++;
1216 VERIFY(cfc == entry->cfe_filter);
1217 }
1218 VERIFY(count == cfc->cf_sock_count);
1219}
1220
1221/*
1222 * Kernel control socket callbacks
1223 */
1224static errno_t
1225cfil_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
1226 void **unitinfo)
1227{
1228 errno_t error = 0;
1229 struct content_filter *cfc = NULL;
1230
1231 CFIL_LOG(LOG_NOTICE, "");
1232
1233 cfc = zalloc_flags(content_filter_zone, Z_WAITOK | Z_ZERO | Z_NOFAIL);
1234
1235 cfil_rw_lock_exclusive(lck: &cfil_lck_rw);
1236
1237 if (sac->sc_unit == 0 || sac->sc_unit > MAX_CONTENT_FILTER) {
1238 CFIL_LOG(LOG_ERR, "bad sc_unit %u", sac->sc_unit);
1239 error = EINVAL;
1240 } else if (content_filters[sac->sc_unit - 1] != NULL) {
1241 CFIL_LOG(LOG_ERR, "sc_unit %u in use", sac->sc_unit);
1242 error = EADDRINUSE;
1243 } else {
1244 /*
1245 * kernel control socket kcunit numbers start at 1
1246 */
1247 content_filters[sac->sc_unit - 1] = cfc;
1248
1249 cfc->cf_kcref = kctlref;
1250 cfc->cf_kcunit = sac->sc_unit;
1251 TAILQ_INIT(&cfc->cf_sock_entries);
1252
1253 *unitinfo = cfc;
1254 cfil_active_count++;
1255
1256 if (cfil_active_count == 1) {
1257 soflow_feat_set_functions(cfil_dgram_gc_needed, cfil_dgram_gc_perform,
1258 cfil_dgram_detach_entry, cfil_dgram_detach_db);
1259 }
1260
1261 // Allocate periodic stats buffer for this filter
1262 if (global_cfil_stats_report_buffers[cfc->cf_kcunit - 1] == NULL) {
1263 cfil_rw_unlock_exclusive(lck: &cfil_lck_rw);
1264
1265 struct cfil_stats_report_buffer *buf;
1266
1267 buf = kalloc_type(struct cfil_stats_report_buffer,
1268 Z_WAITOK | Z_ZERO | Z_NOFAIL);
1269
1270 cfil_rw_lock_exclusive(lck: &cfil_lck_rw);
1271
1272 /* Another thread may have won the race */
1273 if (global_cfil_stats_report_buffers[cfc->cf_kcunit - 1] != NULL) {
1274 kfree_type(struct cfil_stats_report_buffer, buf);
1275 } else {
1276 global_cfil_stats_report_buffers[cfc->cf_kcunit - 1] = buf;
1277 }
1278 }
1279 }
1280 cfil_rw_unlock_exclusive(lck: &cfil_lck_rw);
1281
1282 if (error != 0 && cfc != NULL) {
1283 zfree(content_filter_zone, cfc);
1284 }
1285
1286 if (error == 0) {
1287 OSIncrementAtomic(&cfil_stats.cfs_ctl_connect_ok);
1288 } else {
1289 OSIncrementAtomic(&cfil_stats.cfs_ctl_connect_fail);
1290 }
1291
1292 CFIL_LOG(LOG_INFO, "return %d cfil_active_count %u kcunit %u",
1293 error, cfil_active_count, sac->sc_unit);
1294
1295 return error;
1296}
1297
1298static void
1299cfil_update_behavior_flags(void)
1300{
1301 struct content_filter *cfc = NULL;
1302
1303 // Update global flag
1304 bool preserve_connections = false;
1305 for (int i = 0; i < MAX_CONTENT_FILTER; i++) {
1306 cfc = content_filters[i];
1307 if (cfc != NULL) {
1308 if (cfc->cf_flags & CFF_PRESERVE_CONNECTIONS) {
1309 preserve_connections = true;
1310 } else {
1311 preserve_connections = false;
1312 break;
1313 }
1314 }
1315 }
1316 if (preserve_connections == true) {
1317 cfil_behavior_flags |= CFIL_BEHAVIOR_FLAG_PRESERVE_CONNECTIONS;
1318 } else {
1319 cfil_behavior_flags &= ~CFIL_BEHAVIOR_FLAG_PRESERVE_CONNECTIONS;
1320 }
1321 CFIL_LOG(LOG_INFO, "CFIL Preserve Connections - %s",
1322 (cfil_behavior_flags & CFIL_BEHAVIOR_FLAG_PRESERVE_CONNECTIONS) ? "On" : "Off");
1323}
1324
1325static errno_t
1326cfil_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo)
1327{
1328#pragma unused(kctlref)
1329 errno_t error = 0;
1330 struct content_filter *cfc;
1331 struct cfil_entry *entry;
1332 uint64_t sock_flow_id = 0;
1333
1334 CFIL_LOG(LOG_NOTICE, "");
1335
1336 if (kcunit > MAX_CONTENT_FILTER) {
1337 CFIL_LOG(LOG_ERR, "kcunit %u > MAX_CONTENT_FILTER (%d)",
1338 kcunit, MAX_CONTENT_FILTER);
1339 error = EINVAL;
1340 goto done;
1341 }
1342
1343 cfc = (struct content_filter *)unitinfo;
1344 if (cfc == NULL) {
1345 goto done;
1346 }
1347
1348 cfil_rw_lock_exclusive(lck: &cfil_lck_rw);
1349 if (content_filters[kcunit - 1] != cfc || cfc->cf_kcunit != kcunit) {
1350 CFIL_LOG(LOG_ERR, "bad unit info %u)",
1351 kcunit);
1352 cfil_rw_unlock_exclusive(lck: &cfil_lck_rw);
1353 goto done;
1354 }
1355 cfc->cf_flags |= CFF_DETACHING;
1356 /*
1357 * Remove all sockets from the filter
1358 */
1359 while ((entry = TAILQ_FIRST(&cfc->cf_sock_entries)) != NULL) {
1360 cfil_rw_lock_assert_held(lck: &cfil_lck_rw, exclusive: 1);
1361
1362 verify_content_filter(cfc);
1363 /*
1364 * Accept all outstanding data by pushing to next filter
1365 * or back to socket
1366 *
1367 * TBD: Actually we should make sure all data has been pushed
1368 * back to socket
1369 */
1370 if (entry->cfe_cfil_info && entry->cfe_cfil_info->cfi_so) {
1371 struct cfil_info *cfil_info = entry->cfe_cfil_info;
1372 struct socket *so = cfil_info->cfi_so;
1373 sock_flow_id = cfil_info->cfi_sock_id;
1374
1375 /* Need to let data flow immediately */
1376 entry->cfe_flags |= CFEF_SENT_SOCK_ATTACHED |
1377 CFEF_DATA_START;
1378
1379 // Before we release global lock, retain the cfil_info -
1380 // We attempt to retain a valid cfil_info to prevent any deallocation until
1381 // we are done. Abort retain if cfil_info has already entered the free code path.
1382 if (cfil_info == NULL || os_ref_retain_try(rc: &cfil_info->cfi_ref_count) == false) {
1383 // Failing to retain cfil_info means detach is in progress already,
1384 // remove entry from filter list and move on.
1385 entry->cfe_filter = NULL;
1386 entry->cfe_necp_control_unit = 0;
1387 TAILQ_REMOVE(&cfc->cf_sock_entries, entry, cfe_link);
1388 cfc->cf_sock_count--;
1389 continue;
1390 }
1391
1392 /*
1393 * Respect locking hierarchy
1394 */
1395 cfil_rw_unlock_exclusive(lck: &cfil_lck_rw);
1396
1397 // Search for socket from cfil_info sock_flow_id and lock so
1398 so = cfil_socket_from_sock_id(sock_flow_id, false);
1399 if (so == NULL || so != cfil_info->cfi_so) {
1400 cfil_rw_lock_exclusive(lck: &cfil_lck_rw);
1401
1402 // Socket has already been disconnected and removed from socket list.
1403 // Remove entry from filter list and move on.
1404 if (entry == TAILQ_FIRST(&cfc->cf_sock_entries)) {
1405 entry->cfe_filter = NULL;
1406 entry->cfe_necp_control_unit = 0;
1407 TAILQ_REMOVE(&cfc->cf_sock_entries, entry, cfe_link);
1408 cfc->cf_sock_count--;
1409 }
1410
1411 goto release_cfil_info;
1412 }
1413
1414 /*
1415 * When cfe_filter is NULL the filter is detached
1416 * and the entry has been removed from cf_sock_entries
1417 */
1418 if ((so->so_cfil == NULL && so->so_flow_db == NULL) || entry->cfe_filter == NULL) {
1419 cfil_rw_lock_exclusive(lck: &cfil_lck_rw);
1420 goto release;
1421 }
1422
1423 (void) cfil_action_data_pass(so, cfil_info, kcunit, 1,
1424 CFM_MAX_OFFSET,
1425 CFM_MAX_OFFSET);
1426
1427 (void) cfil_action_data_pass(so, cfil_info, kcunit, 0,
1428 CFM_MAX_OFFSET,
1429 CFM_MAX_OFFSET);
1430
1431 cfil_rw_lock_exclusive(lck: &cfil_lck_rw);
1432
1433 /*
1434 * Check again to make sure if the cfil_info is still valid
1435 * as the socket may have been unlocked when when calling
1436 * cfil_acquire_sockbuf()
1437 */
1438 if (entry->cfe_filter == NULL ||
1439 (so->so_cfil == NULL && soflow_db_get_feature_context(so->so_flow_db, sock_flow_id) == NULL)) {
1440 goto release;
1441 }
1442
1443 /* The filter is now detached */
1444 entry->cfe_flags |= CFEF_CFIL_DETACHED;
1445
1446 if (cfil_info->cfi_debug) {
1447 cfil_info_log(LOG_ERR, cfil_info, "CFIL: FILTER DISCONNECTED");
1448 }
1449
1450 CFIL_LOG(LOG_NOTICE, "so %llx detached %u",
1451 (uint64_t)VM_KERNEL_ADDRPERM(so), kcunit);
1452 if ((cfil_info->cfi_flags & CFIF_CLOSE_WAIT) &&
1453 cfil_filters_attached(so) == 0) {
1454 CFIL_LOG(LOG_NOTICE, "so %llx waking",
1455 (uint64_t)VM_KERNEL_ADDRPERM(so));
1456 wakeup(chan: (caddr_t)cfil_info);
1457 }
1458
1459 /*
1460 * Remove the filter entry from the content filter
1461 * but leave the rest of the state intact as the queues
1462 * may not be empty yet
1463 */
1464 entry->cfe_filter = NULL;
1465 entry->cfe_necp_control_unit = 0;
1466
1467 TAILQ_REMOVE(&cfc->cf_sock_entries, entry, cfe_link);
1468 cfc->cf_sock_count--;
1469
1470 // This is the last filter disconnecting, clear the cfil_info
1471 // saved control unit so we will be able to drop this flow if
1472 // a new filter get installed.
1473 if (cfil_active_count == 1) {
1474 cfil_info->cfi_filter_control_unit = 0;
1475 }
1476release:
1477 socket_unlock(so, refcount: 1);
1478
1479release_cfil_info:
1480 /*
1481 * Release reference on cfil_info. To avoid double locking,
1482 * temporarily unlock in case it has been detached and we
1483 * end up freeing it which will take the global lock again.
1484 */
1485 cfil_rw_unlock_exclusive(lck: &cfil_lck_rw);
1486 CFIL_INFO_FREE(cfil_info);
1487 cfil_rw_lock_exclusive(lck: &cfil_lck_rw);
1488 }
1489 }
1490 verify_content_filter(cfc);
1491
1492 /* Free the stats buffer for this filter */
1493 if (global_cfil_stats_report_buffers[cfc->cf_kcunit - 1] != NULL) {
1494 kfree_type(struct cfil_stats_report_buffer,
1495 global_cfil_stats_report_buffers[cfc->cf_kcunit - 1]);
1496 global_cfil_stats_report_buffers[cfc->cf_kcunit - 1] = NULL;
1497 }
1498 VERIFY(cfc->cf_sock_count == 0);
1499
1500 /*
1501 * Make filter inactive
1502 */
1503 content_filters[kcunit - 1] = NULL;
1504 cfil_active_count--;
1505 cfil_update_behavior_flags();
1506 cfil_rw_unlock_exclusive(lck: &cfil_lck_rw);
1507
1508 if (cfc->cf_crypto_state != NULL) {
1509 cfil_crypto_cleanup_state(state: cfc->cf_crypto_state);
1510 cfc->cf_crypto_state = NULL;
1511 }
1512
1513 zfree(content_filter_zone, cfc);
1514done:
1515 if (error == 0) {
1516 OSIncrementAtomic(&cfil_stats.cfs_ctl_disconnect_ok);
1517 } else {
1518 OSIncrementAtomic(&cfil_stats.cfs_ctl_disconnect_fail);
1519 }
1520
1521 CFIL_LOG(LOG_INFO, "return %d cfil_active_count %u kcunit %u",
1522 error, cfil_active_count, kcunit);
1523
1524 return error;
1525}
1526
1527/*
1528 * cfil_acquire_sockbuf()
1529 *
1530 * Prevent any other thread from acquiring the sockbuf
1531 * We use sb_cfil_thread as a semaphore to prevent other threads from
1532 * messing with the sockbuf -- see sblock()
1533 * Note: We do not set SB_LOCK here because the thread may check or modify
1534 * SB_LOCK several times until it calls cfil_release_sockbuf() -- currently
1535 * sblock(), sbunlock() or sodefunct()
1536 */
1537static int
1538cfil_acquire_sockbuf(struct socket *so, struct cfil_info *cfil_info, int outgoing)
1539{
1540 thread_t tp = current_thread();
1541 struct sockbuf *sb = outgoing ? &so->so_snd : &so->so_rcv;
1542 lck_mtx_t *mutex_held;
1543 int error = 0;
1544
1545 /*
1546 * Wait until no thread is holding the sockbuf and other content
1547 * filter threads have released the sockbuf
1548 */
1549 while ((sb->sb_flags & SB_LOCK) ||
1550 (sb->sb_cfil_thread != NULL && sb->sb_cfil_thread != tp)) {
1551 if (so->so_proto->pr_getlock != NULL) {
1552 mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK);
1553 } else {
1554 mutex_held = so->so_proto->pr_domain->dom_mtx;
1555 }
1556
1557 LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED);
1558
1559 sb->sb_wantlock++;
1560 VERIFY(sb->sb_wantlock != 0);
1561
1562 msleep(chan: &sb->sb_flags, mtx: mutex_held, PSOCK, wmesg: "cfil_acquire_sockbuf",
1563 NULL);
1564
1565 VERIFY(sb->sb_wantlock != 0);
1566 sb->sb_wantlock--;
1567 }
1568 /*
1569 * Use reference count for repetitive calls on same thread
1570 */
1571 if (sb->sb_cfil_refs == 0) {
1572 VERIFY(sb->sb_cfil_thread == NULL);
1573 VERIFY((sb->sb_flags & SB_LOCK) == 0);
1574
1575 sb->sb_cfil_thread = tp;
1576 sb->sb_flags |= SB_LOCK;
1577 }
1578 sb->sb_cfil_refs++;
1579
1580 /* We acquire the socket buffer when we need to cleanup */
1581 if (cfil_info == NULL) {
1582 CFIL_LOG(LOG_ERR, "so %llx cfil detached",
1583 (uint64_t)VM_KERNEL_ADDRPERM(so));
1584 error = 0;
1585 } else if (cfil_info->cfi_flags & CFIF_DROP) {
1586 CFIL_LOG(LOG_ERR, "so %llx drop set",
1587 (uint64_t)VM_KERNEL_ADDRPERM(so));
1588 error = EPIPE;
1589 }
1590
1591 return error;
1592}
1593
1594static void
1595cfil_release_sockbuf(struct socket *so, int outgoing)
1596{
1597 struct sockbuf *sb = outgoing ? &so->so_snd : &so->so_rcv;
1598 thread_t tp = current_thread();
1599
1600 socket_lock_assert_owned(so);
1601
1602 if (sb->sb_cfil_thread != NULL && sb->sb_cfil_thread != tp) {
1603 panic("%s sb_cfil_thread %p not current %p", __func__,
1604 sb->sb_cfil_thread, tp);
1605 }
1606 /*
1607 * Don't panic if we are defunct because SB_LOCK has
1608 * been cleared by sodefunct()
1609 */
1610 if (!(so->so_flags & SOF_DEFUNCT) && !(sb->sb_flags & SB_LOCK)) {
1611 panic("%s SB_LOCK not set on %p", __func__,
1612 sb);
1613 }
1614 /*
1615 * We can unlock when the thread unwinds to the last reference
1616 */
1617 sb->sb_cfil_refs--;
1618 if (sb->sb_cfil_refs == 0) {
1619 sb->sb_cfil_thread = NULL;
1620 sb->sb_flags &= ~SB_LOCK;
1621
1622 if (sb->sb_wantlock > 0) {
1623 wakeup(chan: &sb->sb_flags);
1624 }
1625 }
1626}
1627
1628cfil_sock_id_t
1629cfil_sock_id_from_socket(struct socket *so)
1630{
1631 if ((so->so_flags & SOF_CONTENT_FILTER) && so->so_cfil) {
1632 return so->so_cfil->cfi_sock_id;
1633 } else {
1634 return CFIL_SOCK_ID_NONE;
1635 }
1636}
1637
1638/*
1639 * cfil_socket_safe_lock -
1640 * This routine attempts to lock the socket safely.
1641 *
1642 * The passed in pcbinfo is assumed to be locked and must be unlocked once the
1643 * inp state is safeguarded and before we attempt to lock/unlock the socket.
1644 * This is to prevent getting blocked by socket_lock() while holding the pcbinfo
1645 * lock, avoiding potential deadlock with other processes contending for the same
1646 * resources. This is also to avoid double locking the pcbinfo for rip sockets
1647 * since rip_unlock() will lock ripcbinfo if it needs to dispose inpcb when
1648 * so_usecount is 0.
1649 */
1650static bool
1651cfil_socket_safe_lock(struct inpcb *inp, struct inpcbinfo *pcbinfo)
1652{
1653 struct socket *so = NULL;
1654
1655 VERIFY(pcbinfo != NULL);
1656
1657 if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) != WNT_STOPUSING) {
1658 // Safeguarded the inp state, unlock pcbinfo before locking socket.
1659 lck_rw_done(lck: &pcbinfo->ipi_lock);
1660
1661 so = inp->inp_socket;
1662 socket_lock(so, refcount: 1);
1663 if (in_pcb_checkstate(inp, WNT_RELEASE, 1) != WNT_STOPUSING) {
1664 return true;
1665 }
1666 } else {
1667 // Failed to safeguarded the inp state, unlock pcbinfo and abort.
1668 lck_rw_done(lck: &pcbinfo->ipi_lock);
1669 }
1670
1671 if (so) {
1672 socket_unlock(so, refcount: 1);
1673 }
1674 return false;
1675}
1676
1677static struct socket *
1678cfil_socket_from_sock_id(cfil_sock_id_t cfil_sock_id, bool udp_only)
1679{
1680 struct socket *so = NULL;
1681 u_int64_t gencnt = cfil_sock_id >> 32;
1682 u_int32_t flowhash = (u_int32_t)(cfil_sock_id & 0x0ffffffff);
1683 struct inpcb *inp = NULL;
1684 struct inpcbinfo *pcbinfo = NULL;
1685
1686 if (udp_only) {
1687 goto find_udp;
1688 }
1689
1690 pcbinfo = &tcbinfo;
1691 lck_rw_lock_shared(lck: &pcbinfo->ipi_lock);
1692 LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
1693 if (inp->inp_state != INPCB_STATE_DEAD &&
1694 inp->inp_socket != NULL &&
1695 inp->inp_flowhash == flowhash &&
1696 (inp->inp_socket->so_gencnt & 0x0ffffffff) == gencnt &&
1697 inp->inp_socket->so_cfil != NULL) {
1698 if (cfil_socket_safe_lock(inp, pcbinfo)) {
1699 so = inp->inp_socket;
1700 }
1701 /* pcbinfo is already unlocked, we are done. */
1702 goto done;
1703 }
1704 }
1705 lck_rw_done(lck: &pcbinfo->ipi_lock);
1706 if (so != NULL) {
1707 goto done;
1708 }
1709
1710find_udp:
1711
1712 pcbinfo = &udbinfo;
1713 lck_rw_lock_shared(lck: &pcbinfo->ipi_lock);
1714 LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
1715 if (inp->inp_state != INPCB_STATE_DEAD &&
1716 inp->inp_socket != NULL &&
1717 inp->inp_socket->so_flow_db != NULL &&
1718 (inp->inp_socket->so_gencnt & 0x0ffffffff) == gencnt) {
1719 if (cfil_socket_safe_lock(inp, pcbinfo)) {
1720 so = inp->inp_socket;
1721 }
1722 /* pcbinfo is already unlocked, we are done. */
1723 goto done;
1724 }
1725 }
1726 lck_rw_done(lck: &pcbinfo->ipi_lock);
1727 if (so != NULL) {
1728 goto done;
1729 }
1730
1731 pcbinfo = &ripcbinfo;
1732 lck_rw_lock_shared(lck: &pcbinfo->ipi_lock);
1733 LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
1734 if (inp->inp_state != INPCB_STATE_DEAD &&
1735 inp->inp_socket != NULL &&
1736 inp->inp_socket->so_flow_db != NULL &&
1737 (inp->inp_socket->so_gencnt & 0x0ffffffff) == gencnt) {
1738 if (cfil_socket_safe_lock(inp, pcbinfo)) {
1739 so = inp->inp_socket;
1740 }
1741 /* pcbinfo is already unlocked, we are done. */
1742 goto done;
1743 }
1744 }
1745 lck_rw_done(lck: &pcbinfo->ipi_lock);
1746
1747done:
1748 if (so == NULL) {
1749 OSIncrementAtomic(&cfil_stats.cfs_sock_id_not_found);
1750 CFIL_LOG(LOG_DEBUG,
1751 "no socket for sock_id %llx gencnt %llx flowhash %x",
1752 cfil_sock_id, gencnt, flowhash);
1753 }
1754
1755 return so;
1756}
1757
1758static struct socket *
1759cfil_socket_from_client_uuid(uuid_t necp_client_uuid, bool *cfil_attached)
1760{
1761 struct socket *so = NULL;
1762 struct inpcb *inp = NULL;
1763 struct inpcbinfo *pcbinfo = &tcbinfo;
1764
1765 lck_rw_lock_shared(lck: &pcbinfo->ipi_lock);
1766 LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
1767 if (inp->inp_state != INPCB_STATE_DEAD &&
1768 inp->inp_socket != NULL &&
1769 uuid_compare(uu1: inp->necp_client_uuid, uu2: necp_client_uuid) == 0) {
1770 *cfil_attached = (inp->inp_socket->so_cfil != NULL);
1771 if (cfil_socket_safe_lock(inp, pcbinfo)) {
1772 so = inp->inp_socket;
1773 }
1774 /* pcbinfo is already unlocked, we are done. */
1775 goto done;
1776 }
1777 }
1778 lck_rw_done(lck: &pcbinfo->ipi_lock);
1779 if (so != NULL) {
1780 goto done;
1781 }
1782
1783 pcbinfo = &udbinfo;
1784 lck_rw_lock_shared(lck: &pcbinfo->ipi_lock);
1785 LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
1786 if (inp->inp_state != INPCB_STATE_DEAD &&
1787 inp->inp_socket != NULL &&
1788 uuid_compare(uu1: inp->necp_client_uuid, uu2: necp_client_uuid) == 0) {
1789 *cfil_attached = (inp->inp_socket->so_flow_db != NULL);
1790 if (cfil_socket_safe_lock(inp, pcbinfo)) {
1791 so = inp->inp_socket;
1792 }
1793 /* pcbinfo is already unlocked, we are done. */
1794 goto done;
1795 }
1796 }
1797 lck_rw_done(lck: &pcbinfo->ipi_lock);
1798
1799done:
1800 return so;
1801}
1802
1803static void
1804cfil_info_stats_toggle(struct cfil_info *cfil_info, struct cfil_entry *entry, uint32_t report_frequency)
1805{
1806 struct cfil_info *cfil = NULL;
1807 Boolean found = FALSE;
1808 int kcunit;
1809
1810 if (cfil_info == NULL) {
1811 return;
1812 }
1813
1814 if (report_frequency) {
1815 if (entry == NULL) {
1816 return;
1817 }
1818
1819 // Update stats reporting frequency.
1820 if (entry->cfe_stats_report_frequency != report_frequency) {
1821 entry->cfe_stats_report_frequency = report_frequency;
1822 if (entry->cfe_stats_report_frequency < CFIL_STATS_REPORT_INTERVAL_MIN_MSEC) {
1823 entry->cfe_stats_report_frequency = CFIL_STATS_REPORT_INTERVAL_MIN_MSEC;
1824 }
1825 microuptime(tv: &entry->cfe_stats_report_ts);
1826
1827 // Insert cfil_info into list only if it is not in yet.
1828 TAILQ_FOREACH(cfil, &cfil_sock_head_stats, cfi_link_stats) {
1829 if (cfil == cfil_info) {
1830 return;
1831 }
1832 }
1833
1834 TAILQ_INSERT_TAIL(&cfil_sock_head_stats, cfil_info, cfi_link_stats);
1835
1836 // Wake up stats thread if this is first flow added
1837 if (cfil_sock_attached_stats_count == 0) {
1838 thread_wakeup((caddr_t)&cfil_sock_attached_stats_count);
1839 }
1840 cfil_sock_attached_stats_count++;
1841
1842 if (cfil_info->cfi_debug && cfil_log_stats) {
1843 CFIL_LOG(LOG_ERR, "CFIL: VERDICT RECEIVED - STATS FLOW INSERTED: <so %llx sockID %llu <%llx>> stats frequency %d msecs",
1844 cfil_info->cfi_so ? (uint64_t)VM_KERNEL_ADDRPERM(cfil_info->cfi_so) : 0,
1845 cfil_info->cfi_sock_id, cfil_info->cfi_sock_id,
1846 entry->cfe_stats_report_frequency);
1847 }
1848 }
1849 } else {
1850 // Turn off stats reporting for this filter.
1851 if (entry != NULL) {
1852 // Already off, no change.
1853 if (entry->cfe_stats_report_frequency == 0) {
1854 return;
1855 }
1856
1857 entry->cfe_stats_report_frequency = 0;
1858 // If cfil_info still has filter(s) asking for stats, no need to remove from list.
1859 for (kcunit = 1; kcunit <= MAX_CONTENT_FILTER; kcunit++) {
1860 if (cfil_info->cfi_entries[kcunit - 1].cfe_stats_report_frequency > 0) {
1861 return;
1862 }
1863 }
1864 }
1865
1866 // No more filter asking for stats for this cfil_info, remove from list.
1867 if (!TAILQ_EMPTY(&cfil_sock_head_stats)) {
1868 found = FALSE;
1869 TAILQ_FOREACH(cfil, &cfil_sock_head_stats, cfi_link_stats) {
1870 if (cfil == cfil_info) {
1871 found = TRUE;
1872 break;
1873 }
1874 }
1875 if (found) {
1876 cfil_sock_attached_stats_count--;
1877 TAILQ_REMOVE(&cfil_sock_head_stats, cfil_info, cfi_link_stats);
1878 if (cfil_info->cfi_debug && cfil_log_stats) {
1879 CFIL_LOG(LOG_ERR, "CFIL: VERDICT RECEIVED - STATS FLOW DELETED: <so %llx sockID %llu <%llx>> stats frequency reset",
1880 cfil_info->cfi_so ? (uint64_t)VM_KERNEL_ADDRPERM(cfil_info->cfi_so) : 0,
1881 cfil_info->cfi_sock_id, cfil_info->cfi_sock_id);
1882 }
1883 }
1884 }
1885 }
1886}
1887
1888static errno_t
1889cfil_ctl_send(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo, mbuf_t m,
1890 int flags)
1891{
1892#pragma unused(kctlref, flags)
1893 errno_t error = 0;
1894 struct cfil_msg_hdr *msghdr;
1895 struct content_filter *cfc = (struct content_filter *)unitinfo;
1896 struct socket *so;
1897 struct cfil_msg_action *action_msg;
1898 struct cfil_entry *entry;
1899 struct cfil_info *cfil_info = NULL;
1900 unsigned int data_len = 0;
1901
1902 CFIL_LOG(LOG_INFO, "");
1903
1904 if (cfc == NULL) {
1905 CFIL_LOG(LOG_ERR, "no unitinfo");
1906 error = EINVAL;
1907 goto done;
1908 }
1909
1910 if (kcunit > MAX_CONTENT_FILTER) {
1911 CFIL_LOG(LOG_ERR, "kcunit %u > MAX_CONTENT_FILTER (%d)",
1912 kcunit, MAX_CONTENT_FILTER);
1913 error = EINVAL;
1914 goto done;
1915 }
1916 if (m == NULL) {
1917 CFIL_LOG(LOG_ERR, "null mbuf");
1918 error = EINVAL;
1919 goto done;
1920 }
1921 data_len = m_length(m);
1922
1923 if (data_len < sizeof(struct cfil_msg_hdr)) {
1924 CFIL_LOG(LOG_ERR, "too short %u", data_len);
1925 error = EINVAL;
1926 goto done;
1927 }
1928 msghdr = (struct cfil_msg_hdr *)mbuf_data(mbuf: m);
1929 if (msghdr->cfm_version != CFM_VERSION_CURRENT) {
1930 CFIL_LOG(LOG_ERR, "bad version %u", msghdr->cfm_version);
1931 error = EINVAL;
1932 goto done;
1933 }
1934 if (msghdr->cfm_type != CFM_TYPE_ACTION) {
1935 CFIL_LOG(LOG_ERR, "bad type %u", msghdr->cfm_type);
1936 error = EINVAL;
1937 goto done;
1938 }
1939 if (msghdr->cfm_len > data_len) {
1940 CFIL_LOG(LOG_ERR, "bad length %u", msghdr->cfm_len);
1941 error = EINVAL;
1942 goto done;
1943 }
1944
1945 /* Validate action operation */
1946 switch (msghdr->cfm_op) {
1947 case CFM_OP_DATA_UPDATE:
1948 OSIncrementAtomic(
1949 &cfil_stats.cfs_ctl_action_data_update);
1950 break;
1951 case CFM_OP_DROP:
1952 OSIncrementAtomic(&cfil_stats.cfs_ctl_action_drop);
1953 break;
1954 case CFM_OP_BLESS_CLIENT:
1955 if (msghdr->cfm_len != sizeof(struct cfil_msg_bless_client)) {
1956 OSIncrementAtomic(&cfil_stats.cfs_ctl_action_bad_len);
1957 error = EINVAL;
1958 CFIL_LOG(LOG_ERR, "bad len: %u for op %u",
1959 msghdr->cfm_len,
1960 msghdr->cfm_op);
1961 goto done;
1962 }
1963 error = cfil_action_bless_client(kcunit, msghdr);
1964 goto done;
1965 case CFM_OP_SET_CRYPTO_KEY:
1966 if (msghdr->cfm_len != sizeof(struct cfil_msg_set_crypto_key)) {
1967 OSIncrementAtomic(&cfil_stats.cfs_ctl_action_bad_len);
1968 error = EINVAL;
1969 CFIL_LOG(LOG_ERR, "bad len: %u for op %u",
1970 msghdr->cfm_len,
1971 msghdr->cfm_op);
1972 goto done;
1973 }
1974 error = cfil_action_set_crypto_key(kcunit, msghdr);
1975 goto done;
1976 default:
1977 OSIncrementAtomic(&cfil_stats.cfs_ctl_action_bad_op);
1978 CFIL_LOG(LOG_ERR, "bad op %u", msghdr->cfm_op);
1979 error = EINVAL;
1980 goto done;
1981 }
1982 if (msghdr->cfm_len != sizeof(struct cfil_msg_action)) {
1983 OSIncrementAtomic(&cfil_stats.cfs_ctl_action_bad_len);
1984 error = EINVAL;
1985 CFIL_LOG(LOG_ERR, "bad len: %u for op %u",
1986 msghdr->cfm_len,
1987 msghdr->cfm_op);
1988 goto done;
1989 }
1990 cfil_rw_lock_shared(lck: &cfil_lck_rw);
1991 if (cfc != (void *)content_filters[kcunit - 1]) {
1992 CFIL_LOG(LOG_ERR, "unitinfo does not match for kcunit %u",
1993 kcunit);
1994 error = EINVAL;
1995 cfil_rw_unlock_shared(lck: &cfil_lck_rw);
1996 goto done;
1997 }
1998 cfil_rw_unlock_shared(lck: &cfil_lck_rw);
1999
2000 // Search for socket (TCP+UDP and lock so)
2001 so = cfil_socket_from_sock_id(cfil_sock_id: msghdr->cfm_sock_id, false);
2002 if (so == NULL) {
2003 CFIL_LOG(LOG_NOTICE, "bad sock_id %llx",
2004 msghdr->cfm_sock_id);
2005 error = EINVAL;
2006 goto done;
2007 }
2008
2009 cfil_info = so->so_flow_db != NULL ?
2010 soflow_db_get_feature_context(so->so_flow_db, msghdr->cfm_sock_id) : so->so_cfil;
2011
2012 // We should not obtain global lock here in order to avoid deadlock down the path.
2013 // But we attempt to retain a valid cfil_info to prevent any deallocation until
2014 // we are done. Abort retain if cfil_info has already entered the free code path.
2015 if (cfil_info && os_ref_retain_try(rc: &cfil_info->cfi_ref_count) == false) {
2016 socket_unlock(so, refcount: 1);
2017 goto done;
2018 }
2019
2020 if (cfil_info == NULL) {
2021 CFIL_LOG(LOG_NOTICE, "so %llx <id %llu> not attached",
2022 (uint64_t)VM_KERNEL_ADDRPERM(so), msghdr->cfm_sock_id);
2023 error = EINVAL;
2024 goto unlock;
2025 } else if (cfil_info->cfi_flags & CFIF_DROP) {
2026 CFIL_LOG(LOG_NOTICE, "so %llx drop set",
2027 (uint64_t)VM_KERNEL_ADDRPERM(so));
2028 error = EINVAL;
2029 goto unlock;
2030 }
2031
2032 if (cfil_info->cfi_debug) {
2033 cfil_info_log(LOG_ERR, cfil_info, "CFIL: RECEIVED MSG FROM FILTER");
2034 }
2035
2036 entry = &cfil_info->cfi_entries[kcunit - 1];
2037 if (entry->cfe_filter == NULL) {
2038 CFIL_LOG(LOG_NOTICE, "so %llx no filter",
2039 (uint64_t)VM_KERNEL_ADDRPERM(so));
2040 error = EINVAL;
2041 goto unlock;
2042 }
2043
2044 if (entry->cfe_flags & CFEF_SENT_SOCK_ATTACHED) {
2045 entry->cfe_flags |= CFEF_DATA_START;
2046 } else {
2047 CFIL_LOG(LOG_ERR,
2048 "so %llx attached not sent for %u",
2049 (uint64_t)VM_KERNEL_ADDRPERM(so), kcunit);
2050 error = EINVAL;
2051 goto unlock;
2052 }
2053
2054 microuptime(tv: &entry->cfe_last_action);
2055 CFI_ADD_TIME_LOG(cfil_info, &entry->cfe_last_action, &cfil_info->cfi_first_event, msghdr->cfm_op);
2056
2057 action_msg = (struct cfil_msg_action *)msghdr;
2058
2059 switch (msghdr->cfm_op) {
2060 case CFM_OP_DATA_UPDATE:
2061
2062 if (cfil_info->cfi_debug) {
2063 cfil_info_log(LOG_ERR, cfil_info, "CFIL: RECEIVED CFM_OP_DATA_UPDATE");
2064 CFIL_LOG(LOG_ERR, "CFIL: VERDICT RECEIVED: <so %llx sockID %llu <%llx>> <IN peek:%llu pass:%llu, OUT peek:%llu pass:%llu>",
2065 (uint64_t)VM_KERNEL_ADDRPERM(so),
2066 cfil_info->cfi_sock_id, cfil_info->cfi_sock_id,
2067 action_msg->cfa_in_peek_offset, action_msg->cfa_in_pass_offset,
2068 action_msg->cfa_out_peek_offset, action_msg->cfa_out_pass_offset);
2069 }
2070
2071 /*
2072 * Received verdict, at this point we know this
2073 * socket connection is allowed. Unblock thread
2074 * immediately before proceeding to process the verdict.
2075 */
2076 cfil_sock_received_verdict(so);
2077
2078 if (action_msg->cfa_out_peek_offset != 0 ||
2079 action_msg->cfa_out_pass_offset != 0) {
2080 error = cfil_action_data_pass(so, cfil_info, kcunit, 1,
2081 action_msg->cfa_out_pass_offset,
2082 action_msg->cfa_out_peek_offset);
2083 }
2084 if (error == EJUSTRETURN) {
2085 error = 0;
2086 }
2087 if (error != 0) {
2088 break;
2089 }
2090 if (action_msg->cfa_in_peek_offset != 0 ||
2091 action_msg->cfa_in_pass_offset != 0) {
2092 error = cfil_action_data_pass(so, cfil_info, kcunit, 0,
2093 action_msg->cfa_in_pass_offset,
2094 action_msg->cfa_in_peek_offset);
2095 }
2096 if (error == EJUSTRETURN) {
2097 error = 0;
2098 }
2099
2100 // Toggle stats reporting according to received verdict.
2101 cfil_rw_lock_exclusive(lck: &cfil_lck_rw);
2102 cfil_info_stats_toggle(cfil_info, entry, report_frequency: action_msg->cfa_stats_frequency);
2103 cfil_rw_unlock_exclusive(lck: &cfil_lck_rw);
2104
2105 break;
2106
2107 case CFM_OP_DROP:
2108 if (cfil_info->cfi_debug) {
2109 cfil_info_log(LOG_ERR, cfil_info, "CFIL: RECEIVED CFM_OP_DROP");
2110 CFIL_LOG(LOG_ERR, "CFIL: VERDICT DROP RECEIVED: <so %llx sockID %llu <%llx>> <IN peek:%llu pass:%llu, OUT peek:%llu pass:%llu>",
2111 (uint64_t)VM_KERNEL_ADDRPERM(so),
2112 cfil_info->cfi_sock_id, cfil_info->cfi_sock_id,
2113 action_msg->cfa_in_peek_offset, action_msg->cfa_in_pass_offset,
2114 action_msg->cfa_out_peek_offset, action_msg->cfa_out_pass_offset);
2115 }
2116
2117 error = cfil_action_drop(so, cfil_info, kcunit);
2118 cfil_sock_received_verdict(so);
2119 break;
2120
2121 default:
2122 error = EINVAL;
2123 break;
2124 }
2125unlock:
2126 CFIL_INFO_FREE(cfil_info)
2127 socket_unlock(so, refcount: 1);
2128done:
2129 mbuf_freem(mbuf: m);
2130
2131 if (error == 0) {
2132 OSIncrementAtomic(&cfil_stats.cfs_ctl_send_ok);
2133 } else {
2134 OSIncrementAtomic(&cfil_stats.cfs_ctl_send_bad);
2135 }
2136
2137 return error;
2138}
2139
2140static errno_t
2141cfil_ctl_getopt(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo,
2142 int opt, void *data, size_t *len)
2143{
2144#pragma unused(kctlref, opt)
2145 struct cfil_info *cfil_info = NULL;
2146 errno_t error = 0;
2147 struct content_filter *cfc = (struct content_filter *)unitinfo;
2148
2149 CFIL_LOG(LOG_NOTICE, "");
2150
2151 if (cfc == NULL) {
2152 CFIL_LOG(LOG_ERR, "no unitinfo");
2153 return EINVAL;
2154 }
2155
2156 cfil_rw_lock_shared(lck: &cfil_lck_rw);
2157
2158 if (kcunit > MAX_CONTENT_FILTER) {
2159 CFIL_LOG(LOG_ERR, "kcunit %u > MAX_CONTENT_FILTER (%d)",
2160 kcunit, MAX_CONTENT_FILTER);
2161 error = EINVAL;
2162 goto done;
2163 }
2164 if (cfc != (void *)content_filters[kcunit - 1]) {
2165 CFIL_LOG(LOG_ERR, "unitinfo does not match for kcunit %u",
2166 kcunit);
2167 error = EINVAL;
2168 goto done;
2169 }
2170 switch (opt) {
2171 case CFIL_OPT_NECP_CONTROL_UNIT:
2172 if (*len < sizeof(uint32_t)) {
2173 CFIL_LOG(LOG_ERR, "len too small %lu", *len);
2174 error = EINVAL;
2175 goto done;
2176 }
2177 if (data != NULL) {
2178 *(uint32_t *)data = cfc->cf_necp_control_unit;
2179 }
2180 break;
2181 case CFIL_OPT_PRESERVE_CONNECTIONS:
2182 if (*len < sizeof(uint32_t)) {
2183 CFIL_LOG(LOG_ERR, "CFIL_OPT_PRESERVE_CONNECTIONS len too small %lu", *len);
2184 error = EINVAL;
2185 goto done;
2186 }
2187 if (data != NULL) {
2188 *(uint32_t *)data = (cfc->cf_flags & CFF_PRESERVE_CONNECTIONS) ? true : false;
2189 }
2190 break;
2191 case CFIL_OPT_GET_SOCKET_INFO:
2192 if (*len != sizeof(struct cfil_opt_sock_info)) {
2193 CFIL_LOG(LOG_ERR, "len does not match %lu", *len);
2194 error = EINVAL;
2195 goto done;
2196 }
2197 if (data == NULL) {
2198 CFIL_LOG(LOG_ERR, "data not passed");
2199 error = EINVAL;
2200 goto done;
2201 }
2202
2203 struct cfil_opt_sock_info *sock_info =
2204 (struct cfil_opt_sock_info *) data;
2205
2206 // Unlock here so that we never hold both cfil_lck_rw and the
2207 // socket_lock at the same time. Otherwise, this can deadlock
2208 // because soclose() takes the socket_lock and then exclusive
2209 // cfil_lck_rw and we require the opposite order.
2210
2211 // WARNING: Be sure to never use anything protected
2212 // by cfil_lck_rw beyond this point.
2213 // WARNING: Be sure to avoid fallthrough and
2214 // goto return_already_unlocked from this branch.
2215 cfil_rw_unlock_shared(lck: &cfil_lck_rw);
2216
2217 // Search (TCP+UDP) and lock socket
2218 struct socket *sock =
2219 cfil_socket_from_sock_id(cfil_sock_id: sock_info->cfs_sock_id, false);
2220 if (sock == NULL) {
2221 CFIL_LOG(LOG_ERR, "CFIL: GET_SOCKET_INFO failed: bad sock_id %llu",
2222 sock_info->cfs_sock_id);
2223 error = ENOENT;
2224 goto return_already_unlocked;
2225 }
2226
2227 cfil_info = (sock->so_flow_db != NULL) ?
2228 soflow_db_get_feature_context(sock->so_flow_db, sock_info->cfs_sock_id) : sock->so_cfil;
2229
2230 if (cfil_info == NULL) {
2231 CFIL_LOG(LOG_INFO, "CFIL: GET_SOCKET_INFO failed: so %llx not attached, cannot fetch info",
2232 (uint64_t)VM_KERNEL_ADDRPERM(sock));
2233 error = EINVAL;
2234 socket_unlock(so: sock, refcount: 1);
2235 goto return_already_unlocked;
2236 }
2237
2238 if (sock->so_proto == NULL || sock->so_proto->pr_domain == NULL) {
2239 CFIL_LOG(LOG_INFO, "CFIL: GET_SOCKET_INFO failed: so %llx NULL so_proto / pr_domain",
2240 (uint64_t)VM_KERNEL_ADDRPERM(sock));
2241 error = EINVAL;
2242 socket_unlock(so: sock, refcount: 1);
2243 goto return_already_unlocked;
2244 }
2245
2246 // Fill out family, type, and protocol
2247 sock_info->cfs_sock_family = SOCK_DOM(sock);
2248 sock_info->cfs_sock_type = SOCK_TYPE(sock);
2249 sock_info->cfs_sock_protocol = GET_SO_PROTO(sock);
2250
2251 // Source and destination addresses
2252 struct inpcb *inp = sotoinpcb(sock);
2253 if (inp->inp_vflag & INP_IPV6) {
2254 struct in6_addr *laddr = NULL, *faddr = NULL;
2255 u_int16_t lport = 0, fport = 0;
2256
2257 cfil_get_flow_address_v6(cfil_info->cfi_hash_entry, inp,
2258 &laddr, &faddr, &lport, &fport);
2259 fill_ip6_sockaddr_4_6(&sock_info->cfs_local, laddr, lport, inp->inp_lifscope);
2260 fill_ip6_sockaddr_4_6(&sock_info->cfs_remote, faddr, fport, inp->inp_fifscope);
2261 } else if (inp->inp_vflag & INP_IPV4) {
2262 struct in_addr laddr = {.s_addr = 0}, faddr = {.s_addr = 0};
2263 u_int16_t lport = 0, fport = 0;
2264
2265 cfil_get_flow_address(cfil_info->cfi_hash_entry, inp,
2266 &laddr, &faddr, &lport, &fport);
2267 fill_ip_sockaddr_4_6(&sock_info->cfs_local, laddr, lport);
2268 fill_ip_sockaddr_4_6(&sock_info->cfs_remote, faddr, fport);
2269 }
2270
2271 // Set the pid info
2272 sock_info->cfs_pid = sock->last_pid;
2273 memcpy(dst: sock_info->cfs_uuid, src: sock->last_uuid, n: sizeof(uuid_t));
2274
2275 if (sock->so_flags & SOF_DELEGATED) {
2276 sock_info->cfs_e_pid = sock->e_pid;
2277 memcpy(dst: sock_info->cfs_e_uuid, src: sock->e_uuid, n: sizeof(uuid_t));
2278 } else {
2279 sock_info->cfs_e_pid = sock->last_pid;
2280 memcpy(dst: sock_info->cfs_e_uuid, src: sock->last_uuid, n: sizeof(uuid_t));
2281 }
2282
2283 socket_unlock(so: sock, refcount: 1);
2284
2285 goto return_already_unlocked;
2286 default:
2287 error = ENOPROTOOPT;
2288 break;
2289 }
2290done:
2291 cfil_rw_unlock_shared(lck: &cfil_lck_rw);
2292
2293 return error;
2294
2295return_already_unlocked:
2296
2297 return error;
2298}
2299
2300static errno_t
2301cfil_ctl_setopt(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo,
2302 int opt, void *data, size_t len)
2303{
2304#pragma unused(kctlref, opt)
2305 errno_t error = 0;
2306 struct content_filter *cfc = (struct content_filter *)unitinfo;
2307
2308 CFIL_LOG(LOG_NOTICE, "");
2309
2310 if (cfc == NULL) {
2311 CFIL_LOG(LOG_ERR, "no unitinfo");
2312 return EINVAL;
2313 }
2314
2315 cfil_rw_lock_exclusive(lck: &cfil_lck_rw);
2316
2317 if (kcunit > MAX_CONTENT_FILTER) {
2318 CFIL_LOG(LOG_ERR, "kcunit %u > MAX_CONTENT_FILTER (%d)",
2319 kcunit, MAX_CONTENT_FILTER);
2320 error = EINVAL;
2321 goto done;
2322 }
2323 if (cfc != (void *)content_filters[kcunit - 1]) {
2324 CFIL_LOG(LOG_ERR, "unitinfo does not match for kcunit %u",
2325 kcunit);
2326 error = EINVAL;
2327 goto done;
2328 }
2329 switch (opt) {
2330 case CFIL_OPT_NECP_CONTROL_UNIT:
2331 if (len < sizeof(uint32_t)) {
2332 CFIL_LOG(LOG_ERR, "CFIL_OPT_NECP_CONTROL_UNIT "
2333 "len too small %lu", len);
2334 error = EINVAL;
2335 goto done;
2336 }
2337 if (cfc->cf_necp_control_unit != 0) {
2338 CFIL_LOG(LOG_ERR, "CFIL_OPT_NECP_CONTROL_UNIT "
2339 "already set %u",
2340 cfc->cf_necp_control_unit);
2341 error = EINVAL;
2342 goto done;
2343 }
2344 cfc->cf_necp_control_unit = *(uint32_t *)data;
2345 break;
2346 case CFIL_OPT_PRESERVE_CONNECTIONS:
2347 if (len < sizeof(uint32_t)) {
2348 CFIL_LOG(LOG_ERR, "CFIL_OPT_PRESERVE_CONNECTIONS "
2349 "len too small %lu", len);
2350 error = EINVAL;
2351 goto done;
2352 }
2353 uint32_t preserve_connections = *((uint32_t *)data);
2354 CFIL_LOG(LOG_INFO, "CFIL_OPT_PRESERVE_CONNECTIONS got %d (kcunit %d)", preserve_connections, kcunit);
2355 if (preserve_connections) {
2356 cfc->cf_flags |= CFF_PRESERVE_CONNECTIONS;
2357 } else {
2358 cfc->cf_flags &= ~CFF_PRESERVE_CONNECTIONS;
2359 }
2360
2361 cfil_update_behavior_flags();
2362 break;
2363 default:
2364 error = ENOPROTOOPT;
2365 break;
2366 }
2367done:
2368 cfil_rw_unlock_exclusive(lck: &cfil_lck_rw);
2369
2370 return error;
2371}
2372
2373
2374static void
2375cfil_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo, int flags)
2376{
2377#pragma unused(kctlref, flags)
2378 struct content_filter *cfc = (struct content_filter *)unitinfo;
2379 struct socket *so = NULL;
2380 int error;
2381 struct cfil_entry *entry;
2382 struct cfil_info *cfil_info = NULL;
2383
2384 CFIL_LOG(LOG_INFO, "");
2385
2386 if (cfc == NULL) {
2387 CFIL_LOG(LOG_ERR, "no unitinfo");
2388 OSIncrementAtomic(&cfil_stats.cfs_ctl_rcvd_bad);
2389 return;
2390 }
2391
2392 if (kcunit > MAX_CONTENT_FILTER) {
2393 CFIL_LOG(LOG_ERR, "kcunit %u > MAX_CONTENT_FILTER (%d)",
2394 kcunit, MAX_CONTENT_FILTER);
2395 OSIncrementAtomic(&cfil_stats.cfs_ctl_rcvd_bad);
2396 return;
2397 }
2398 cfil_rw_lock_shared(lck: &cfil_lck_rw);
2399 if (cfc != (void *)content_filters[kcunit - 1]) {
2400 CFIL_LOG(LOG_ERR, "unitinfo does not match for kcunit %u",
2401 kcunit);
2402 OSIncrementAtomic(&cfil_stats.cfs_ctl_rcvd_bad);
2403 goto done;
2404 }
2405 /* Let's assume the flow control is lifted */
2406 if (cfc->cf_flags & CFF_FLOW_CONTROLLED) {
2407 if (!cfil_rw_lock_shared_to_exclusive(lck: &cfil_lck_rw)) {
2408 cfil_rw_lock_exclusive(lck: &cfil_lck_rw);
2409 }
2410
2411 cfc->cf_flags &= ~CFF_FLOW_CONTROLLED;
2412
2413 cfil_rw_lock_exclusive_to_shared(lck: &cfil_lck_rw);
2414 LCK_RW_ASSERT(&cfil_lck_rw, LCK_RW_ASSERT_SHARED);
2415 }
2416 /*
2417 * Flow control will be raised again as soon as an entry cannot enqueue
2418 * to the kernel control socket
2419 */
2420 while ((cfc->cf_flags & CFF_FLOW_CONTROLLED) == 0) {
2421 verify_content_filter(cfc);
2422
2423 cfil_rw_lock_assert_held(lck: &cfil_lck_rw, exclusive: 0);
2424
2425 /* Find an entry that is flow controlled */
2426 TAILQ_FOREACH(entry, &cfc->cf_sock_entries, cfe_link) {
2427 if (entry->cfe_cfil_info == NULL ||
2428 entry->cfe_cfil_info->cfi_so == NULL) {
2429 continue;
2430 }
2431 if ((entry->cfe_flags & CFEF_FLOW_CONTROLLED) == 0) {
2432 continue;
2433 }
2434 }
2435 if (entry == NULL) {
2436 break;
2437 }
2438
2439 OSIncrementAtomic(&cfil_stats.cfs_ctl_rcvd_flow_lift);
2440
2441 cfil_info = entry->cfe_cfil_info;
2442 so = cfil_info->cfi_so;
2443
2444 if (cfil_info == NULL || os_ref_retain_try(rc: &cfil_info->cfi_ref_count) == false) {
2445 break;
2446 }
2447
2448 cfil_rw_unlock_shared(lck: &cfil_lck_rw);
2449 socket_lock(so, refcount: 1);
2450
2451 do {
2452 error = cfil_acquire_sockbuf(so, cfil_info, outgoing: 1);
2453 if (error == 0) {
2454 error = cfil_data_service_ctl_q(so, cfil_info, kcunit, 1);
2455 }
2456 cfil_release_sockbuf(so, outgoing: 1);
2457 if (error != 0) {
2458 break;
2459 }
2460
2461 error = cfil_acquire_sockbuf(so, cfil_info, outgoing: 0);
2462 if (error == 0) {
2463 error = cfil_data_service_ctl_q(so, cfil_info, kcunit, 0);
2464 }
2465 cfil_release_sockbuf(so, outgoing: 0);
2466 } while (0);
2467
2468 CFIL_INFO_FREE(cfil_info);
2469 socket_lock_assert_owned(so);
2470 socket_unlock(so, refcount: 1);
2471
2472 cfil_rw_lock_shared(lck: &cfil_lck_rw);
2473 }
2474done:
2475 cfil_rw_unlock_shared(lck: &cfil_lck_rw);
2476}
2477
2478struct cflil_tag_container {
2479 struct m_tag cfil_m_tag;
2480 struct cfil_tag cfil_tag;
2481};
2482
2483static struct m_tag *
2484m_tag_kalloc_cfil_udp(u_int32_t id, u_int16_t type, uint16_t len, int wait)
2485{
2486 struct cflil_tag_container *tag_container;
2487 struct m_tag *tag = NULL;
2488
2489 assert3u(id, ==, KERNEL_MODULE_TAG_ID);
2490 assert3u(type, ==, KERNEL_TAG_TYPE_CFIL_UDP);
2491 assert3u(len, ==, sizeof(struct cfil_tag));
2492
2493 if (len != sizeof(struct cfil_tag)) {
2494 return NULL;
2495 }
2496
2497 tag_container = kalloc_type(struct cflil_tag_container, wait | M_ZERO);
2498 if (tag_container != NULL) {
2499 tag = &tag_container->cfil_m_tag;
2500
2501 assert3p(tag, ==, tag_container);
2502
2503 M_TAG_INIT(tag, id, type, len, &tag_container->cfil_tag, NULL);
2504 }
2505
2506 return tag;
2507}
2508
2509static void
2510m_tag_kfree_cfil_udp(struct m_tag *tag)
2511{
2512 struct cflil_tag_container *tag_container = (struct cflil_tag_container *)tag;
2513
2514 kfree_type(struct cflil_tag_container, tag_container);
2515}
2516
2517void
2518cfil_register_m_tag(void)
2519{
2520 errno_t error = 0;
2521
2522 error = m_register_internal_tag_type(type: KERNEL_TAG_TYPE_CFIL_UDP, len: sizeof(struct cfil_tag),
2523 alloc_func: m_tag_kalloc_cfil_udp, free_func: m_tag_kfree_cfil_udp);
2524
2525 assert3u(error, ==, 0);
2526}
2527
2528void
2529cfil_init(void)
2530{
2531 struct kern_ctl_reg kern_ctl;
2532 errno_t error = 0;
2533 unsigned int mbuf_limit = 0;
2534
2535 CFIL_LOG(LOG_NOTICE, "");
2536
2537 /*
2538 * Compile time verifications
2539 */
2540 _CASSERT(CFIL_MAX_FILTER_COUNT == MAX_CONTENT_FILTER);
2541 _CASSERT(sizeof(struct cfil_filter_stat) % sizeof(uint32_t) == 0);
2542 _CASSERT(sizeof(struct cfil_entry_stat) % sizeof(uint32_t) == 0);
2543 _CASSERT(sizeof(struct cfil_sock_stat) % sizeof(uint32_t) == 0);
2544
2545 /*
2546 * Runtime time verifications
2547 */
2548 VERIFY(IS_P2ALIGNED(&cfil_stats.cfs_ctl_q_in_enqueued,
2549 sizeof(uint32_t)));
2550 VERIFY(IS_P2ALIGNED(&cfil_stats.cfs_ctl_q_out_enqueued,
2551 sizeof(uint32_t)));
2552 VERIFY(IS_P2ALIGNED(&cfil_stats.cfs_ctl_q_in_peeked,
2553 sizeof(uint32_t)));
2554 VERIFY(IS_P2ALIGNED(&cfil_stats.cfs_ctl_q_out_peeked,
2555 sizeof(uint32_t)));
2556
2557 VERIFY(IS_P2ALIGNED(&cfil_stats.cfs_pending_q_in_enqueued,
2558 sizeof(uint32_t)));
2559 VERIFY(IS_P2ALIGNED(&cfil_stats.cfs_pending_q_out_enqueued,
2560 sizeof(uint32_t)));
2561
2562 VERIFY(IS_P2ALIGNED(&cfil_stats.cfs_inject_q_in_enqueued,
2563 sizeof(uint32_t)));
2564 VERIFY(IS_P2ALIGNED(&cfil_stats.cfs_inject_q_out_enqueued,
2565 sizeof(uint32_t)));
2566 VERIFY(IS_P2ALIGNED(&cfil_stats.cfs_inject_q_in_passed,
2567 sizeof(uint32_t)));
2568 VERIFY(IS_P2ALIGNED(&cfil_stats.cfs_inject_q_out_passed,
2569 sizeof(uint32_t)));
2570
2571 /*
2572 * Allocate locks
2573 */
2574 TAILQ_INIT(&cfil_sock_head);
2575 TAILQ_INIT(&cfil_sock_head_stats);
2576
2577 /*
2578 * Register kernel control
2579 */
2580 bzero(s: &kern_ctl, n: sizeof(kern_ctl));
2581 strlcpy(dst: kern_ctl.ctl_name, CONTENT_FILTER_CONTROL_NAME,
2582 n: sizeof(kern_ctl.ctl_name));
2583 kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED;
2584 kern_ctl.ctl_sendsize = 512 * 1024; /* enough? */
2585 kern_ctl.ctl_recvsize = 512 * 1024; /* enough? */
2586 kern_ctl.ctl_connect = cfil_ctl_connect;
2587 kern_ctl.ctl_disconnect = cfil_ctl_disconnect;
2588 kern_ctl.ctl_send = cfil_ctl_send;
2589 kern_ctl.ctl_getopt = cfil_ctl_getopt;
2590 kern_ctl.ctl_setopt = cfil_ctl_setopt;
2591 kern_ctl.ctl_rcvd = cfil_ctl_rcvd;
2592 error = ctl_register(userkctl: &kern_ctl, kctlref: &cfil_kctlref);
2593 if (error != 0) {
2594 CFIL_LOG(LOG_ERR, "ctl_register failed: %d", error);
2595 return;
2596 }
2597
2598 // Spawn thread for statistics reporting
2599 if (kernel_thread_start(continuation: cfil_stats_report_thread_func, NULL,
2600 new_thread: &cfil_stats_report_thread) != KERN_SUCCESS) {
2601 panic_plain("%s: Can't create statistics report thread", __func__);
2602 /* NOTREACHED */
2603 }
2604 /* this must not fail */
2605 VERIFY(cfil_stats_report_thread != NULL);
2606
2607 // Set UDP per-flow mbuf thresholds to 1/32 of platform max
2608 mbuf_limit = MAX(UDP_FLOW_GC_MBUF_CNT_MAX, (nmbclusters << MCLSHIFT) >> UDP_FLOW_GC_MBUF_SHIFT);
2609 cfil_udp_gc_mbuf_num_max = (mbuf_limit >> MCLSHIFT);
2610 cfil_udp_gc_mbuf_cnt_max = mbuf_limit;
2611
2612 memset(s: &global_cfil_stats_report_buffers, c: 0, n: sizeof(global_cfil_stats_report_buffers));
2613}
2614
2615struct cfil_info *
2616cfil_info_alloc(struct socket *so, struct soflow_hash_entry *hash_entry)
2617{
2618 int kcunit;
2619 struct cfil_info *cfil_info = NULL;
2620 struct inpcb *inp = sotoinpcb(so);
2621
2622 CFIL_LOG(LOG_INFO, "");
2623
2624 socket_lock_assert_owned(so);
2625
2626 cfil_info = zalloc_flags(cfil_info_zone, Z_WAITOK | Z_ZERO | Z_NOFAIL);
2627 os_ref_init(&cfil_info->cfi_ref_count, &cfil_refgrp);
2628
2629 cfil_queue_init(cfq: &cfil_info->cfi_snd.cfi_inject_q);
2630 cfil_queue_init(cfq: &cfil_info->cfi_rcv.cfi_inject_q);
2631
2632 for (kcunit = 1; kcunit <= MAX_CONTENT_FILTER; kcunit++) {
2633 struct cfil_entry *entry;
2634
2635 entry = &cfil_info->cfi_entries[kcunit - 1];
2636 entry->cfe_cfil_info = cfil_info;
2637
2638 /* Initialize the filter entry */
2639 entry->cfe_filter = NULL;
2640 entry->cfe_flags = 0;
2641 entry->cfe_necp_control_unit = 0;
2642 entry->cfe_snd.cfe_pass_offset = 0;
2643 entry->cfe_snd.cfe_peek_offset = 0;
2644 entry->cfe_snd.cfe_peeked = 0;
2645 entry->cfe_rcv.cfe_pass_offset = 0;
2646 entry->cfe_rcv.cfe_peek_offset = 0;
2647 entry->cfe_rcv.cfe_peeked = 0;
2648 /*
2649 * Timestamp the last action to avoid pre-maturely
2650 * triggering garbage collection
2651 */
2652 microuptime(tv: &entry->cfe_last_action);
2653
2654 cfil_queue_init(cfq: &entry->cfe_snd.cfe_pending_q);
2655 cfil_queue_init(cfq: &entry->cfe_rcv.cfe_pending_q);
2656 cfil_queue_init(cfq: &entry->cfe_snd.cfe_ctl_q);
2657 cfil_queue_init(cfq: &entry->cfe_rcv.cfe_ctl_q);
2658 }
2659
2660 cfil_rw_lock_exclusive(lck: &cfil_lck_rw);
2661
2662 /*
2663 * Create a cfi_sock_id that's not the socket pointer!
2664 */
2665
2666 if (hash_entry == NULL) {
2667 // This is the TCP case, cfil_info is tracked per socket
2668 if (inp->inp_flowhash == 0) {
2669 inp_calc_flowhash(inp);
2670 ASSERT(inp->inp_flowhash != 0);
2671 }
2672
2673 so->so_cfil = cfil_info;
2674 cfil_info->cfi_so = so;
2675 cfil_info->cfi_sock_id =
2676 ((so->so_gencnt << 32) | inp->inp_flowhash);
2677 } else {
2678 // This is the UDP case, cfil_info is tracked in per-socket hash
2679 cfil_info->cfi_so = so;
2680 cfil_info->cfi_hash_entry = hash_entry;
2681 cfil_info->cfi_sock_id = ((so->so_gencnt << 32) | (hash_entry->soflow_flowhash & 0xffffffff));
2682 }
2683
2684 TAILQ_INSERT_TAIL(&cfil_sock_head, cfil_info, cfi_link);
2685 SLIST_INIT(&cfil_info->cfi_ordered_entries);
2686
2687 cfil_sock_attached_count++;
2688
2689 cfil_rw_unlock_exclusive(lck: &cfil_lck_rw);
2690
2691 if (cfil_info != NULL) {
2692 OSIncrementAtomic(&cfil_stats.cfs_cfi_alloc_ok);
2693 } else {
2694 OSIncrementAtomic(&cfil_stats.cfs_cfi_alloc_fail);
2695 }
2696
2697 return cfil_info;
2698}
2699
2700int
2701cfil_info_attach_unit(struct socket *so, uint32_t filter_control_unit, struct cfil_info *cfil_info)
2702{
2703 int kcunit;
2704 int attached = 0;
2705
2706 CFIL_LOG(LOG_INFO, "");
2707
2708 socket_lock_assert_owned(so);
2709
2710 cfil_rw_lock_exclusive(lck: &cfil_lck_rw);
2711
2712 for (kcunit = 1; kcunit <= MAX_CONTENT_FILTER; kcunit++) {
2713 struct content_filter *cfc = content_filters[kcunit - 1];
2714 struct cfil_entry *entry;
2715 struct cfil_entry *iter_entry;
2716 struct cfil_entry *iter_prev;
2717
2718 if (cfc == NULL) {
2719 continue;
2720 }
2721 if (!(cfc->cf_necp_control_unit & filter_control_unit)) {
2722 continue;
2723 }
2724
2725 entry = &cfil_info->cfi_entries[kcunit - 1];
2726
2727 entry->cfe_filter = cfc;
2728 entry->cfe_necp_control_unit = cfc->cf_necp_control_unit;
2729 TAILQ_INSERT_TAIL(&cfc->cf_sock_entries, entry, cfe_link);
2730 cfc->cf_sock_count++;
2731
2732 /* Insert the entry into the list ordered by control unit */
2733 iter_prev = NULL;
2734 SLIST_FOREACH(iter_entry, &cfil_info->cfi_ordered_entries, cfe_order_link) {
2735 if (entry->cfe_necp_control_unit < iter_entry->cfe_necp_control_unit) {
2736 break;
2737 }
2738 iter_prev = iter_entry;
2739 }
2740
2741 if (iter_prev == NULL) {
2742 SLIST_INSERT_HEAD(&cfil_info->cfi_ordered_entries, entry, cfe_order_link);
2743 } else {
2744 SLIST_INSERT_AFTER(iter_prev, entry, cfe_order_link);
2745 }
2746
2747 verify_content_filter(cfc);
2748 attached = 1;
2749 entry->cfe_flags |= CFEF_CFIL_ATTACHED;
2750 }
2751
2752 cfil_rw_unlock_exclusive(lck: &cfil_lck_rw);
2753
2754 return attached;
2755}
2756
2757static void
2758cfil_info_free(struct cfil_info *cfil_info)
2759{
2760 int kcunit;
2761 uint64_t in_drain = 0;
2762 uint64_t out_drained = 0;
2763
2764 if (cfil_info == NULL) {
2765 return;
2766 }
2767
2768 CFIL_LOG(LOG_INFO, "");
2769
2770 cfil_rw_lock_exclusive(lck: &cfil_lck_rw);
2771
2772 if (cfil_info->cfi_debug) {
2773 cfil_info_log(LOG_ERR, cfil_info, "CFIL: FREEING CFIL_INFO");
2774 }
2775
2776 for (kcunit = 1; kcunit <= MAX_CONTENT_FILTER; kcunit++) {
2777 struct cfil_entry *entry;
2778 struct content_filter *cfc;
2779
2780 entry = &cfil_info->cfi_entries[kcunit - 1];
2781
2782 /* Don't be silly and try to detach twice */
2783 if (entry->cfe_filter == NULL) {
2784 continue;
2785 }
2786
2787 cfc = content_filters[kcunit - 1];
2788
2789 VERIFY(cfc == entry->cfe_filter);
2790
2791 entry->cfe_filter = NULL;
2792 entry->cfe_necp_control_unit = 0;
2793 TAILQ_REMOVE(&cfc->cf_sock_entries, entry, cfe_link);
2794 cfc->cf_sock_count--;
2795
2796 verify_content_filter(cfc);
2797 }
2798
2799 cfil_sock_attached_count--;
2800 TAILQ_REMOVE(&cfil_sock_head, cfil_info, cfi_link);
2801
2802 // Turn off stats reporting for cfil_info.
2803 cfil_info_stats_toggle(cfil_info, NULL, report_frequency: 0);
2804
2805 out_drained += cfil_queue_drain(cfq: &cfil_info->cfi_snd.cfi_inject_q);
2806 in_drain += cfil_queue_drain(cfq: &cfil_info->cfi_rcv.cfi_inject_q);
2807
2808 for (kcunit = 1; kcunit <= MAX_CONTENT_FILTER; kcunit++) {
2809 struct cfil_entry *entry;
2810
2811 entry = &cfil_info->cfi_entries[kcunit - 1];
2812 out_drained += cfil_queue_drain(cfq: &entry->cfe_snd.cfe_pending_q);
2813 in_drain += cfil_queue_drain(cfq: &entry->cfe_rcv.cfe_pending_q);
2814 out_drained += cfil_queue_drain(cfq: &entry->cfe_snd.cfe_ctl_q);
2815 in_drain += cfil_queue_drain(cfq: &entry->cfe_rcv.cfe_ctl_q);
2816 }
2817 cfil_rw_unlock_exclusive(lck: &cfil_lck_rw);
2818
2819 if (out_drained) {
2820 OSIncrementAtomic(&cfil_stats.cfs_flush_out_free);
2821 }
2822 if (in_drain) {
2823 OSIncrementAtomic(&cfil_stats.cfs_flush_in_free);
2824 }
2825
2826 zfree(cfil_info_zone, cfil_info);
2827}
2828
2829/*
2830 * Received a verdict from userspace for a socket.
2831 * Perform any delayed operation if needed.
2832 */
2833static void
2834cfil_sock_received_verdict(struct socket *so)
2835{
2836 if (so == NULL || so->so_cfil == NULL) {
2837 return;
2838 }
2839
2840 so->so_cfil->cfi_flags |= CFIF_INITIAL_VERDICT;
2841
2842 /*
2843 * If socket has already been connected, trigger
2844 * soisconnected now.
2845 */
2846 if (so->so_cfil->cfi_flags & CFIF_SOCKET_CONNECTED) {
2847 so->so_cfil->cfi_flags &= ~CFIF_SOCKET_CONNECTED;
2848 soisconnected(so);
2849 return;
2850 }
2851}
2852
2853/*
2854 * Entry point from Sockets layer
2855 * The socket is locked.
2856 *
2857 * Checks if a connected socket is subject to filter and
2858 * pending the initial verdict.
2859 */
2860boolean_t
2861cfil_sock_connected_pending_verdict(struct socket *so)
2862{
2863 if (so == NULL || so->so_cfil == NULL) {
2864 return false;
2865 }
2866
2867 if (so->so_cfil->cfi_flags & CFIF_INITIAL_VERDICT) {
2868 return false;
2869 } else {
2870 /*
2871 * Remember that this protocol is already connected, so
2872 * we will trigger soisconnected() upon receipt of
2873 * initial verdict later.
2874 */
2875 so->so_cfil->cfi_flags |= CFIF_SOCKET_CONNECTED;
2876 return true;
2877 }
2878}
2879
2880/*
2881 * Entry point from Flow Divert
2882 * The socket is locked.
2883 *
2884 * Mark socket as DEAD if all CFIL data has been processed by filter(s).
2885 * Otherwise, delay the marking until all data has been processed.
2886 */
2887boolean_t
2888cfil_sock_is_dead(struct socket *so)
2889{
2890 struct inpcb *inp = NULL;
2891
2892 if (so == NULL) {
2893 return false;
2894 }
2895
2896 socket_lock_assert_owned(so);
2897
2898 if ((so->so_flags & SOF_CONTENT_FILTER) != 0) {
2899 int32_t pending_snd = cfil_sock_data_pending(sb: &so->so_snd);
2900 int32_t pending_rcv = cfil_sock_data_pending(sb: &so->so_rcv);
2901 if (pending_snd || pending_rcv) {
2902 SO_DELAYED_DEAD_SET(so, true)
2903 return false;
2904 }
2905 }
2906
2907 inp = sotoinpcb(so);
2908 if (inp != NULL) {
2909 inp->inp_state = INPCB_STATE_DEAD;
2910 inpcb_gc_sched(inp->inp_pcbinfo, type: INPCB_TIMER_FAST);
2911 SO_DELAYED_DEAD_SET(so, false)
2912 return true;
2913 }
2914 return false;
2915}
2916
2917/*
2918 * Entry point from tcp_timer.c
2919 * The socket is locked.
2920 *
2921 * Perform TCP FIN time wait handling if all CFIL data has been processed by filter(s).
2922 * Otherwise, delay until all data has been processed.
2923 */
2924boolean_t
2925cfil_sock_tcp_add_time_wait(struct socket *so)
2926{
2927 struct inpcb *inp = NULL;
2928 struct tcpcb *tp = NULL;
2929
2930 // Only handle TCP sockets
2931 if (so == NULL || !IS_TCP(so)) {
2932 return false;
2933 }
2934
2935 socket_lock_assert_owned(so);
2936
2937 if ((so->so_flags & SOF_CONTENT_FILTER) != 0) {
2938 int32_t pending_snd = cfil_sock_data_pending(sb: &so->so_snd);
2939 int32_t pending_rcv = cfil_sock_data_pending(sb: &so->so_rcv);
2940 if (pending_snd || pending_rcv) {
2941 SO_DELAYED_TCP_TIME_WAIT_SET(so, true)
2942 return false;
2943 }
2944 }
2945
2946 inp = sotoinpcb(so);
2947 tp = inp ? intotcpcb(inp) : NULL;
2948 if (tp != NULL) {
2949 add_to_time_wait_now(tp, delay: 2 * tcp_msl);
2950 SO_DELAYED_TCP_TIME_WAIT_SET(so, false)
2951 return true;
2952 }
2953 return false;
2954}
2955
2956boolean_t
2957cfil_filter_present(void)
2958{
2959 return cfil_active_count > 0;
2960}
2961
2962/*
2963 * Entry point from Sockets layer
2964 * The socket is locked.
2965 */
2966errno_t
2967cfil_sock_attach(struct socket *so, struct sockaddr *local, struct sockaddr *remote, int dir)
2968{
2969 errno_t error = 0;
2970 uint32_t filter_control_unit;
2971 int debug = 0;
2972
2973 socket_lock_assert_owned(so);
2974
2975 if (so->so_flags1 & SOF1_FLOW_DIVERT_SKIP) {
2976 /*
2977 * This socket has already been evaluated (and ultimately skipped) by
2978 * flow divert, so it has also already been through content filter if there
2979 * is one.
2980 */
2981 goto done;
2982 }
2983
2984 /* Limit ourselves to TCP that are not MPTCP subflows */
2985 if (SKIP_FILTER_FOR_TCP_SOCKET(so)) {
2986 goto done;
2987 }
2988
2989 debug = DEBUG_FLOW(sotoinpcb(so), so, local, remote);
2990 if (debug) {
2991 CFIL_LOG(LOG_INFO, "CFIL: TCP (dir %d) - debug flow with port %d", dir, cfil_log_port);
2992 }
2993
2994 filter_control_unit = necp_socket_get_content_filter_control_unit(so);
2995 if (filter_control_unit == 0) {
2996 goto done;
2997 }
2998
2999 if (filter_control_unit == NECP_FILTER_UNIT_NO_FILTER) {
3000 goto done;
3001 }
3002 if ((filter_control_unit & NECP_MASK_USERSPACE_ONLY) != 0) {
3003 OSIncrementAtomic(&cfil_stats.cfs_sock_userspace_only);
3004 goto done;
3005 }
3006 if (cfil_active_count == 0) {
3007 OSIncrementAtomic(&cfil_stats.cfs_sock_attach_in_vain);
3008 goto done;
3009 }
3010 if (so->so_cfil != NULL) {
3011 OSIncrementAtomic(&cfil_stats.cfs_sock_attach_already);
3012 CFIL_LOG(LOG_ERR, "already attached");
3013 goto done;
3014 } else {
3015 cfil_info_alloc(so, NULL);
3016 if (so->so_cfil == NULL) {
3017 error = ENOMEM;
3018 OSIncrementAtomic(&cfil_stats.cfs_sock_attach_no_mem);
3019 goto done;
3020 }
3021 so->so_cfil->cfi_dir = dir;
3022 so->so_cfil->cfi_filter_control_unit = filter_control_unit;
3023 so->so_cfil->cfi_debug = debug;
3024 }
3025 if (cfil_info_attach_unit(so, filter_control_unit, cfil_info: so->so_cfil) == 0) {
3026 CFIL_LOG(LOG_ERR, "cfil_info_attach_unit(%u) failed",
3027 filter_control_unit);
3028 OSIncrementAtomic(&cfil_stats.cfs_sock_attach_failed);
3029 goto done;
3030 }
3031 CFIL_LOG(LOG_INFO, "so %llx filter_control_unit %u sockID %llu <%llx>",
3032 (uint64_t)VM_KERNEL_ADDRPERM(so),
3033 filter_control_unit, so->so_cfil->cfi_sock_id, so->so_cfil->cfi_sock_id);
3034
3035 so->so_flags |= SOF_CONTENT_FILTER;
3036 OSIncrementAtomic(&cfil_stats.cfs_sock_attached);
3037
3038 /* Hold a reference on the socket */
3039 so->so_usecount++;
3040
3041 /*
3042 * Save passed addresses for attach event msg (in case resend
3043 * is needed.
3044 */
3045 if (remote != NULL && (remote->sa_len <= sizeof(union sockaddr_in_4_6))) {
3046 memcpy(dst: &so->so_cfil->cfi_so_attach_faddr, src: remote, n: remote->sa_len);
3047 }
3048 if (local != NULL && (local->sa_len <= sizeof(union sockaddr_in_4_6))) {
3049 memcpy(dst: &so->so_cfil->cfi_so_attach_laddr, src: local, n: local->sa_len);
3050 }
3051
3052 error = cfil_dispatch_attach_event(so, so->so_cfil, 0, dir);
3053 /* We can recover from flow control or out of memory errors */
3054 if (error == ENOBUFS || error == ENOMEM) {
3055 error = 0;
3056 } else if (error != 0) {
3057 goto done;
3058 }
3059
3060 CFIL_INFO_VERIFY(so->so_cfil);
3061done:
3062 return error;
3063}
3064
3065/*
3066 * Entry point from Sockets layer
3067 * The socket is locked.
3068 */
3069errno_t
3070cfil_sock_detach(struct socket *so)
3071{
3072 if (NEED_DGRAM_FLOW_TRACKING(so)) {
3073 return 0;
3074 }
3075
3076 if (so->so_cfil) {
3077 if (so->so_flags & SOF_CONTENT_FILTER) {
3078 so->so_flags &= ~SOF_CONTENT_FILTER;
3079 VERIFY(so->so_usecount > 0);
3080 so->so_usecount--;
3081 }
3082 CFIL_INFO_FREE(so->so_cfil);
3083 so->so_cfil = NULL;
3084 OSIncrementAtomic(&cfil_stats.cfs_sock_detached);
3085 }
3086 return 0;
3087}
3088
3089/*
3090 * Fill in the address info of an event message from either
3091 * the socket or passed in address info.
3092 */
3093static void
3094cfil_fill_event_msg_addresses(struct soflow_hash_entry *entry, struct inpcb *inp,
3095 union sockaddr_in_4_6 *sin_src, union sockaddr_in_4_6 *sin_dst,
3096 boolean_t isIPv4, boolean_t outgoing)
3097{
3098 if (isIPv4) {
3099 struct in_addr laddr = {0}, faddr = {0};
3100 u_int16_t lport = 0, fport = 0;
3101
3102 cfil_get_flow_address(entry, inp, &laddr, &faddr, &lport, &fport);
3103
3104 if (outgoing) {
3105 fill_ip_sockaddr_4_6(sin_src, laddr, lport);
3106 fill_ip_sockaddr_4_6(sin_dst, faddr, fport);
3107 } else {
3108 fill_ip_sockaddr_4_6(sin_src, faddr, fport);
3109 fill_ip_sockaddr_4_6(sin_dst, laddr, lport);
3110 }
3111 } else {
3112 struct in6_addr *laddr = NULL, *faddr = NULL;
3113 u_int16_t lport = 0, fport = 0;
3114 const u_int32_t lifscope = inp ? inp->inp_lifscope : IFSCOPE_UNKNOWN;
3115 const u_int32_t fifscope = inp ? inp->inp_fifscope : IFSCOPE_UNKNOWN;
3116
3117 cfil_get_flow_address_v6(entry, inp, &laddr, &faddr, &lport, &fport);
3118 if (outgoing) {
3119 fill_ip6_sockaddr_4_6(sin_src, laddr, lport, lifscope);
3120 fill_ip6_sockaddr_4_6(sin_dst, faddr, fport, fifscope);
3121 } else {
3122 fill_ip6_sockaddr_4_6(sin_src, faddr, fport, fifscope);
3123 fill_ip6_sockaddr_4_6(sin_dst, laddr, lport, lifscope);
3124 }
3125 }
3126}
3127
3128static boolean_t
3129cfil_dispatch_attach_event_sign(cfil_crypto_state_t crypto_state,
3130 struct cfil_info *cfil_info,
3131 struct cfil_msg_sock_attached *msg)
3132{
3133 struct cfil_crypto_data data = {};
3134 struct iovec extra_data[1] = { { NULL, 0 } };
3135
3136 if (crypto_state == NULL || msg == NULL || cfil_info == NULL) {
3137 return false;
3138 }
3139
3140 data.sock_id = msg->cfs_msghdr.cfm_sock_id;
3141 data.direction = msg->cfs_conn_dir;
3142
3143 data.pid = msg->cfs_pid;
3144 data.effective_pid = msg->cfs_e_pid;
3145 uuid_copy(dst: data.uuid, src: msg->cfs_uuid);
3146 uuid_copy(dst: data.effective_uuid, src: msg->cfs_e_uuid);
3147 data.socketProtocol = msg->cfs_sock_protocol;
3148 if (data.direction == CFS_CONNECTION_DIR_OUT) {
3149 data.remote.sin6 = msg->cfs_dst.sin6;
3150 data.local.sin6 = msg->cfs_src.sin6;
3151 } else {
3152 data.remote.sin6 = msg->cfs_src.sin6;
3153 data.local.sin6 = msg->cfs_dst.sin6;
3154 }
3155
3156 if (strlen(s: msg->cfs_remote_domain_name) > 0) {
3157 extra_data[0].iov_base = msg->cfs_remote_domain_name;
3158 extra_data[0].iov_len = strlen(s: msg->cfs_remote_domain_name);
3159 }
3160
3161 // At attach, if local address is already present, no need to re-sign subsequent data messages.
3162 if (!NULLADDRESS(data.local)) {
3163 cfil_info->cfi_isSignatureLatest = true;
3164 }
3165
3166 msg->cfs_signature_length = sizeof(cfil_crypto_signature);
3167 if (cfil_crypto_sign_data(state: crypto_state, data: &data, extra_data, extra_data_count: sizeof(extra_data) / sizeof(extra_data[0]), signature: msg->cfs_signature, signature_length: &msg->cfs_signature_length) != 0) {
3168 msg->cfs_signature_length = 0;
3169 CFIL_LOG(LOG_ERR, "CFIL: Failed to sign attached msg <sockID %llu <%llx>>",
3170 msg->cfs_msghdr.cfm_sock_id, msg->cfs_msghdr.cfm_sock_id);
3171 return false;
3172 }
3173
3174 return true;
3175}
3176
3177struct cfil_sign_parameters {
3178 cfil_crypto_state_t csp_state;
3179 struct cfil_crypto_data *csp_data;
3180 uint8_t *csp_signature;
3181 uint32_t *csp_signature_size;
3182};
3183
3184static void
3185cfil_sign_with_domain_name(char *domain_name, void *ctx)
3186{
3187 struct cfil_sign_parameters *parameters = (struct cfil_sign_parameters *)ctx;
3188 struct iovec extra_data[1] = { { NULL, 0 } };
3189
3190 if (parameters == NULL) {
3191 return;
3192 }
3193
3194 if (domain_name != NULL) {
3195 extra_data[0].iov_base = domain_name;
3196 extra_data[0].iov_len = strlen(s: domain_name);
3197 }
3198
3199 *(parameters->csp_signature_size) = sizeof(cfil_crypto_signature);
3200 if (cfil_crypto_sign_data(state: parameters->csp_state, data: parameters->csp_data,
3201 extra_data, extra_data_count: sizeof(extra_data) / sizeof(extra_data[0]),
3202 signature: parameters->csp_signature, signature_length: parameters->csp_signature_size) != 0) {
3203 *(parameters->csp_signature_size) = 0;
3204 }
3205}
3206
3207static boolean_t
3208cfil_dispatch_data_event_sign(cfil_crypto_state_t crypto_state,
3209 struct socket *so, struct cfil_info *cfil_info,
3210 struct cfil_msg_data_event *msg)
3211{
3212 struct cfil_crypto_data data = {};
3213
3214 if (crypto_state == NULL || msg == NULL ||
3215 so == NULL || cfil_info == NULL) {
3216 return false;
3217 }
3218
3219 data.sock_id = cfil_info->cfi_sock_id;
3220 data.direction = cfil_info->cfi_dir;
3221 data.pid = so->last_pid;
3222 memcpy(dst: data.uuid, src: so->last_uuid, n: sizeof(uuid_t));
3223 if (so->so_flags & SOF_DELEGATED) {
3224 data.effective_pid = so->e_pid;
3225 memcpy(dst: data.effective_uuid, src: so->e_uuid, n: sizeof(uuid_t));
3226 } else {
3227 data.effective_pid = so->last_pid;
3228 memcpy(dst: data.effective_uuid, src: so->last_uuid, n: sizeof(uuid_t));
3229 }
3230 data.socketProtocol = GET_SO_PROTO(so);
3231
3232 if (data.direction == CFS_CONNECTION_DIR_OUT) {
3233 data.remote.sin6 = msg->cfc_dst.sin6;
3234 data.local.sin6 = msg->cfc_src.sin6;
3235 } else {
3236 data.remote.sin6 = msg->cfc_src.sin6;
3237 data.local.sin6 = msg->cfc_dst.sin6;
3238 }
3239
3240 // At first data, local address may show up for the first time, update address cache and
3241 // no need to re-sign subsequent data messages anymore.
3242 if (!NULLADDRESS(data.local)) {
3243 memcpy(dst: &cfil_info->cfi_so_attach_laddr, src: &data.local, n: data.local.sa.sa_len);
3244 cfil_info->cfi_isSignatureLatest = true;
3245 }
3246
3247 struct cfil_sign_parameters parameters = {
3248 .csp_state = crypto_state,
3249 .csp_data = &data,
3250 .csp_signature = msg->cfd_signature,
3251 .csp_signature_size = &msg->cfd_signature_length,
3252 };
3253 necp_with_inp_domain_name(so, ctx: &parameters, with_func: cfil_sign_with_domain_name);
3254
3255 if (msg->cfd_signature_length == 0) {
3256 CFIL_LOG(LOG_ERR, "CFIL: Failed to sign data msg <sockID %llu <%llx>>",
3257 msg->cfd_msghdr.cfm_sock_id, msg->cfd_msghdr.cfm_sock_id);
3258 return false;
3259 }
3260
3261 return true;
3262}
3263
3264static boolean_t
3265cfil_dispatch_closed_event_sign(cfil_crypto_state_t crypto_state,
3266 struct socket *so, struct cfil_info *cfil_info,
3267 struct cfil_msg_sock_closed *msg)
3268{
3269 struct cfil_crypto_data data = {};
3270 struct soflow_hash_entry hash_entry = {};
3271 struct soflow_hash_entry *hash_entry_ptr = NULL;
3272 struct inpcb *inp = (struct inpcb *)so->so_pcb;
3273
3274 if (crypto_state == NULL || msg == NULL ||
3275 so == NULL || inp == NULL || cfil_info == NULL) {
3276 return false;
3277 }
3278
3279 data.sock_id = cfil_info->cfi_sock_id;
3280 data.direction = cfil_info->cfi_dir;
3281
3282 data.pid = so->last_pid;
3283 memcpy(dst: data.uuid, src: so->last_uuid, n: sizeof(uuid_t));
3284 if (so->so_flags & SOF_DELEGATED) {
3285 data.effective_pid = so->e_pid;
3286 memcpy(dst: data.effective_uuid, src: so->e_uuid, n: sizeof(uuid_t));
3287 } else {
3288 data.effective_pid = so->last_pid;
3289 memcpy(dst: data.effective_uuid, src: so->last_uuid, n: sizeof(uuid_t));
3290 }
3291 data.socketProtocol = GET_SO_PROTO(so);
3292
3293 /*
3294 * Fill in address info:
3295 * For UDP, use the cfil_info hash entry directly.
3296 * For TCP, compose an hash entry with the saved addresses.
3297 */
3298 if (cfil_info->cfi_hash_entry != NULL) {
3299 hash_entry_ptr = cfil_info->cfi_hash_entry;
3300 } else if (cfil_info->cfi_so_attach_faddr.sa.sa_len > 0 ||
3301 cfil_info->cfi_so_attach_laddr.sa.sa_len > 0) {
3302 soflow_fill_hash_entry_from_address(&hash_entry, TRUE, SA(&cfil_info->cfi_so_attach_laddr.sa), FALSE);
3303 soflow_fill_hash_entry_from_address(&hash_entry, FALSE, SA(&cfil_info->cfi_so_attach_faddr.sa), FALSE);
3304 hash_entry_ptr = &hash_entry;
3305 }
3306 if (hash_entry_ptr != NULL) {
3307 boolean_t outgoing = (cfil_info->cfi_dir == CFS_CONNECTION_DIR_OUT);
3308 union sockaddr_in_4_6 *src = outgoing ? &data.local : &data.remote;
3309 union sockaddr_in_4_6 *dst = outgoing ? &data.remote : &data.local;
3310 cfil_fill_event_msg_addresses(entry: hash_entry_ptr, inp, sin_src: src, sin_dst: dst, isIPv4: !IS_INP_V6(inp), outgoing);
3311 }
3312
3313 data.byte_count_in = cfil_info->cfi_byte_inbound_count;
3314 data.byte_count_out = cfil_info->cfi_byte_outbound_count;
3315
3316 struct cfil_sign_parameters parameters = {
3317 .csp_state = crypto_state,
3318 .csp_data = &data,
3319 .csp_signature = msg->cfc_signature,
3320 .csp_signature_size = &msg->cfc_signature_length
3321 };
3322 necp_with_inp_domain_name(so, ctx: &parameters, with_func: cfil_sign_with_domain_name);
3323
3324 if (msg->cfc_signature_length == 0) {
3325 CFIL_LOG(LOG_ERR, "CFIL: Failed to sign closed msg <sockID %llu <%llx>>",
3326 msg->cfc_msghdr.cfm_sock_id, msg->cfc_msghdr.cfm_sock_id);
3327 return false;
3328 }
3329
3330 return true;
3331}
3332
3333static void
3334cfil_populate_attached_msg_domain_name(char *domain_name, void *ctx)
3335{
3336 struct cfil_msg_sock_attached *msg_attached = (struct cfil_msg_sock_attached *)ctx;
3337
3338 if (msg_attached == NULL) {
3339 return;
3340 }
3341
3342 if (domain_name != NULL) {
3343 strlcpy(dst: msg_attached->cfs_remote_domain_name, src: domain_name, n: sizeof(msg_attached->cfs_remote_domain_name));
3344 }
3345}
3346
3347static bool
3348cfil_copy_audit_token(pid_t pid, audit_token_t *buffer)
3349{
3350 bool success = false;
3351 proc_t p = proc_find(pid);
3352 if (p != PROC_NULL) {
3353 task_t t = proc_task(p);
3354 if (t != TASK_NULL) {
3355 audit_token_t audit_token = {};
3356 mach_msg_type_number_t count = TASK_AUDIT_TOKEN_COUNT;
3357 if (task_info(task: t, TASK_AUDIT_TOKEN, task_info_out: (task_info_t)&audit_token, task_info_count: &count) == KERN_SUCCESS) {
3358 memcpy(dst: buffer, src: &audit_token, n: sizeof(audit_token_t));
3359 success = true;
3360 }
3361 }
3362 proc_rele(p);
3363 }
3364 return success;
3365}
3366
3367static int
3368cfil_dispatch_attach_event(struct socket *so, struct cfil_info *cfil_info,
3369 uint32_t kcunit, int conn_dir)
3370{
3371 errno_t error = 0;
3372 struct cfil_entry *entry = NULL;
3373 struct cfil_msg_sock_attached *msg_attached;
3374 struct content_filter *cfc = NULL;
3375 struct inpcb *inp = (struct inpcb *)so->so_pcb;
3376 struct soflow_hash_entry *hash_entry_ptr = NULL;
3377 struct soflow_hash_entry hash_entry;
3378
3379 memset(s: &hash_entry, c: 0, n: sizeof(struct soflow_hash_entry));
3380
3381 socket_lock_assert_owned(so);
3382
3383 cfil_rw_lock_shared(lck: &cfil_lck_rw);
3384
3385 if (so->so_proto == NULL || so->so_proto->pr_domain == NULL) {
3386 error = EINVAL;
3387 goto done;
3388 }
3389
3390 if (kcunit == 0) {
3391 entry = SLIST_FIRST(&cfil_info->cfi_ordered_entries);
3392 } else {
3393 entry = &cfil_info->cfi_entries[kcunit - 1];
3394 }
3395
3396 if (entry == NULL) {
3397 goto done;
3398 }
3399
3400 cfc = entry->cfe_filter;
3401 if (cfc == NULL) {
3402 goto done;
3403 }
3404
3405 if ((entry->cfe_flags & CFEF_SENT_SOCK_ATTACHED)) {
3406 goto done;
3407 }
3408
3409 if (kcunit == 0) {
3410 kcunit = CFI_ENTRY_KCUNIT(cfil_info, entry);
3411 }
3412
3413 CFIL_LOG(LOG_INFO, "so %llx filter_control_unit %u kcunit %u",
3414 (uint64_t)VM_KERNEL_ADDRPERM(so), entry->cfe_necp_control_unit, kcunit);
3415
3416 /* Would be wasteful to try when flow controlled */
3417 if (cfc->cf_flags & CFF_FLOW_CONTROLLED) {
3418 error = ENOBUFS;
3419 goto done;
3420 }
3421
3422 msg_attached = kalloc_data(sizeof(struct cfil_msg_sock_attached), Z_WAITOK);
3423 if (msg_attached == NULL) {
3424 error = ENOMEM;
3425 goto done;
3426 }
3427
3428 bzero(s: msg_attached, n: sizeof(struct cfil_msg_sock_attached));
3429 msg_attached->cfs_msghdr.cfm_len = sizeof(struct cfil_msg_sock_attached);
3430 msg_attached->cfs_msghdr.cfm_version = CFM_VERSION_CURRENT;
3431 msg_attached->cfs_msghdr.cfm_type = CFM_TYPE_EVENT;
3432 msg_attached->cfs_msghdr.cfm_op = CFM_OP_SOCKET_ATTACHED;
3433 msg_attached->cfs_msghdr.cfm_sock_id = entry->cfe_cfil_info->cfi_sock_id;
3434
3435 msg_attached->cfs_sock_family = SOCK_DOM(so);
3436 msg_attached->cfs_sock_type = SOCK_TYPE(so);
3437 msg_attached->cfs_sock_protocol = GET_SO_PROTO(so);
3438 msg_attached->cfs_pid = so->last_pid;
3439 memcpy(dst: msg_attached->cfs_uuid, src: so->last_uuid, n: sizeof(uuid_t));
3440 if (so->so_flags & SOF_DELEGATED) {
3441 msg_attached->cfs_e_pid = so->e_pid;
3442 memcpy(dst: msg_attached->cfs_e_uuid, src: so->e_uuid, n: sizeof(uuid_t));
3443 } else {
3444 msg_attached->cfs_e_pid = so->last_pid;
3445 memcpy(dst: msg_attached->cfs_e_uuid, src: so->last_uuid, n: sizeof(uuid_t));
3446 }
3447
3448 /*
3449 * Fill in address info:
3450 * For UDP, use the cfil_info hash entry directly.
3451 * For TCP, compose an hash entry with the saved addresses.
3452 */
3453 if (cfil_info->cfi_hash_entry != NULL) {
3454 hash_entry_ptr = cfil_info->cfi_hash_entry;
3455 } else if (cfil_info->cfi_so_attach_faddr.sa.sa_len > 0 ||
3456 cfil_info->cfi_so_attach_laddr.sa.sa_len > 0) {
3457 soflow_fill_hash_entry_from_address(&hash_entry, TRUE, SA(&cfil_info->cfi_so_attach_laddr.sa), FALSE);
3458 soflow_fill_hash_entry_from_address(&hash_entry, FALSE, SA(&cfil_info->cfi_so_attach_faddr.sa), FALSE);
3459 hash_entry_ptr = &hash_entry;
3460 }
3461 if (hash_entry_ptr != NULL) {
3462 cfil_fill_event_msg_addresses(entry: hash_entry_ptr, inp,
3463 sin_src: &msg_attached->cfs_src, sin_dst: &msg_attached->cfs_dst,
3464 isIPv4: !IS_INP_V6(inp), outgoing: conn_dir == CFS_CONNECTION_DIR_OUT);
3465 }
3466 msg_attached->cfs_conn_dir = conn_dir;
3467
3468 if (msg_attached->cfs_e_pid != 0) {
3469 if (!cfil_copy_audit_token(pid: msg_attached->cfs_e_pid, buffer: (audit_token_t *)&msg_attached->cfs_audit_token)) {
3470 CFIL_LOG(LOG_ERR, "CFIL: Failed to get effective audit token for <sockID %llu <%llx>> ",
3471 entry->cfe_cfil_info->cfi_sock_id, entry->cfe_cfil_info->cfi_sock_id);
3472 }
3473 }
3474
3475 if (msg_attached->cfs_pid != 0) {
3476 if (msg_attached->cfs_pid == msg_attached->cfs_e_pid) {
3477 memcpy(dst: &msg_attached->cfs_real_audit_token, src: &msg_attached->cfs_audit_token, n: sizeof(msg_attached->cfs_real_audit_token));
3478 } else if (!cfil_copy_audit_token(pid: msg_attached->cfs_pid, buffer: (audit_token_t *)&msg_attached->cfs_real_audit_token)) {
3479 CFIL_LOG(LOG_ERR, "CFIL: Failed to get real audit token for <sockID %llu <%llx>> ",
3480 entry->cfe_cfil_info->cfi_sock_id, entry->cfe_cfil_info->cfi_sock_id);
3481 }
3482 }
3483
3484 necp_with_inp_domain_name(so, ctx: msg_attached, with_func: cfil_populate_attached_msg_domain_name);
3485
3486 if (cfil_info->cfi_debug) {
3487 cfil_info_log(LOG_ERR, cfil_info, "CFIL: SENDING ATTACH UP");
3488 }
3489
3490 cfil_dispatch_attach_event_sign(crypto_state: entry->cfe_filter->cf_crypto_state, cfil_info, msg: msg_attached);
3491
3492 error = ctl_enqueuedata(kctlref: entry->cfe_filter->cf_kcref,
3493 unit: entry->cfe_filter->cf_kcunit,
3494 data: msg_attached,
3495 len: sizeof(struct cfil_msg_sock_attached),
3496 CTL_DATA_EOR);
3497
3498 kfree_data(msg_attached, sizeof(struct cfil_msg_sock_attached));
3499
3500 if (error != 0) {
3501 CFIL_LOG(LOG_ERR, "ctl_enqueuedata() failed: %d", error);
3502 goto done;
3503 }
3504 microuptime(tv: &entry->cfe_last_event);
3505 cfil_info->cfi_first_event.tv_sec = entry->cfe_last_event.tv_sec;
3506 cfil_info->cfi_first_event.tv_usec = entry->cfe_last_event.tv_usec;
3507
3508 entry->cfe_flags |= CFEF_SENT_SOCK_ATTACHED;
3509 OSIncrementAtomic(&cfil_stats.cfs_attach_event_ok);
3510done:
3511
3512 /* We can recover from flow control */
3513 if (error == ENOBUFS) {
3514 entry->cfe_flags |= CFEF_FLOW_CONTROLLED;
3515 OSIncrementAtomic(&cfil_stats.cfs_attach_event_flow_control);
3516
3517 if (!cfil_rw_lock_shared_to_exclusive(lck: &cfil_lck_rw)) {
3518 cfil_rw_lock_exclusive(lck: &cfil_lck_rw);
3519 }
3520
3521 cfc->cf_flags |= CFF_FLOW_CONTROLLED;
3522
3523 cfil_rw_unlock_exclusive(lck: &cfil_lck_rw);
3524 } else {
3525 if (error != 0) {
3526 OSIncrementAtomic(&cfil_stats.cfs_attach_event_fail);
3527 }
3528
3529 cfil_rw_unlock_shared(lck: &cfil_lck_rw);
3530 }
3531 return error;
3532}
3533
3534static int
3535cfil_dispatch_disconnect_event(struct socket *so, struct cfil_info *cfil_info, uint32_t kcunit, int outgoing)
3536{
3537 errno_t error = 0;
3538 struct mbuf *msg = NULL;
3539 struct cfil_entry *entry;
3540 struct cfe_buf *entrybuf;
3541 struct cfil_msg_hdr msg_disconnected;
3542 struct content_filter *cfc;
3543
3544 socket_lock_assert_owned(so);
3545
3546 cfil_rw_lock_shared(lck: &cfil_lck_rw);
3547
3548 entry = &cfil_info->cfi_entries[kcunit - 1];
3549 if (outgoing) {
3550 entrybuf = &entry->cfe_snd;
3551 } else {
3552 entrybuf = &entry->cfe_rcv;
3553 }
3554
3555 cfc = entry->cfe_filter;
3556 if (cfc == NULL) {
3557 goto done;
3558 }
3559
3560 // Mark if this flow qualifies for immediate close.
3561 SET_NO_CLOSE_WAIT(sotoinpcb(so), cfil_info);
3562
3563 CFIL_LOG(LOG_INFO, "so %llx kcunit %u outgoing %d",
3564 (uint64_t)VM_KERNEL_ADDRPERM(so), kcunit, outgoing);
3565
3566 /*
3567 * Send the disconnection event once
3568 */
3569 if ((outgoing && (entry->cfe_flags & CFEF_SENT_DISCONNECT_OUT)) ||
3570 (!outgoing && (entry->cfe_flags & CFEF_SENT_DISCONNECT_IN))) {
3571 CFIL_LOG(LOG_INFO, "so %llx disconnect already sent",
3572 (uint64_t)VM_KERNEL_ADDRPERM(so));
3573 goto done;
3574 }
3575
3576 /*
3577 * We're not disconnected as long as some data is waiting
3578 * to be delivered to the filter
3579 */
3580 if (outgoing && cfil_queue_empty(cfq: &entrybuf->cfe_ctl_q) == 0) {
3581 CFIL_LOG(LOG_INFO, "so %llx control queue not empty",
3582 (uint64_t)VM_KERNEL_ADDRPERM(so));
3583 error = EBUSY;
3584 goto done;
3585 }
3586 /* Would be wasteful to try when flow controlled */
3587 if (cfc->cf_flags & CFF_FLOW_CONTROLLED) {
3588 error = ENOBUFS;
3589 goto done;
3590 }
3591
3592 if (cfil_info->cfi_debug) {
3593 cfil_info_log(LOG_ERR, cfil_info, outgoing ?
3594 "CFIL: OUT - SENDING DISCONNECT UP":
3595 "CFIL: IN - SENDING DISCONNECT UP");
3596 }
3597
3598 bzero(s: &msg_disconnected, n: sizeof(struct cfil_msg_hdr));
3599 msg_disconnected.cfm_len = sizeof(struct cfil_msg_hdr);
3600 msg_disconnected.cfm_version = CFM_VERSION_CURRENT;
3601 msg_disconnected.cfm_type = CFM_TYPE_EVENT;
3602 msg_disconnected.cfm_op = outgoing ? CFM_OP_DISCONNECT_OUT :
3603 CFM_OP_DISCONNECT_IN;
3604 msg_disconnected.cfm_sock_id = entry->cfe_cfil_info->cfi_sock_id;
3605 error = ctl_enqueuedata(kctlref: entry->cfe_filter->cf_kcref,
3606 unit: entry->cfe_filter->cf_kcunit,
3607 data: &msg_disconnected,
3608 len: sizeof(struct cfil_msg_hdr),
3609 CTL_DATA_EOR);
3610 if (error != 0) {
3611 CFIL_LOG(LOG_ERR, "ctl_enqueuembuf() failed: %d", error);
3612 mbuf_freem(mbuf: msg);
3613 goto done;
3614 }
3615 microuptime(tv: &entry->cfe_last_event);
3616 CFI_ADD_TIME_LOG(cfil_info, &entry->cfe_last_event, &cfil_info->cfi_first_event, msg_disconnected.cfm_op);
3617
3618 /* Remember we have sent the disconnection message */
3619 if (outgoing) {
3620 entry->cfe_flags |= CFEF_SENT_DISCONNECT_OUT;
3621 OSIncrementAtomic(&cfil_stats.cfs_disconnect_out_event_ok);
3622 } else {
3623 entry->cfe_flags |= CFEF_SENT_DISCONNECT_IN;
3624 OSIncrementAtomic(&cfil_stats.cfs_disconnect_in_event_ok);
3625 }
3626done:
3627 if (error == ENOBUFS) {
3628 entry->cfe_flags |= CFEF_FLOW_CONTROLLED;
3629 OSIncrementAtomic(
3630 &cfil_stats.cfs_disconnect_event_flow_control);
3631
3632 if (!cfil_rw_lock_shared_to_exclusive(lck: &cfil_lck_rw)) {
3633 cfil_rw_lock_exclusive(lck: &cfil_lck_rw);
3634 }
3635
3636 cfc->cf_flags |= CFF_FLOW_CONTROLLED;
3637
3638 cfil_rw_unlock_exclusive(lck: &cfil_lck_rw);
3639 } else {
3640 if (error != 0) {
3641 OSIncrementAtomic(
3642 &cfil_stats.cfs_disconnect_event_fail);
3643 }
3644
3645 cfil_rw_unlock_shared(lck: &cfil_lck_rw);
3646 }
3647 return error;
3648}
3649
3650int
3651cfil_dispatch_closed_event(struct socket *so, struct cfil_info *cfil_info, int kcunit)
3652{
3653 struct cfil_entry *entry;
3654 struct cfil_msg_sock_closed msg_closed;
3655 errno_t error = 0;
3656 struct content_filter *cfc;
3657 struct inpcb *inp = NULL;
3658
3659 socket_lock_assert_owned(so);
3660
3661 cfil_rw_lock_shared(lck: &cfil_lck_rw);
3662
3663 entry = &cfil_info->cfi_entries[kcunit - 1];
3664 cfc = entry->cfe_filter;
3665 if (cfc == NULL) {
3666 goto done;
3667 }
3668
3669 CFIL_LOG(LOG_INFO, "so %llx kcunit %d",
3670 (uint64_t)VM_KERNEL_ADDRPERM(so), kcunit);
3671
3672 /* Would be wasteful to try when flow controlled */
3673 if (cfc->cf_flags & CFF_FLOW_CONTROLLED) {
3674 error = ENOBUFS;
3675 goto done;
3676 }
3677 /*
3678 * Send a single closed message per filter
3679 */
3680 if ((entry->cfe_flags & CFEF_SENT_SOCK_CLOSED) != 0) {
3681 goto done;
3682 }
3683 if ((entry->cfe_flags & CFEF_SENT_SOCK_ATTACHED) == 0) {
3684 goto done;
3685 }
3686
3687 microuptime(tv: &entry->cfe_last_event);
3688 CFI_ADD_TIME_LOG(cfil_info, &entry->cfe_last_event, &cfil_info->cfi_first_event, CFM_OP_SOCKET_CLOSED);
3689
3690 bzero(s: &msg_closed, n: sizeof(struct cfil_msg_sock_closed));
3691 msg_closed.cfc_msghdr.cfm_len = sizeof(struct cfil_msg_sock_closed);
3692 msg_closed.cfc_msghdr.cfm_version = CFM_VERSION_CURRENT;
3693 msg_closed.cfc_msghdr.cfm_type = CFM_TYPE_EVENT;
3694 msg_closed.cfc_msghdr.cfm_op = CFM_OP_SOCKET_CLOSED;
3695 msg_closed.cfc_msghdr.cfm_sock_id = entry->cfe_cfil_info->cfi_sock_id;
3696 msg_closed.cfc_first_event.tv_sec = cfil_info->cfi_first_event.tv_sec;
3697 msg_closed.cfc_first_event.tv_usec = cfil_info->cfi_first_event.tv_usec;
3698 memcpy(dst: msg_closed.cfc_op_time, src: cfil_info->cfi_op_time, n: sizeof(uint32_t) * CFI_MAX_TIME_LOG_ENTRY);
3699 memcpy(dst: msg_closed.cfc_op_list, src: cfil_info->cfi_op_list, n: sizeof(unsigned char) * CFI_MAX_TIME_LOG_ENTRY);
3700 msg_closed.cfc_op_list_ctr = cfil_info->cfi_op_list_ctr;
3701 msg_closed.cfc_byte_inbound_count = cfil_info->cfi_byte_inbound_count;
3702 msg_closed.cfc_byte_outbound_count = cfil_info->cfi_byte_outbound_count;
3703
3704 if (entry->cfe_laddr_sent == false) {
3705 /* cache it if necessary */
3706 if (cfil_info->cfi_so_attach_laddr.sa.sa_len == 0) {
3707 inp = cfil_info->cfi_so ? sotoinpcb(cfil_info->cfi_so) : NULL;
3708 if (inp != NULL) {
3709 boolean_t outgoing = (cfil_info->cfi_dir == CFS_CONNECTION_DIR_OUT);
3710 union sockaddr_in_4_6 *src = outgoing ? &cfil_info->cfi_so_attach_laddr : NULL;
3711 union sockaddr_in_4_6 *dst = outgoing ? NULL : &cfil_info->cfi_so_attach_laddr;
3712 cfil_fill_event_msg_addresses(entry: cfil_info->cfi_hash_entry, inp,
3713 sin_src: src, sin_dst: dst, isIPv4: !IS_INP_V6(inp), outgoing);
3714 }
3715 }
3716
3717 if (cfil_info->cfi_so_attach_laddr.sa.sa_len != 0) {
3718 msg_closed.cfc_laddr.sin6 = cfil_info->cfi_so_attach_laddr.sin6;
3719 entry->cfe_laddr_sent = true;
3720 }
3721 }
3722
3723 cfil_dispatch_closed_event_sign(crypto_state: entry->cfe_filter->cf_crypto_state, so, cfil_info, msg: &msg_closed);
3724
3725 if (cfil_info->cfi_debug) {
3726 cfil_info_log(LOG_ERR, cfil_info, "CFIL: SENDING CLOSED UP");
3727 }
3728
3729 /* for debugging
3730 * if (msg_closed.cfc_op_list_ctr > CFI_MAX_TIME_LOG_ENTRY) {
3731 * msg_closed.cfc_op_list_ctr = CFI_MAX_TIME_LOG_ENTRY; // just in case
3732 * }
3733 * for (unsigned int i = 0; i < msg_closed.cfc_op_list_ctr ; i++) {
3734 * CFIL_LOG(LOG_ERR, "MD: socket %llu event %2u, time + %u msec", msg_closed.cfc_msghdr.cfm_sock_id, (unsigned short)msg_closed.cfc_op_list[i], msg_closed.cfc_op_time[i]);
3735 * }
3736 */
3737
3738 error = ctl_enqueuedata(kctlref: entry->cfe_filter->cf_kcref,
3739 unit: entry->cfe_filter->cf_kcunit,
3740 data: &msg_closed,
3741 len: sizeof(struct cfil_msg_sock_closed),
3742 CTL_DATA_EOR);
3743 if (error != 0) {
3744 CFIL_LOG(LOG_ERR, "ctl_enqueuedata() failed: %d",
3745 error);
3746 goto done;
3747 }
3748
3749 entry->cfe_flags |= CFEF_SENT_SOCK_CLOSED;
3750 OSIncrementAtomic(&cfil_stats.cfs_closed_event_ok);
3751done:
3752 /* We can recover from flow control */
3753 if (error == ENOBUFS) {
3754 entry->cfe_flags |= CFEF_FLOW_CONTROLLED;
3755 OSIncrementAtomic(&cfil_stats.cfs_closed_event_flow_control);
3756
3757 if (!cfil_rw_lock_shared_to_exclusive(lck: &cfil_lck_rw)) {
3758 cfil_rw_lock_exclusive(lck: &cfil_lck_rw);
3759 }
3760
3761 cfc->cf_flags |= CFF_FLOW_CONTROLLED;
3762
3763 cfil_rw_unlock_exclusive(lck: &cfil_lck_rw);
3764 } else {
3765 if (error != 0) {
3766 OSIncrementAtomic(&cfil_stats.cfs_closed_event_fail);
3767 }
3768
3769 cfil_rw_unlock_shared(lck: &cfil_lck_rw);
3770 }
3771
3772 return error;
3773}
3774
3775static void
3776fill_ip6_sockaddr_4_6(union sockaddr_in_4_6 *sin46,
3777 struct in6_addr *ip6, u_int16_t port, uint32_t ifscope)
3778{
3779 if (sin46 == NULL) {
3780 return;
3781 }
3782
3783 struct sockaddr_in6 *sin6 = &sin46->sin6;
3784
3785 sin6->sin6_family = AF_INET6;
3786 sin6->sin6_len = sizeof(*sin6);
3787 sin6->sin6_port = port;
3788 sin6->sin6_addr = *ip6;
3789 if (IN6_IS_SCOPE_EMBED(&sin6->sin6_addr)) {
3790 sin6->sin6_scope_id = ifscope;
3791 if (in6_embedded_scope) {
3792 in6_verify_ifscope(&sin6->sin6_addr, sin6->sin6_scope_id);
3793 if (sin6->sin6_addr.s6_addr16[1] != 0) {
3794 sin6->sin6_scope_id = ntohs(sin6->sin6_addr.s6_addr16[1]);
3795 sin6->sin6_addr.s6_addr16[1] = 0;
3796 }
3797 }
3798 }
3799}
3800
3801static void
3802fill_ip_sockaddr_4_6(union sockaddr_in_4_6 *sin46,
3803 struct in_addr ip, u_int16_t port)
3804{
3805 if (sin46 == NULL) {
3806 return;
3807 }
3808
3809 struct sockaddr_in *sin = &sin46->sin;
3810
3811 sin->sin_family = AF_INET;
3812 sin->sin_len = sizeof(*sin);
3813 sin->sin_port = port;
3814 sin->sin_addr.s_addr = ip.s_addr;
3815}
3816
3817static void
3818cfil_get_flow_address_v6(struct soflow_hash_entry *entry, struct inpcb *inp,
3819 struct in6_addr **laddr, struct in6_addr **faddr,
3820 u_int16_t *lport, u_int16_t *fport)
3821{
3822 if (entry != NULL) {
3823 *laddr = &entry->soflow_laddr.addr6;
3824 *faddr = &entry->soflow_faddr.addr6;
3825 *lport = entry->soflow_lport;
3826 *fport = entry->soflow_fport;
3827 } else {
3828 *laddr = &inp->in6p_laddr;
3829 *faddr = &inp->in6p_faddr;
3830 *lport = inp->inp_lport;
3831 *fport = inp->inp_fport;
3832 }
3833}
3834
3835static void
3836cfil_get_flow_address(struct soflow_hash_entry *entry, struct inpcb *inp,
3837 struct in_addr *laddr, struct in_addr *faddr,
3838 u_int16_t *lport, u_int16_t *fport)
3839{
3840 if (entry != NULL) {
3841 *laddr = entry->soflow_laddr.addr46.ia46_addr4;
3842 *faddr = entry->soflow_faddr.addr46.ia46_addr4;
3843 *lport = entry->soflow_lport;
3844 *fport = entry->soflow_fport;
3845 } else {
3846 *laddr = inp->inp_laddr;
3847 *faddr = inp->inp_faddr;
3848 *lport = inp->inp_lport;
3849 *fport = inp->inp_fport;
3850 }
3851}
3852
3853static int
3854cfil_dispatch_data_event(struct socket *so, struct cfil_info *cfil_info, uint32_t kcunit, int outgoing,
3855 struct mbuf *data, unsigned int copyoffset, unsigned int copylen)
3856{
3857 errno_t error = 0;
3858 struct mbuf *copy = NULL;
3859 struct mbuf *msg = NULL;
3860 unsigned int one = 1;
3861 struct cfil_msg_data_event *data_req;
3862 size_t hdrsize;
3863 struct inpcb *inp = (struct inpcb *)so->so_pcb;
3864 struct cfil_entry *entry;
3865 struct cfe_buf *entrybuf;
3866 struct content_filter *cfc;
3867 struct timeval tv;
3868 int inp_flags = 0;
3869
3870 cfil_rw_lock_shared(lck: &cfil_lck_rw);
3871
3872 entry = &cfil_info->cfi_entries[kcunit - 1];
3873 if (outgoing) {
3874 entrybuf = &entry->cfe_snd;
3875 } else {
3876 entrybuf = &entry->cfe_rcv;
3877 }
3878
3879 cfc = entry->cfe_filter;
3880 if (cfc == NULL) {
3881 goto done;
3882 }
3883
3884 data = cfil_data_start(m: data);
3885 if (data == NULL) {
3886 CFIL_LOG(LOG_ERR, "No data start");
3887 goto done;
3888 }
3889
3890 CFIL_LOG(LOG_INFO, "so %llx kcunit %u outgoing %d",
3891 (uint64_t)VM_KERNEL_ADDRPERM(so), kcunit, outgoing);
3892
3893 socket_lock_assert_owned(so);
3894
3895 /* Would be wasteful to try */
3896 if (cfc->cf_flags & CFF_FLOW_CONTROLLED) {
3897 error = ENOBUFS;
3898 goto done;
3899 }
3900
3901 /* Make a copy of the data to pass to kernel control socket */
3902 copy = m_copym_mode(data, copyoffset, copylen, M_DONTWAIT, NULL, NULL,
3903 M_COPYM_NOOP_HDR);
3904 if (copy == NULL) {
3905 CFIL_LOG(LOG_ERR, "m_copym_mode() failed");
3906 error = ENOMEM;
3907 goto done;
3908 }
3909
3910 /* We need an mbuf packet for the message header */
3911 hdrsize = sizeof(struct cfil_msg_data_event);
3912 error = mbuf_allocpacket(how: MBUF_DONTWAIT, packetlen: hdrsize, maxchunks: &one, mbuf: &msg);
3913 if (error != 0) {
3914 CFIL_LOG(LOG_ERR, "mbuf_allocpacket() failed");
3915 m_freem(copy);
3916 /*
3917 * ENOBUFS is to indicate flow control
3918 */
3919 error = ENOMEM;
3920 goto done;
3921 }
3922 mbuf_setlen(mbuf: msg, len: hdrsize);
3923 mbuf_pkthdr_setlen(mbuf: msg, len: hdrsize + copylen);
3924 msg->m_next = copy;
3925 data_req = (struct cfil_msg_data_event *)mbuf_data(mbuf: msg);
3926 bzero(s: data_req, n: hdrsize);
3927 data_req->cfd_msghdr.cfm_len = (uint32_t)hdrsize + copylen;
3928 data_req->cfd_msghdr.cfm_version = 1;
3929 data_req->cfd_msghdr.cfm_type = CFM_TYPE_EVENT;
3930 data_req->cfd_msghdr.cfm_op =
3931 outgoing ? CFM_OP_DATA_OUT : CFM_OP_DATA_IN;
3932 data_req->cfd_msghdr.cfm_sock_id =
3933 entry->cfe_cfil_info->cfi_sock_id;
3934 data_req->cfd_start_offset = entrybuf->cfe_peeked;
3935 data_req->cfd_end_offset = entrybuf->cfe_peeked + copylen;
3936
3937 data_req->cfd_flags = 0;
3938 if (OPTIONAL_IP_HEADER(so)) {
3939 /*
3940 * For non-UDP/TCP traffic, indicate to filters if optional
3941 * IP header is present:
3942 * outgoing - indicate according to INP_HDRINCL flag
3943 * incoming - For IPv4 only, stripping of IP header is
3944 * optional. But for CFIL, we delay stripping
3945 * at rip_input. So CFIL always expects IP
3946 * frames. IP header will be stripped according
3947 * to INP_STRIPHDR flag later at reinjection.
3948 */
3949 if ((!outgoing && !IS_INP_V6(inp)) ||
3950 (outgoing && cfil_dgram_peek_socket_state(m: data, inp_flags: &inp_flags) && (inp_flags & INP_HDRINCL))) {
3951 data_req->cfd_flags |= CFD_DATA_FLAG_IP_HEADER;
3952 }
3953 }
3954
3955 /*
3956 * Copy address/port into event msg.
3957 * For non connected sockets need to copy addresses from passed
3958 * parameters
3959 */
3960 cfil_fill_event_msg_addresses(entry: cfil_info->cfi_hash_entry, inp,
3961 sin_src: &data_req->cfc_src, sin_dst: &data_req->cfc_dst,
3962