1/*
2 * Copyright (c) 2000-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 *
28 */
29/*-
30 * Copyright (c) 1999,2000,2001 Jonathan Lemon <jlemon@FreeBSD.org>
31 * All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 * 1. Redistributions of source code must retain the above copyright
37 * notice, this list of conditions and the following disclaimer.
38 * 2. Redistributions in binary form must reproduce the above copyright
39 * notice, this list of conditions and the following disclaimer in the
40 * documentation and/or other materials provided with the distribution.
41 *
42 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
43 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
44 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
45 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
46 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
47 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
48 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
49 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
50 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
51 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
52 * SUCH DAMAGE.
53 */
54/*
55 * @(#)kern_event.c 1.0 (3/31/2000)
56 */
57#include <stdint.h>
58#include <machine/atomic.h>
59
60#include <sys/param.h>
61#include <sys/systm.h>
62#include <sys/filedesc.h>
63#include <sys/kernel.h>
64#include <sys/proc_internal.h>
65#include <sys/kauth.h>
66#include <sys/malloc.h>
67#include <sys/unistd.h>
68#include <sys/file_internal.h>
69#include <sys/fcntl.h>
70#include <sys/select.h>
71#include <sys/queue.h>
72#include <sys/event.h>
73#include <sys/eventvar.h>
74#include <sys/protosw.h>
75#include <sys/socket.h>
76#include <sys/socketvar.h>
77#include <sys/stat.h>
78#include <sys/syscall.h> // SYS_* constants
79#include <sys/sysctl.h>
80#include <sys/uio.h>
81#include <sys/sysproto.h>
82#include <sys/user.h>
83#include <sys/vnode_internal.h>
84#include <string.h>
85#include <sys/proc_info.h>
86#include <sys/codesign.h>
87#include <sys/pthread_shims.h>
88#include <sys/kdebug.h>
89#include <os/base.h>
90#include <pexpert/pexpert.h>
91
92#include <kern/thread_group.h>
93#include <kern/locks.h>
94#include <kern/clock.h>
95#include <kern/cpu_data.h>
96#include <kern/policy_internal.h>
97#include <kern/thread_call.h>
98#include <kern/sched_prim.h>
99#include <kern/waitq.h>
100#include <kern/zalloc.h>
101#include <kern/kalloc.h>
102#include <kern/assert.h>
103#include <kern/ast.h>
104#include <kern/thread.h>
105#include <kern/kcdata.h>
106#include <kern/work_interval.h>
107
108#include <pthread/priority_private.h>
109#include <pthread/workqueue_syscalls.h>
110#include <pthread/workqueue_internal.h>
111#include <libkern/libkern.h>
112
113#include <os/log.h>
114
115#include "net/net_str_id.h"
116
117#if SKYWALK && defined(XNU_TARGET_OS_OSX)
118#include <skywalk/lib/net_filter_event.h>
119
120extern bool net_check_compatible_alf(void);
121#endif /* SKYWALK && XNU_TARGET_OS_OSX */
122
123#include <mach/task.h>
124#include <libkern/section_keywords.h>
125
126#if CONFIG_MEMORYSTATUS
127#include <sys/kern_memorystatus.h>
128#endif
129
130#if DEVELOPMENT || DEBUG
131#define KEVENT_PANIC_ON_WORKLOOP_OWNERSHIP_LEAK (1U << 0)
132#define KEVENT_PANIC_ON_NON_ENQUEUED_PROCESS (1U << 1)
133TUNABLE(uint32_t, kevent_debug_flags, "kevent_debug", 0);
134#endif
135
136static LCK_GRP_DECLARE(kq_lck_grp, "kqueue");
137SECURITY_READ_ONLY_EARLY(vm_packing_params_t) kn_kq_packing_params =
138 VM_PACKING_PARAMS(KNOTE_KQ_PACKED);
139
140extern mach_port_name_t ipc_entry_name_mask(mach_port_name_t name); /* osfmk/ipc/ipc_entry.h */
141extern int cansignal(struct proc *, kauth_cred_t, struct proc *, int); /* bsd/kern/kern_sig.c */
142
143#define KEV_EVTID(code) BSDDBG_CODE(DBG_BSD_KEVENT, (code))
144
145static int kqueue_select(struct fileproc *fp, int which, void *wq_link_id,
146 vfs_context_t ctx);
147static int kqueue_close(struct fileglob *fg, vfs_context_t ctx);
148static int kqueue_kqfilter(struct fileproc *fp, struct knote *kn,
149 struct kevent_qos_s *kev);
150static int kqueue_drain(struct fileproc *fp, vfs_context_t ctx);
151
152static const struct fileops kqueueops = {
153 .fo_type = DTYPE_KQUEUE,
154 .fo_read = fo_no_read,
155 .fo_write = fo_no_write,
156 .fo_ioctl = fo_no_ioctl,
157 .fo_select = kqueue_select,
158 .fo_close = kqueue_close,
159 .fo_drain = kqueue_drain,
160 .fo_kqfilter = kqueue_kqfilter,
161};
162
163static inline int kevent_modern_copyout(struct kevent_qos_s *, user_addr_t *);
164static int kevent_register_wait_prepare(struct knote *kn, struct kevent_qos_s *kev, int result);
165static void kevent_register_wait_block(struct turnstile *ts, thread_t handoff_thread,
166 thread_continue_t cont, struct _kevent_register *cont_args) __dead2;
167static void kevent_register_wait_return(struct _kevent_register *cont_args) __dead2;
168static void kevent_register_wait_cleanup(struct knote *kn);
169
170static struct kqtailq *kqueue_get_suppressed_queue(kqueue_t kq, struct knote *kn);
171static void kqueue_threadreq_initiate(struct kqueue *kq, workq_threadreq_t, kq_index_t qos, int flags);
172
173static void kqworkq_unbind(proc_t p, workq_threadreq_t);
174static thread_qos_t kqworkq_unbind_locked(struct kqworkq *kqwq, workq_threadreq_t, thread_t thread);
175static workq_threadreq_t kqworkq_get_request(struct kqworkq *kqwq, kq_index_t qos_index);
176static void kqueue_update_iotier_override(kqueue_t kqu);
177
178static void kqworkloop_unbind(struct kqworkloop *kwql);
179
180enum kqwl_unbind_locked_mode {
181 KQWL_OVERRIDE_DROP_IMMEDIATELY,
182 KQWL_OVERRIDE_DROP_DELAYED,
183};
184static void kqworkloop_unbind_locked(struct kqworkloop *kwql, thread_t thread,
185 enum kqwl_unbind_locked_mode how);
186static void kqworkloop_unbind_delayed_override_drop(thread_t thread);
187static kq_index_t kqworkloop_override(struct kqworkloop *kqwl);
188static void kqworkloop_set_overcommit(struct kqworkloop *kqwl);
189enum {
190 KQWL_UTQ_NONE,
191 /*
192 * The wakeup qos is the qos of QUEUED knotes.
193 *
194 * This QoS is accounted for with the events override in the
195 * kqr_override_index field. It is raised each time a new knote is queued at
196 * a given QoS. The kqwl_wakeup_qos field is a superset of the non empty
197 * knote buckets and is recomputed after each event delivery.
198 */
199 KQWL_UTQ_UPDATE_WAKEUP_QOS,
200 KQWL_UTQ_RECOMPUTE_WAKEUP_QOS,
201 KQWL_UTQ_UNBINDING, /* attempt to rebind */
202 KQWL_UTQ_PARKING,
203 /*
204 * The wakeup override is for suppressed knotes that have fired again at
205 * a higher QoS than the one for which they are suppressed already.
206 * This override is cleared when the knote suppressed list becomes empty.
207 */
208 KQWL_UTQ_UPDATE_WAKEUP_OVERRIDE,
209 KQWL_UTQ_RESET_WAKEUP_OVERRIDE,
210 /*
211 * The QoS is the maximum QoS of an event enqueued on this workloop in
212 * userland. It is copied from the only EVFILT_WORKLOOP knote with
213 * a NOTE_WL_THREAD_REQUEST bit set allowed on this workloop. If there is no
214 * such knote, this QoS is 0.
215 */
216 KQWL_UTQ_SET_QOS_INDEX,
217 KQWL_UTQ_REDRIVE_EVENTS,
218};
219static void kqworkloop_update_threads_qos(struct kqworkloop *kqwl, int op, kq_index_t qos);
220static int kqworkloop_end_processing(struct kqworkloop *kqwl, int flags, int kevent_flags);
221
222static struct knote *knote_alloc(void);
223static void knote_free(struct knote *kn);
224static int kq_add_knote(struct kqueue *kq, struct knote *kn,
225 struct knote_lock_ctx *knlc, struct proc *p);
226static struct knote *kq_find_knote_and_kq_lock(struct kqueue *kq,
227 struct kevent_qos_s *kev, bool is_fd, struct proc *p);
228
229static void knote_activate(kqueue_t kqu, struct knote *kn, int result);
230static void knote_dequeue(kqueue_t kqu, struct knote *kn);
231
232static void knote_apply_touch(kqueue_t kqu, struct knote *kn,
233 struct kevent_qos_s *kev, int result);
234static void knote_suppress(kqueue_t kqu, struct knote *kn);
235static void knote_unsuppress(kqueue_t kqu, struct knote *kn);
236static void knote_drop(kqueue_t kqu, struct knote *kn, struct knote_lock_ctx *knlc);
237
238// both these functions may dequeue the knote and it is up to the caller
239// to enqueue the knote back
240static void knote_adjust_qos(struct kqueue *kq, struct knote *kn, int result);
241static void knote_reset_priority(kqueue_t kqu, struct knote *kn, pthread_priority_t pp);
242
243static ZONE_DEFINE(knote_zone, "knote zone",
244 sizeof(struct knote), ZC_CACHING | ZC_ZFREE_CLEARMEM);
245static ZONE_DEFINE(kqfile_zone, "kqueue file zone",
246 sizeof(struct kqfile), ZC_ZFREE_CLEARMEM | ZC_NOTBITAG);
247static ZONE_DEFINE(kqworkq_zone, "kqueue workq zone",
248 sizeof(struct kqworkq), ZC_ZFREE_CLEARMEM | ZC_NOTBITAG);
249static ZONE_DEFINE(kqworkloop_zone, "kqueue workloop zone",
250 sizeof(struct kqworkloop), ZC_CACHING | ZC_ZFREE_CLEARMEM | ZC_NOTBITAG);
251
252#define KN_HASH(val, mask) (((val) ^ (val >> 8)) & (mask))
253
254static int filt_no_attach(struct knote *kn, struct kevent_qos_s *kev);
255static void filt_no_detach(struct knote *kn);
256static int filt_bad_event(struct knote *kn, long hint);
257static int filt_bad_touch(struct knote *kn, struct kevent_qos_s *kev);
258static int filt_bad_process(struct knote *kn, struct kevent_qos_s *kev);
259
260SECURITY_READ_ONLY_EARLY(static struct filterops) bad_filtops = {
261 .f_attach = filt_no_attach,
262 .f_detach = filt_no_detach,
263 .f_event = filt_bad_event,
264 .f_touch = filt_bad_touch,
265 .f_process = filt_bad_process,
266};
267
268#if CONFIG_MEMORYSTATUS
269extern const struct filterops memorystatus_filtops;
270#endif /* CONFIG_MEMORYSTATUS */
271extern const struct filterops fs_filtops;
272extern const struct filterops sig_filtops;
273extern const struct filterops machport_attach_filtops;
274extern const struct filterops mach_port_filtops;
275extern const struct filterops mach_port_set_filtops;
276extern const struct filterops pipe_nfiltops;
277extern const struct filterops pipe_rfiltops;
278extern const struct filterops pipe_wfiltops;
279extern const struct filterops ptsd_kqops;
280extern const struct filterops ptmx_kqops;
281extern const struct filterops soread_filtops;
282extern const struct filterops sowrite_filtops;
283extern const struct filterops sock_filtops;
284extern const struct filterops soexcept_filtops;
285extern const struct filterops spec_filtops;
286extern const struct filterops bpfread_filtops;
287extern const struct filterops necp_fd_rfiltops;
288#if SKYWALK
289extern const struct filterops skywalk_channel_rfiltops;
290extern const struct filterops skywalk_channel_wfiltops;
291extern const struct filterops skywalk_channel_efiltops;
292#endif /* SKYWALK */
293extern const struct filterops fsevent_filtops;
294extern const struct filterops vnode_filtops;
295extern const struct filterops tty_filtops;
296
297const static struct filterops file_filtops;
298const static struct filterops kqread_filtops;
299const static struct filterops proc_filtops;
300const static struct filterops timer_filtops;
301const static struct filterops user_filtops;
302const static struct filterops workloop_filtops;
303#if CONFIG_EXCLAVES
304extern const struct filterops exclaves_notification_filtops;
305#endif /* CONFIG_EXCLAVES */
306
307/*
308 *
309 * Rules for adding new filters to the system:
310 * Public filters:
311 * - Add a new "EVFILT_" option value to bsd/sys/event.h (typically a negative value)
312 * in the exported section of the header
313 * - Update the EVFILT_SYSCOUNT value to reflect the new addition
314 * - Add a filterops to the sysfilt_ops array. Public filters should be added at the end
315 * of the Public Filters section in the array.
316 * Private filters:
317 * - Add a new "EVFILT_" value to bsd/sys/event_private.h (typically a positive value)
318 * - Update the EVFILTID_MAX value to reflect the new addition
319 * - Add a filterops to the sysfilt_ops. Private filters should be added at the end of
320 * the Private filters section of the array.
321 */
322static_assert(EVFILTID_MAX < UINT8_MAX, "kn_filtid expects this to be true");
323static const struct filterops * const sysfilt_ops[EVFILTID_MAX] = {
324 /* Public Filters */
325 [~EVFILT_READ] = &file_filtops,
326 [~EVFILT_WRITE] = &file_filtops,
327 [~EVFILT_AIO] = &bad_filtops,
328 [~EVFILT_VNODE] = &file_filtops,
329 [~EVFILT_PROC] = &proc_filtops,
330 [~EVFILT_SIGNAL] = &sig_filtops,
331 [~EVFILT_TIMER] = &timer_filtops,
332 [~EVFILT_MACHPORT] = &machport_attach_filtops,
333 [~EVFILT_FS] = &fs_filtops,
334 [~EVFILT_USER] = &user_filtops,
335 [~EVFILT_UNUSED_11] = &bad_filtops,
336 [~EVFILT_VM] = &bad_filtops,
337 [~EVFILT_SOCK] = &file_filtops,
338#if CONFIG_MEMORYSTATUS
339 [~EVFILT_MEMORYSTATUS] = &memorystatus_filtops,
340#else
341 [~EVFILT_MEMORYSTATUS] = &bad_filtops,
342#endif
343 [~EVFILT_EXCEPT] = &file_filtops,
344#if SKYWALK
345 [~EVFILT_NW_CHANNEL] = &file_filtops,
346#else /* !SKYWALK */
347 [~EVFILT_NW_CHANNEL] = &bad_filtops,
348#endif /* !SKYWALK */
349 [~EVFILT_WORKLOOP] = &workloop_filtops,
350#if CONFIG_EXCLAVES
351 [~EVFILT_EXCLAVES_NOTIFICATION] = &exclaves_notification_filtops,
352#else /* !CONFIG_EXCLAVES */
353 [~EVFILT_EXCLAVES_NOTIFICATION] = &bad_filtops,
354#endif /* CONFIG_EXCLAVES*/
355
356 /* Private filters */
357 [EVFILTID_KQREAD] = &kqread_filtops,
358 [EVFILTID_PIPE_N] = &pipe_nfiltops,
359 [EVFILTID_PIPE_R] = &pipe_rfiltops,
360 [EVFILTID_PIPE_W] = &pipe_wfiltops,
361 [EVFILTID_PTSD] = &ptsd_kqops,
362 [EVFILTID_SOREAD] = &soread_filtops,
363 [EVFILTID_SOWRITE] = &sowrite_filtops,
364 [EVFILTID_SCK] = &sock_filtops,
365 [EVFILTID_SOEXCEPT] = &soexcept_filtops,
366 [EVFILTID_SPEC] = &spec_filtops,
367 [EVFILTID_BPFREAD] = &bpfread_filtops,
368 [EVFILTID_NECP_FD] = &necp_fd_rfiltops,
369#if SKYWALK
370 [EVFILTID_SKYWALK_CHANNEL_W] = &skywalk_channel_wfiltops,
371 [EVFILTID_SKYWALK_CHANNEL_R] = &skywalk_channel_rfiltops,
372 [EVFILTID_SKYWALK_CHANNEL_E] = &skywalk_channel_efiltops,
373#else /* !SKYWALK */
374 [EVFILTID_SKYWALK_CHANNEL_W] = &bad_filtops,
375 [EVFILTID_SKYWALK_CHANNEL_R] = &bad_filtops,
376 [EVFILTID_SKYWALK_CHANNEL_E] = &bad_filtops,
377#endif /* !SKYWALK */
378 [EVFILTID_FSEVENT] = &fsevent_filtops,
379 [EVFILTID_VN] = &vnode_filtops,
380 [EVFILTID_TTY] = &tty_filtops,
381 [EVFILTID_PTMX] = &ptmx_kqops,
382 [EVFILTID_MACH_PORT] = &mach_port_filtops,
383 [EVFILTID_MACH_PORT_SET] = &mach_port_set_filtops,
384
385 /* fake filter for detached knotes, keep last */
386 [EVFILTID_DETACHED] = &bad_filtops,
387};
388
389static inline bool
390kqr_thread_bound(workq_threadreq_t kqr)
391{
392 return kqr->tr_state == WORKQ_TR_STATE_BOUND;
393}
394
395static inline bool
396kqr_thread_requested_pending(workq_threadreq_t kqr)
397{
398 workq_tr_state_t tr_state = kqr->tr_state;
399 return tr_state > WORKQ_TR_STATE_IDLE && tr_state < WORKQ_TR_STATE_BOUND;
400}
401
402static inline bool
403kqr_thread_requested(workq_threadreq_t kqr)
404{
405 return kqr->tr_state != WORKQ_TR_STATE_IDLE;
406}
407
408static inline thread_t
409kqr_thread_fast(workq_threadreq_t kqr)
410{
411 assert(kqr_thread_bound(kqr));
412 return kqr->tr_thread;
413}
414
415static inline thread_t
416kqr_thread(workq_threadreq_t kqr)
417{
418 return kqr_thread_bound(kqr) ? kqr->tr_thread : THREAD_NULL;
419}
420
421static inline struct kqworkloop *
422kqr_kqworkloop(workq_threadreq_t kqr)
423{
424 if (kqr->tr_flags & WORKQ_TR_FLAG_WORKLOOP) {
425 return __container_of(kqr, struct kqworkloop, kqwl_request);
426 }
427 return NULL;
428}
429
430static inline kqueue_t
431kqr_kqueue(proc_t p, workq_threadreq_t kqr)
432{
433 kqueue_t kqu;
434 if (kqr->tr_flags & WORKQ_TR_FLAG_WORKLOOP) {
435 kqu.kqwl = kqr_kqworkloop(kqr);
436 } else {
437 kqu.kqwq = p->p_fd.fd_wqkqueue;
438 assert(kqr >= kqu.kqwq->kqwq_request &&
439 kqr < kqu.kqwq->kqwq_request + KQWQ_NBUCKETS);
440 }
441 return kqu;
442}
443
444#if CONFIG_PREADOPT_TG
445/* There are no guarantees about which locks are held when this is called */
446inline thread_group_qos_t
447kqr_preadopt_thread_group(workq_threadreq_t req)
448{
449 struct kqworkloop *kqwl = kqr_kqworkloop(kqr: req);
450 return kqwl ? os_atomic_load(&kqwl->kqwl_preadopt_tg, relaxed) : NULL;
451}
452
453/* There are no guarantees about which locks are held when this is called */
454inline _Atomic(thread_group_qos_t) *
455kqr_preadopt_thread_group_addr(workq_threadreq_t req)
456{
457 struct kqworkloop *kqwl = kqr_kqworkloop(kqr: req);
458 return kqwl ? (&kqwl->kqwl_preadopt_tg) : NULL;
459}
460#endif
461
462/*
463 * kqueue/note lock implementations
464 *
465 * The kqueue lock guards the kq state, the state of its queues,
466 * and the kqueue-aware status and locks of individual knotes.
467 *
468 * The kqueue workq lock is used to protect state guarding the
469 * interaction of the kqueue with the workq. This state cannot
470 * be guarded by the kq lock - as it needs to be taken when we
471 * already have the waitq set lock held (during the waitq hook
472 * callback). It might be better to use the waitq lock itself
473 * for this, but the IRQ requirements make that difficult).
474 *
475 * Knote flags, filter flags, and associated data are protected
476 * by the underlying object lock - and are only ever looked at
477 * by calling the filter to get a [consistent] snapshot of that
478 * data.
479 */
480
481static inline void
482kqlock(kqueue_t kqu)
483{
484 lck_spin_lock(lck: &kqu.kq->kq_lock);
485}
486
487static inline void
488kqlock_held(__assert_only kqueue_t kqu)
489{
490 LCK_SPIN_ASSERT(&kqu.kq->kq_lock, LCK_ASSERT_OWNED);
491}
492
493static inline void
494kqunlock(kqueue_t kqu)
495{
496 lck_spin_unlock(lck: &kqu.kq->kq_lock);
497}
498
499static inline void
500knhash_lock(struct filedesc *fdp)
501{
502 lck_mtx_lock(lck: &fdp->fd_knhashlock);
503}
504
505static inline void
506knhash_unlock(struct filedesc *fdp)
507{
508 lck_mtx_unlock(lck: &fdp->fd_knhashlock);
509}
510
511/* wait event for knote locks */
512static inline event_t
513knote_lock_wev(struct knote *kn)
514{
515 return (event_t)(&kn->kn_hook);
516}
517
518/* wait event for kevent_register_wait_* */
519static inline event64_t
520knote_filt_wev64(struct knote *kn)
521{
522 /* kdp_workloop_sync_wait_find_owner knows about this */
523 return CAST_EVENT64_T(kn);
524}
525
526/* wait event for knote_post/knote_drop */
527static inline event_t
528knote_post_wev(struct knote *kn)
529{
530 return &kn->kn_kevent;
531}
532
533/*!
534 * @function knote_has_qos
535 *
536 * @brief
537 * Whether the knote has a regular QoS.
538 *
539 * @discussion
540 * kn_qos_override is:
541 * - 0 on kqfiles
542 * - THREAD_QOS_LAST for special buckets (manager)
543 *
544 * Other values mean the knote participates to QoS propagation.
545 */
546static inline bool
547knote_has_qos(struct knote *kn)
548{
549 return kn->kn_qos_override > 0 && kn->kn_qos_override < THREAD_QOS_LAST;
550}
551
552#pragma mark knote locks
553
554/*
555 * Enum used by the knote_lock_* functions.
556 *
557 * KNOTE_KQ_LOCK_ALWAYS
558 * The function will always return with the kq lock held.
559 *
560 * KNOTE_KQ_LOCK_ON_SUCCESS
561 * The function will return with the kq lock held if it was successful
562 * (knote_lock() is the only function that can fail).
563 *
564 * KNOTE_KQ_LOCK_ON_FAILURE
565 * The function will return with the kq lock held if it was unsuccessful
566 * (knote_lock() is the only function that can fail).
567 *
568 * KNOTE_KQ_UNLOCK:
569 * The function returns with the kq unlocked.
570 */
571enum kqlocking {
572 KNOTE_KQ_LOCK_ALWAYS,
573 KNOTE_KQ_LOCK_ON_SUCCESS,
574 KNOTE_KQ_LOCK_ON_FAILURE,
575 KNOTE_KQ_UNLOCK,
576};
577
578static struct knote_lock_ctx *
579knote_lock_ctx_find(kqueue_t kqu, struct knote *kn)
580{
581 struct knote_lock_ctx *ctx;
582 LIST_FOREACH(ctx, &kqu.kq->kq_knlocks, knlc_link) {
583 if (ctx->knlc_knote == kn) {
584 return ctx;
585 }
586 }
587 panic("knote lock context not found: %p", kn);
588 __builtin_trap();
589}
590
591/* slowpath of knote_lock() */
592__attribute__((noinline))
593static bool __result_use_check
594knote_lock_slow(kqueue_t kqu, struct knote *kn,
595 struct knote_lock_ctx *knlc, int kqlocking)
596{
597 struct knote_lock_ctx *owner_lc;
598 struct uthread *uth = current_uthread();
599 wait_result_t wr;
600
601 kqlock_held(kqu);
602
603 owner_lc = knote_lock_ctx_find(kqu, kn);
604#if DEBUG || DEVELOPMENT
605 knlc->knlc_state = KNOTE_LOCK_CTX_WAITING;
606#endif
607 owner_lc->knlc_waiters++;
608
609 /*
610 * Make our lock context visible to knote_unlock()
611 */
612 uth->uu_knlock = knlc;
613
614 wr = lck_spin_sleep_with_inheritor(lock: &kqu.kq->kq_lock, lck_sleep_action: LCK_SLEEP_UNLOCK,
615 event: knote_lock_wev(kn), inheritor: owner_lc->knlc_thread,
616 THREAD_UNINT | THREAD_WAIT_NOREPORT, TIMEOUT_WAIT_FOREVER);
617
618 if (wr == THREAD_RESTART) {
619 /*
620 * We haven't been woken up by knote_unlock() but knote_unlock_cancel.
621 * We need to cleanup the state since no one did.
622 */
623 uth->uu_knlock = NULL;
624#if DEBUG || DEVELOPMENT
625 assert(knlc->knlc_state == KNOTE_LOCK_CTX_WAITING);
626 knlc->knlc_state = KNOTE_LOCK_CTX_UNLOCKED;
627#endif
628
629 if (kqlocking == KNOTE_KQ_LOCK_ALWAYS ||
630 kqlocking == KNOTE_KQ_LOCK_ON_FAILURE) {
631 kqlock(kqu);
632 }
633 return false;
634 } else {
635 if (kqlocking == KNOTE_KQ_LOCK_ALWAYS ||
636 kqlocking == KNOTE_KQ_LOCK_ON_SUCCESS) {
637 kqlock(kqu);
638#if DEBUG || DEVELOPMENT
639 /*
640 * This state is set under the lock so we can't
641 * really assert this unless we hold the lock.
642 */
643 assert(knlc->knlc_state == KNOTE_LOCK_CTX_LOCKED);
644#endif
645 }
646 return true;
647 }
648}
649
650/*
651 * Attempts to take the "knote" lock.
652 *
653 * Called with the kqueue lock held.
654 *
655 * Returns true if the knote lock is acquired, false if it has been dropped
656 */
657static bool __result_use_check
658knote_lock(kqueue_t kqu, struct knote *kn, struct knote_lock_ctx *knlc,
659 enum kqlocking kqlocking)
660{
661 kqlock_held(kqu);
662
663#if DEBUG || DEVELOPMENT
664 assert(knlc->knlc_state == KNOTE_LOCK_CTX_UNLOCKED);
665#endif
666 knlc->knlc_knote = kn;
667 knlc->knlc_thread = current_thread();
668 knlc->knlc_waiters = 0;
669
670 if (__improbable(kn->kn_status & KN_LOCKED)) {
671 return knote_lock_slow(kqu, kn, knlc, kqlocking);
672 }
673
674 /*
675 * When the knote will be dropped, the knote lock is taken before
676 * KN_DROPPING is set, and then the knote will be removed from any
677 * hash table that references it before the lock is canceled.
678 */
679 assert((kn->kn_status & KN_DROPPING) == 0);
680 LIST_INSERT_HEAD(&kqu.kq->kq_knlocks, knlc, knlc_link);
681 kn->kn_status |= KN_LOCKED;
682#if DEBUG || DEVELOPMENT
683 knlc->knlc_state = KNOTE_LOCK_CTX_LOCKED;
684#endif
685
686 if (kqlocking == KNOTE_KQ_UNLOCK ||
687 kqlocking == KNOTE_KQ_LOCK_ON_FAILURE) {
688 kqunlock(kqu);
689 }
690 return true;
691}
692
693/*
694 * Unlocks a knote successfully locked with knote_lock().
695 *
696 * Called with the kqueue lock held.
697 *
698 * Returns with the kqueue lock held according to KNOTE_KQ_* mode.
699 */
700static void
701knote_unlock(kqueue_t kqu, struct knote *kn,
702 struct knote_lock_ctx *knlc, enum kqlocking kqlocking)
703{
704 kqlock_held(kqu);
705
706 assert(knlc->knlc_knote == kn);
707 assert(kn->kn_status & KN_LOCKED);
708#if DEBUG || DEVELOPMENT
709 assert(knlc->knlc_state == KNOTE_LOCK_CTX_LOCKED);
710#endif
711
712 LIST_REMOVE(knlc, knlc_link);
713
714 if (knlc->knlc_waiters) {
715 thread_t thread = THREAD_NULL;
716
717 wakeup_one_with_inheritor(event: knote_lock_wev(kn), THREAD_AWAKENED,
718 action: LCK_WAKE_DEFAULT, thread_wokenup: &thread);
719
720 /*
721 * knote_lock_slow() publishes the lock context of waiters
722 * in uthread::uu_knlock.
723 *
724 * Reach out and make this context the new owner.
725 */
726 struct uthread *ut = get_bsdthread_info(thread);
727 struct knote_lock_ctx *next_owner_lc = ut->uu_knlock;
728
729 assert(next_owner_lc->knlc_knote == kn);
730 next_owner_lc->knlc_waiters = knlc->knlc_waiters - 1;
731 LIST_INSERT_HEAD(&kqu.kq->kq_knlocks, next_owner_lc, knlc_link);
732#if DEBUG || DEVELOPMENT
733 next_owner_lc->knlc_state = KNOTE_LOCK_CTX_LOCKED;
734#endif
735 ut->uu_knlock = NULL;
736 thread_deallocate_safe(thread);
737 } else {
738 kn->kn_status &= ~KN_LOCKED;
739 }
740
741 if ((kn->kn_status & KN_MERGE_QOS) && !(kn->kn_status & KN_POSTING)) {
742 /*
743 * No f_event() in flight anymore, we can leave QoS "Merge" mode
744 *
745 * See knote_adjust_qos()
746 */
747 kn->kn_status &= ~KN_MERGE_QOS;
748 }
749 if (kqlocking == KNOTE_KQ_UNLOCK) {
750 kqunlock(kqu);
751 }
752#if DEBUG || DEVELOPMENT
753 knlc->knlc_state = KNOTE_LOCK_CTX_UNLOCKED;
754#endif
755}
756
757/*
758 * Aborts all waiters for a knote lock, and unlock the knote.
759 *
760 * Called with the kqueue lock held.
761 *
762 * Returns with the kqueue unlocked.
763 */
764static void
765knote_unlock_cancel(struct kqueue *kq, struct knote *kn,
766 struct knote_lock_ctx *knlc)
767{
768 kqlock_held(kqu: kq);
769
770 assert(knlc->knlc_knote == kn);
771 assert(kn->kn_status & KN_LOCKED);
772 assert(kn->kn_status & KN_DROPPING);
773
774 LIST_REMOVE(knlc, knlc_link);
775 kn->kn_status &= ~KN_LOCKED;
776 kqunlock(kqu: kq);
777
778 if (knlc->knlc_waiters) {
779 wakeup_all_with_inheritor(event: knote_lock_wev(kn), THREAD_RESTART);
780 }
781#if DEBUG || DEVELOPMENT
782 knlc->knlc_state = KNOTE_LOCK_CTX_UNLOCKED;
783#endif
784}
785
786/*
787 * Call the f_event hook of a given filter.
788 *
789 * Takes a use count to protect against concurrent drops.
790 * Called with the object lock held.
791 */
792static void
793knote_post(struct knote *kn, long hint)
794{
795 struct kqueue *kq = knote_get_kq(kn);
796 int dropping, result;
797
798 kqlock(kqu: kq);
799
800 if (__improbable(kn->kn_status & (KN_DROPPING | KN_VANISHED))) {
801 return kqunlock(kqu: kq);
802 }
803
804 if (__improbable(kn->kn_status & KN_POSTING)) {
805 panic("KNOTE() called concurrently on knote %p", kn);
806 }
807
808 kn->kn_status |= KN_POSTING;
809
810 kqunlock(kqu: kq);
811 result = filter_call(knote_fops(kn), f_event(kn, hint));
812 kqlock(kqu: kq);
813
814 /* Someone dropped the knote/the monitored object vanished while we
815 * were in f_event, swallow the side effects of the post.
816 */
817 dropping = (kn->kn_status & (KN_DROPPING | KN_VANISHED));
818
819 if (!dropping && (result & FILTER_ADJUST_EVENT_IOTIER_BIT)) {
820 kqueue_update_iotier_override(kqu: kq);
821 }
822
823 if (!dropping && (result & FILTER_ACTIVE)) {
824 knote_activate(kqu: kq, kn, result);
825 }
826
827 if ((kn->kn_status & KN_LOCKED) == 0) {
828 /*
829 * There's no other f_* call in flight, we can leave QoS "Merge" mode.
830 *
831 * See knote_adjust_qos()
832 */
833 kn->kn_status &= ~(KN_POSTING | KN_MERGE_QOS);
834 } else {
835 kn->kn_status &= ~KN_POSTING;
836 }
837
838 if (__improbable(dropping)) {
839 thread_wakeup(knote_post_wev(kn));
840 }
841
842 kqunlock(kqu: kq);
843}
844
845/*
846 * Called by knote_drop() and knote_fdclose() to wait for the last f_event()
847 * caller to be done.
848 *
849 * - kq locked at entry
850 * - kq unlocked at exit
851 */
852static void
853knote_wait_for_post(struct kqueue *kq, struct knote *kn)
854{
855 kqlock_held(kqu: kq);
856
857 assert(kn->kn_status & (KN_DROPPING | KN_VANISHED));
858
859 if (kn->kn_status & KN_POSTING) {
860 lck_spin_sleep(lck: &kq->kq_lock, lck_sleep_action: LCK_SLEEP_UNLOCK, event: knote_post_wev(kn),
861 THREAD_UNINT | THREAD_WAIT_NOREPORT);
862 } else {
863 kqunlock(kqu: kq);
864 }
865}
866
867#pragma mark knote helpers for filters
868
869OS_ALWAYS_INLINE
870void *
871knote_kn_hook_get_raw(struct knote *kn)
872{
873 uintptr_t *addr = &kn->kn_hook;
874
875 void *hook = (void *) *addr;
876#if __has_feature(ptrauth_calls)
877 if (hook) {
878 uint16_t blend = kn->kn_filter;
879 blend |= (kn->kn_filtid << 8);
880 blend ^= OS_PTRAUTH_DISCRIMINATOR("kn.kn_hook");
881
882 hook = ptrauth_auth_data(hook, ptrauth_key_process_independent_data,
883 ptrauth_blend_discriminator(addr, blend));
884 }
885#endif
886
887 return hook;
888}
889
890OS_ALWAYS_INLINE void
891knote_kn_hook_set_raw(struct knote *kn, void *kn_hook)
892{
893 uintptr_t *addr = &kn->kn_hook;
894#if __has_feature(ptrauth_calls)
895 if (kn_hook) {
896 uint16_t blend = kn->kn_filter;
897 blend |= (kn->kn_filtid << 8);
898 blend ^= OS_PTRAUTH_DISCRIMINATOR("kn.kn_hook");
899
900 kn_hook = ptrauth_sign_unauthenticated(kn_hook,
901 ptrauth_key_process_independent_data,
902 ptrauth_blend_discriminator(addr, blend));
903 }
904#endif
905 *addr = (uintptr_t) kn_hook;
906}
907
908OS_ALWAYS_INLINE
909void
910knote_set_error(struct knote *kn, int error)
911{
912 kn->kn_flags |= EV_ERROR;
913 kn->kn_sdata = error;
914}
915
916OS_ALWAYS_INLINE
917int64_t
918knote_low_watermark(const struct knote *kn)
919{
920 return (kn->kn_sfflags & NOTE_LOWAT) ? kn->kn_sdata : 1;
921}
922
923/*!
924 * @function knote_fill_kevent_with_sdata
925 *
926 * @brief
927 * Fills in a kevent from the current content of a knote.
928 *
929 * @discussion
930 * This is meant to be called from filter's f_process hooks.
931 * The kevent data is filled with kn->kn_sdata.
932 *
933 * kn->kn_fflags is cleared if kn->kn_flags has EV_CLEAR set.
934 *
935 * Using knote_fill_kevent is typically preferred.
936 */
937OS_ALWAYS_INLINE
938void
939knote_fill_kevent_with_sdata(struct knote *kn, struct kevent_qos_s *kev)
940{
941#define knote_assert_aliases(name1, offs1, name2) \
942 static_assert(offsetof(struct kevent_qos_s, name1) + offs1 == \
943 offsetof(struct kevent_internal_s, name2), \
944 "kevent_qos_s::" #name1 " and kevent_internal_s::" #name2 "need to alias")
945 /*
946 * All the code makes assumptions on these aliasing,
947 * so make sure we fail the build if we ever ever ever break them.
948 */
949 knote_assert_aliases(ident, 0, kei_ident);
950#ifdef __LITTLE_ENDIAN__
951 knote_assert_aliases(filter, 0, kei_filter); // non trivial overlap
952 knote_assert_aliases(filter, 1, kei_filtid); // non trivial overlap
953#else
954 knote_assert_aliases(filter, 0, kei_filtid); // non trivial overlap
955 knote_assert_aliases(filter, 1, kei_filter); // non trivial overlap
956#endif
957 knote_assert_aliases(flags, 0, kei_flags);
958 knote_assert_aliases(qos, 0, kei_qos);
959 knote_assert_aliases(udata, 0, kei_udata);
960 knote_assert_aliases(fflags, 0, kei_fflags);
961 knote_assert_aliases(xflags, 0, kei_sfflags); // non trivial overlap
962 knote_assert_aliases(data, 0, kei_sdata); // non trivial overlap
963 knote_assert_aliases(ext, 0, kei_ext);
964#undef knote_assert_aliases
965
966 /*
967 * Fix the differences between kevent_qos_s and kevent_internal_s:
968 * - xflags is where kn_sfflags lives, we need to zero it
969 * - fixup the high bits of `filter` where kn_filtid lives
970 */
971 *kev = *(struct kevent_qos_s *)&kn->kn_kevent;
972 kev->xflags = 0;
973 kev->filter |= 0xff00;
974 if (kn->kn_flags & EV_CLEAR) {
975 kn->kn_fflags = 0;
976 }
977}
978
979/*!
980 * @function knote_fill_kevent
981 *
982 * @brief
983 * Fills in a kevent from the current content of a knote.
984 *
985 * @discussion
986 * This is meant to be called from filter's f_process hooks.
987 * The kevent data is filled with the passed in data.
988 *
989 * kn->kn_fflags is cleared if kn->kn_flags has EV_CLEAR set.
990 */
991OS_ALWAYS_INLINE
992void
993knote_fill_kevent(struct knote *kn, struct kevent_qos_s *kev, int64_t data)
994{
995 knote_fill_kevent_with_sdata(kn, kev);
996 kev->filter = kn->kn_filter;
997 kev->data = data;
998}
999
1000
1001#pragma mark file_filtops
1002
1003static int
1004filt_fileattach(struct knote *kn, struct kevent_qos_s *kev)
1005{
1006 return fo_kqfilter(fp: kn->kn_fp, kn, kev);
1007}
1008
1009SECURITY_READ_ONLY_EARLY(static struct filterops) file_filtops = {
1010 .f_isfd = 1,
1011 .f_attach = filt_fileattach,
1012};
1013
1014#pragma mark kqread_filtops
1015
1016#define f_flag fp_glob->fg_flag
1017#define f_ops fp_glob->fg_ops
1018#define f_lflags fp_glob->fg_lflags
1019
1020static void
1021filt_kqdetach(struct knote *kn)
1022{
1023 struct kqfile *kqf = (struct kqfile *)fp_get_data(fp: kn->kn_fp);
1024 struct kqueue *kq = &kqf->kqf_kqueue;
1025
1026 kqlock(kqu: kq);
1027 KNOTE_DETACH(&kqf->kqf_sel.si_note, kn);
1028 kqunlock(kqu: kq);
1029}
1030
1031static int
1032filt_kqueue(struct knote *kn, __unused long hint)
1033{
1034 struct kqueue *kq = (struct kqueue *)fp_get_data(fp: kn->kn_fp);
1035
1036 return kq->kq_count > 0;
1037}
1038
1039static int
1040filt_kqtouch(struct knote *kn, struct kevent_qos_s *kev)
1041{
1042#pragma unused(kev)
1043 struct kqueue *kq = (struct kqueue *)fp_get_data(fp: kn->kn_fp);
1044 int res;
1045
1046 kqlock(kqu: kq);
1047 res = (kq->kq_count > 0);
1048 kqunlock(kqu: kq);
1049
1050 return res;
1051}
1052
1053static int
1054filt_kqprocess(struct knote *kn, struct kevent_qos_s *kev)
1055{
1056 struct kqueue *kq = (struct kqueue *)fp_get_data(fp: kn->kn_fp);
1057 int res = 0;
1058
1059 kqlock(kqu: kq);
1060 if (kq->kq_count) {
1061 knote_fill_kevent(kn, kev, data: kq->kq_count);
1062 res = 1;
1063 }
1064 kqunlock(kqu: kq);
1065
1066 return res;
1067}
1068
1069SECURITY_READ_ONLY_EARLY(static struct filterops) kqread_filtops = {
1070 .f_isfd = 1,
1071 .f_detach = filt_kqdetach,
1072 .f_event = filt_kqueue,
1073 .f_touch = filt_kqtouch,
1074 .f_process = filt_kqprocess,
1075};
1076
1077#pragma mark proc_filtops
1078
1079static int
1080filt_procattach(struct knote *kn, __unused struct kevent_qos_s *kev)
1081{
1082 struct proc *p;
1083
1084 assert(PID_MAX < NOTE_PDATAMASK);
1085
1086 if ((kn->kn_sfflags & (NOTE_TRACK | NOTE_TRACKERR | NOTE_CHILD)) != 0) {
1087 knote_set_error(kn, ENOTSUP);
1088 return 0;
1089 }
1090
1091 p = proc_find(pid: (int)kn->kn_id);
1092 if (p == NULL) {
1093 knote_set_error(kn, ESRCH);
1094 return 0;
1095 }
1096
1097 const uint32_t NoteExitStatusBits = NOTE_EXIT | NOTE_EXITSTATUS;
1098
1099 if ((kn->kn_sfflags & NoteExitStatusBits) == NoteExitStatusBits) {
1100 do {
1101 pid_t selfpid = proc_selfpid();
1102
1103 if (p->p_ppid == selfpid) {
1104 break; /* parent => ok */
1105 }
1106 if ((p->p_lflag & P_LTRACED) != 0 &&
1107 (p->p_oppid == selfpid)) {
1108 break; /* parent-in-waiting => ok */
1109 }
1110 if (cansignal(current_proc(), kauth_cred_get(), p, SIGKILL)) {
1111 break; /* allowed to signal => ok */
1112 }
1113 proc_rele(p);
1114 knote_set_error(kn, EACCES);
1115 return 0;
1116 } while (0);
1117 }
1118
1119 kn->kn_proc = p;
1120 kn->kn_flags |= EV_CLEAR; /* automatically set */
1121 kn->kn_sdata = 0; /* incoming data is ignored */
1122
1123 proc_klist_lock();
1124
1125 KNOTE_ATTACH(&p->p_klist, kn);
1126
1127 proc_klist_unlock();
1128
1129 proc_rele(p);
1130
1131 /*
1132 * only captures edge-triggered events after this point
1133 * so it can't already be fired.
1134 */
1135 return 0;
1136}
1137
1138
1139/*
1140 * The knote may be attached to a different process, which may exit,
1141 * leaving nothing for the knote to be attached to. In that case,
1142 * the pointer to the process will have already been nulled out.
1143 */
1144static void
1145filt_procdetach(struct knote *kn)
1146{
1147 struct proc *p;
1148
1149 proc_klist_lock();
1150
1151 p = kn->kn_proc;
1152 if (p != PROC_NULL) {
1153 kn->kn_proc = PROC_NULL;
1154 KNOTE_DETACH(&p->p_klist, kn);
1155 }
1156
1157 proc_klist_unlock();
1158}
1159
1160static int
1161filt_procevent(struct knote *kn, long hint)
1162{
1163 u_int event;
1164
1165 /* ALWAYS CALLED WITH proc_klist_lock */
1166
1167 /*
1168 * Note: a lot of bits in hint may be obtained from the knote
1169 * To free some of those bits, see <rdar://problem/12592988> Freeing up
1170 * bits in hint for filt_procevent
1171 *
1172 * mask off extra data
1173 */
1174 event = (u_int)hint & NOTE_PCTRLMASK;
1175
1176 /*
1177 * termination lifecycle events can happen while a debugger
1178 * has reparented a process, in which case notifications
1179 * should be quashed except to the tracing parent. When
1180 * the debugger reaps the child (either via wait4(2) or
1181 * process exit), the child will be reparented to the original
1182 * parent and these knotes re-fired.
1183 */
1184 if (event & NOTE_EXIT) {
1185 if ((kn->kn_proc->p_oppid != 0)
1186 && (proc_getpid(knote_get_kq(kn)->kq_p) != kn->kn_proc->p_ppid)) {
1187 /*
1188 * This knote is not for the current ptrace(2) parent, ignore.
1189 */
1190 return 0;
1191 }
1192 }
1193
1194 /*
1195 * if the user is interested in this event, record it.
1196 */
1197 if (kn->kn_sfflags & event) {
1198 kn->kn_fflags |= event;
1199 }
1200
1201#pragma clang diagnostic push
1202#pragma clang diagnostic ignored "-Wdeprecated-declarations"
1203 if ((event == NOTE_REAP) || ((event == NOTE_EXIT) && !(kn->kn_sfflags & NOTE_REAP))) {
1204 kn->kn_flags |= (EV_EOF | EV_ONESHOT);
1205 }
1206#pragma clang diagnostic pop
1207
1208
1209 /*
1210 * The kernel has a wrapper in place that returns the same data
1211 * as is collected here, in kn_hook32. Any changes to how
1212 * NOTE_EXITSTATUS and NOTE_EXIT_DETAIL are collected
1213 * should also be reflected in the proc_pidnoteexit() wrapper.
1214 */
1215 if (event == NOTE_EXIT) {
1216 kn->kn_hook32 = 0;
1217 if ((kn->kn_sfflags & NOTE_EXITSTATUS) != 0) {
1218 kn->kn_fflags |= NOTE_EXITSTATUS;
1219 kn->kn_hook32 |= (hint & NOTE_PDATAMASK);
1220 }
1221 if ((kn->kn_sfflags & NOTE_EXIT_DETAIL) != 0) {
1222 kn->kn_fflags |= NOTE_EXIT_DETAIL;
1223 if ((kn->kn_proc->p_lflag &
1224 P_LTERM_DECRYPTFAIL) != 0) {
1225 kn->kn_hook32 |= NOTE_EXIT_DECRYPTFAIL;
1226 }
1227 if ((kn->kn_proc->p_lflag &
1228 P_LTERM_JETSAM) != 0) {
1229 kn->kn_hook32 |= NOTE_EXIT_MEMORY;
1230 switch (kn->kn_proc->p_lflag & P_JETSAM_MASK) {
1231 case P_JETSAM_VMPAGESHORTAGE:
1232 kn->kn_hook32 |= NOTE_EXIT_MEMORY_VMPAGESHORTAGE;
1233 break;
1234 case P_JETSAM_VMTHRASHING:
1235 kn->kn_hook32 |= NOTE_EXIT_MEMORY_VMTHRASHING;
1236 break;
1237 case P_JETSAM_FCTHRASHING:
1238 kn->kn_hook32 |= NOTE_EXIT_MEMORY_FCTHRASHING;
1239 break;
1240 case P_JETSAM_VNODE:
1241 kn->kn_hook32 |= NOTE_EXIT_MEMORY_VNODE;
1242 break;
1243 case P_JETSAM_HIWAT:
1244 kn->kn_hook32 |= NOTE_EXIT_MEMORY_HIWAT;
1245 break;
1246 case P_JETSAM_PID:
1247 kn->kn_hook32 |= NOTE_EXIT_MEMORY_PID;
1248 break;
1249 case P_JETSAM_IDLEEXIT:
1250 kn->kn_hook32 |= NOTE_EXIT_MEMORY_IDLE;
1251 break;
1252 }
1253 }
1254 if ((proc_getcsflags(kn->kn_proc) &
1255 CS_KILLED) != 0) {
1256 kn->kn_hook32 |= NOTE_EXIT_CSERROR;
1257 }
1258 }
1259 }
1260
1261 /* if we have any matching state, activate the knote */
1262 return kn->kn_fflags != 0;
1263}
1264
1265static int
1266filt_proctouch(struct knote *kn, struct kevent_qos_s *kev)
1267{
1268 int res;
1269
1270 proc_klist_lock();
1271
1272 /* accept new filter flags and mask off output events no long interesting */
1273 kn->kn_sfflags = kev->fflags;
1274
1275 /* restrict the current results to the (smaller?) set of new interest */
1276 /*
1277 * For compatibility with previous implementations, we leave kn_fflags
1278 * as they were before.
1279 */
1280 //kn->kn_fflags &= kn->kn_sfflags;
1281
1282 res = (kn->kn_fflags != 0);
1283
1284 proc_klist_unlock();
1285
1286 return res;
1287}
1288
1289static int
1290filt_procprocess(struct knote *kn, struct kevent_qos_s *kev)
1291{
1292 int res = 0;
1293
1294 proc_klist_lock();
1295 if (kn->kn_fflags) {
1296 knote_fill_kevent(kn, kev, data: kn->kn_hook32);
1297 kn->kn_hook32 = 0;
1298 res = 1;
1299 }
1300 proc_klist_unlock();
1301 return res;
1302}
1303
1304SECURITY_READ_ONLY_EARLY(static struct filterops) proc_filtops = {
1305 .f_attach = filt_procattach,
1306 .f_detach = filt_procdetach,
1307 .f_event = filt_procevent,
1308 .f_touch = filt_proctouch,
1309 .f_process = filt_procprocess,
1310};
1311
1312#pragma mark timer_filtops
1313
1314struct filt_timer_params {
1315 uint64_t deadline; /* deadline in abs/cont time
1316 * (or 0 if NOTE_ABSOLUTE and deadline is in past) */
1317 uint64_t leeway; /* leeway in abstime, or 0 if none */
1318 uint64_t interval; /* interval in abstime or 0 if non-repeating timer */
1319};
1320
1321/*
1322 * Values stored in the knote at rest (using Mach absolute time units)
1323 *
1324 * kn->kn_thcall where the thread_call object is stored
1325 * kn->kn_ext[0] next deadline or 0 if immediate expiration
1326 * kn->kn_ext[1] leeway value
1327 * kn->kn_sdata interval timer: the interval
1328 * absolute/deadline timer: 0
1329 * kn->kn_hook32 timer state (with gencount)
1330 *
1331 * TIMER_IDLE:
1332 * The timer has either never been scheduled or been cancelled.
1333 * It is safe to schedule a new one in this state.
1334 *
1335 * TIMER_ARMED:
1336 * The timer has been scheduled
1337 *
1338 * TIMER_FIRED
1339 * The timer has fired and an event needs to be delivered.
1340 * When in this state, the callout may still be running.
1341 *
1342 * TIMER_IMMEDIATE
1343 * The timer has fired at registration time, and the callout was never
1344 * dispatched.
1345 */
1346#define TIMER_IDLE 0x0
1347#define TIMER_ARMED 0x1
1348#define TIMER_FIRED 0x2
1349#define TIMER_IMMEDIATE 0x3
1350#define TIMER_STATE_MASK 0x3
1351#define TIMER_GEN_INC 0x4
1352
1353static void
1354filt_timer_set_params(struct knote *kn, struct filt_timer_params *params)
1355{
1356 kn->kn_ext[0] = params->deadline;
1357 kn->kn_ext[1] = params->leeway;
1358 kn->kn_sdata = params->interval;
1359}
1360
1361/*
1362 * filt_timervalidate - process data from user
1363 *
1364 * Sets up the deadline, interval, and leeway from the provided user data
1365 *
1366 * Input:
1367 * kn_sdata timer deadline or interval time
1368 * kn_sfflags style of timer, unit of measurement
1369 *
1370 * Output:
1371 * struct filter_timer_params to apply to the filter with
1372 * filt_timer_set_params when changes are ready to be commited.
1373 *
1374 * Returns:
1375 * EINVAL Invalid user data parameters
1376 * ERANGE Various overflows with the parameters
1377 *
1378 * Called with timer filter lock held.
1379 */
1380static int
1381filt_timervalidate(const struct kevent_qos_s *kev,
1382 struct filt_timer_params *params)
1383{
1384 /*
1385 * There are 5 knobs that need to be chosen for a timer registration:
1386 *
1387 * A) Units of time (what is the time duration of the specified number)
1388 * Absolute and interval take:
1389 * NOTE_SECONDS, NOTE_USECONDS, NOTE_NSECONDS, NOTE_MACHTIME
1390 * Defaults to milliseconds if not specified
1391 *
1392 * B) Clock epoch (what is the zero point of the specified number)
1393 * For interval, there is none
1394 * For absolute, defaults to the gettimeofday/calendar epoch
1395 * With NOTE_MACHTIME, uses mach_absolute_time()
1396 * With NOTE_MACHTIME and NOTE_MACH_CONTINUOUS_TIME, uses mach_continuous_time()
1397 *
1398 * C) The knote's behavior on delivery
1399 * Interval timer causes the knote to arm for the next interval unless one-shot is set
1400 * Absolute is a forced one-shot timer which deletes on delivery
1401 * TODO: Add a way for absolute to be not forced one-shot
1402 *
1403 * D) Whether the time duration is relative to now or absolute
1404 * Interval fires at now + duration when it is set up
1405 * Absolute fires at now + difference between now walltime and passed in walltime
1406 * With NOTE_MACHTIME it fires at an absolute MAT or MCT.
1407 *
1408 * E) Whether the timer continues to tick across sleep
1409 * By default all three do not.
1410 * For interval and absolute, NOTE_MACH_CONTINUOUS_TIME causes them to tick across sleep
1411 * With NOTE_ABSOLUTE | NOTE_MACHTIME | NOTE_MACH_CONTINUOUS_TIME:
1412 * expires when mach_continuous_time() is > the passed in value.
1413 */
1414
1415 uint64_t multiplier;
1416
1417 boolean_t use_abstime = FALSE;
1418
1419 switch (kev->fflags & (NOTE_SECONDS | NOTE_USECONDS | NOTE_NSECONDS | NOTE_MACHTIME)) {
1420 case NOTE_SECONDS:
1421 multiplier = NSEC_PER_SEC;
1422 break;
1423 case NOTE_USECONDS:
1424 multiplier = NSEC_PER_USEC;
1425 break;
1426 case NOTE_NSECONDS:
1427 multiplier = 1;
1428 break;
1429 case NOTE_MACHTIME:
1430 multiplier = 0;
1431 use_abstime = TRUE;
1432 break;
1433 case 0: /* milliseconds (default) */
1434 multiplier = NSEC_PER_SEC / 1000;
1435 break;
1436 default:
1437 return EINVAL;
1438 }
1439
1440 /* transform the leeway in kn_ext[1] to same time scale */
1441 if (kev->fflags & NOTE_LEEWAY) {
1442 uint64_t leeway_abs;
1443
1444 if (use_abstime) {
1445 leeway_abs = (uint64_t)kev->ext[1];
1446 } else {
1447 uint64_t leeway_ns;
1448 if (os_mul_overflow((uint64_t)kev->ext[1], multiplier, &leeway_ns)) {
1449 return ERANGE;
1450 }
1451
1452 nanoseconds_to_absolutetime(nanoseconds: leeway_ns, result: &leeway_abs);
1453 }
1454
1455 params->leeway = leeway_abs;
1456 } else {
1457 params->leeway = 0;
1458 }
1459
1460 if (kev->fflags & NOTE_ABSOLUTE) {
1461 uint64_t deadline_abs;
1462
1463 if (use_abstime) {
1464 deadline_abs = (uint64_t)kev->data;
1465 } else {
1466 uint64_t calendar_deadline_ns;
1467
1468 if (os_mul_overflow((uint64_t)kev->data, multiplier, &calendar_deadline_ns)) {
1469 return ERANGE;
1470 }
1471
1472 /* calendar_deadline_ns is in nanoseconds since the epoch */
1473
1474 clock_sec_t seconds;
1475 clock_nsec_t nanoseconds;
1476
1477 /*
1478 * Note that the conversion through wall-time is only done once.
1479 *
1480 * If the relationship between MAT and gettimeofday changes,
1481 * the underlying timer does not update.
1482 *
1483 * TODO: build a wall-time denominated timer_call queue
1484 * and a flag to request DTRTing with wall-time timers
1485 */
1486 clock_get_calendar_nanotime(secs: &seconds, nanosecs: &nanoseconds);
1487
1488 uint64_t calendar_now_ns = (uint64_t)seconds * NSEC_PER_SEC + nanoseconds;
1489
1490 /* if deadline is in the future */
1491 if (calendar_now_ns < calendar_deadline_ns) {
1492 uint64_t interval_ns = calendar_deadline_ns - calendar_now_ns;
1493 uint64_t interval_abs;
1494
1495 nanoseconds_to_absolutetime(nanoseconds: interval_ns, result: &interval_abs);
1496
1497 /*
1498 * Note that the NOTE_MACH_CONTINUOUS_TIME flag here only
1499 * causes the timer to keep ticking across sleep, but
1500 * it does not change the calendar timebase.
1501 */
1502
1503 if (kev->fflags & NOTE_MACH_CONTINUOUS_TIME) {
1504 clock_continuoustime_interval_to_deadline(abstime: interval_abs,
1505 result: &deadline_abs);
1506 } else {
1507 clock_absolutetime_interval_to_deadline(abstime: interval_abs,
1508 result: &deadline_abs);
1509 }
1510 } else {
1511 deadline_abs = 0; /* cause immediate expiration */
1512 }
1513 }
1514
1515 params->deadline = deadline_abs;
1516 params->interval = 0; /* NOTE_ABSOLUTE is non-repeating */
1517 } else if (kev->data < 0) {
1518 /*
1519 * Negative interval timers fire immediately, once.
1520 *
1521 * Ideally a negative interval would be an error, but certain clients
1522 * pass negative values on accident, and expect an event back.
1523 *
1524 * In the old implementation the timer would repeat with no delay
1525 * N times until mach_absolute_time() + (N * interval) underflowed,
1526 * then it would wait ~forever by accidentally arming a timer for the far future.
1527 *
1528 * We now skip the power-wasting hot spin phase and go straight to the idle phase.
1529 */
1530
1531 params->deadline = 0; /* expire immediately */
1532 params->interval = 0; /* non-repeating */
1533 } else {
1534 uint64_t interval_abs = 0;
1535
1536 if (use_abstime) {
1537 interval_abs = (uint64_t)kev->data;
1538 } else {
1539 uint64_t interval_ns;
1540 if (os_mul_overflow((uint64_t)kev->data, multiplier, &interval_ns)) {
1541 return ERANGE;
1542 }
1543
1544 nanoseconds_to_absolutetime(nanoseconds: interval_ns, result: &interval_abs);
1545 }
1546
1547 uint64_t deadline = 0;
1548
1549 if (kev->fflags & NOTE_MACH_CONTINUOUS_TIME) {
1550 clock_continuoustime_interval_to_deadline(abstime: interval_abs, result: &deadline);
1551 } else {
1552 clock_absolutetime_interval_to_deadline(abstime: interval_abs, result: &deadline);
1553 }
1554
1555 params->deadline = deadline;
1556 params->interval = interval_abs;
1557 }
1558
1559 return 0;
1560}
1561
1562/*
1563 * filt_timerexpire - the timer callout routine
1564 */
1565static void
1566filt_timerexpire(void *knx, void *state_on_arm)
1567{
1568 struct knote *kn = knx;
1569
1570 uint32_t state = (uint32_t)(uintptr_t)state_on_arm;
1571 uint32_t fired_state = state ^ TIMER_ARMED ^ TIMER_FIRED;
1572
1573 if (os_atomic_cmpxchg(&kn->kn_hook32, state, fired_state, relaxed)) {
1574 // our f_event always would say FILTER_ACTIVE,
1575 // so be leaner and just do it.
1576 struct kqueue *kq = knote_get_kq(kn);
1577 kqlock(kqu: kq);
1578 knote_activate(kqu: kq, kn, FILTER_ACTIVE);
1579 kqunlock(kqu: kq);
1580 } else {
1581 /*
1582 * The timer has been reprogrammed or canceled since it was armed,
1583 * and this is a late firing for the timer, just ignore it.
1584 */
1585 }
1586}
1587
1588/*
1589 * Does this deadline needs a timer armed for it, or has it expired?
1590 */
1591static bool
1592filt_timer_is_ready(struct knote *kn)
1593{
1594 uint64_t now, deadline = kn->kn_ext[0];
1595
1596 if (deadline == 0) {
1597 return true;
1598 }
1599
1600 if (kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) {
1601 now = mach_continuous_time();
1602 } else {
1603 now = mach_absolute_time();
1604 }
1605 return deadline <= now;
1606}
1607
1608/*
1609 * Arm a timer
1610 *
1611 * It is the responsibility of the caller to make sure the timer call
1612 * has completed or been cancelled properly prior to arming it.
1613 */
1614static void
1615filt_timerarm(struct knote *kn)
1616{
1617 uint64_t deadline = kn->kn_ext[0];
1618 uint64_t leeway = kn->kn_ext[1];
1619 uint32_t state;
1620
1621 int filter_flags = kn->kn_sfflags;
1622 unsigned int timer_flags = 0;
1623
1624 if (filter_flags & NOTE_CRITICAL) {
1625 timer_flags |= THREAD_CALL_DELAY_USER_CRITICAL;
1626 } else if (filter_flags & NOTE_BACKGROUND) {
1627 timer_flags |= THREAD_CALL_DELAY_USER_BACKGROUND;
1628 } else {
1629 timer_flags |= THREAD_CALL_DELAY_USER_NORMAL;
1630 }
1631
1632 if (filter_flags & NOTE_LEEWAY) {
1633 timer_flags |= THREAD_CALL_DELAY_LEEWAY;
1634 }
1635
1636 if (filter_flags & NOTE_MACH_CONTINUOUS_TIME) {
1637 timer_flags |= THREAD_CALL_CONTINUOUS;
1638 }
1639
1640 /*
1641 * Move to ARMED.
1642 *
1643 * We increase the gencount, and setup the thread call with this expected
1644 * state. It means that if there was a previous generation of the timer in
1645 * flight that needs to be ignored, then 3 things are possible:
1646 *
1647 * - the timer fires first, filt_timerexpire() and sets the state to FIRED
1648 * but we clobber it with ARMED and a new gencount. The knote will still
1649 * be activated, but filt_timerprocess() which is serialized with this
1650 * call will not see the FIRED bit set and will not deliver an event.
1651 *
1652 * - this code runs first, but filt_timerexpire() comes second. Because it
1653 * knows an old gencount, it will debounce and not activate the knote.
1654 *
1655 * - filt_timerexpire() wasn't in flight yet, and thread_call_enter below
1656 * will just cancel it properly.
1657 *
1658 * This is important as userspace expects to never be woken up for past
1659 * timers after filt_timertouch ran.
1660 */
1661 state = os_atomic_load(&kn->kn_hook32, relaxed);
1662 state &= ~TIMER_STATE_MASK;
1663 state += TIMER_GEN_INC + TIMER_ARMED;
1664 os_atomic_store(&kn->kn_hook32, state, relaxed);
1665
1666 thread_call_enter_delayed_with_leeway(call: kn->kn_thcall,
1667 param1: (void *)(uintptr_t)state, deadline, leeway, flags: timer_flags);
1668}
1669
1670/*
1671 * Mark a timer as "already fired" when it is being reprogrammed
1672 *
1673 * If there is a timer in flight, this will do a best effort at canceling it,
1674 * but will not wait. If the thread call was in flight, having set the
1675 * TIMER_IMMEDIATE bit will debounce a filt_timerexpire() racing with this
1676 * cancelation.
1677 */
1678static void
1679filt_timerfire_immediate(struct knote *kn)
1680{
1681 uint32_t state;
1682
1683 static_assert(TIMER_IMMEDIATE == TIMER_STATE_MASK,
1684 "validate that this atomic or will transition to IMMEDIATE");
1685 state = os_atomic_or_orig(&kn->kn_hook32, TIMER_IMMEDIATE, relaxed);
1686
1687 if ((state & TIMER_STATE_MASK) == TIMER_ARMED) {
1688 thread_call_cancel(call: kn->kn_thcall);
1689 }
1690}
1691
1692/*
1693 * Allocate a thread call for the knote's lifetime, and kick off the timer.
1694 */
1695static int
1696filt_timerattach(struct knote *kn, struct kevent_qos_s *kev)
1697{
1698 thread_call_t callout;
1699 struct filt_timer_params params;
1700 int error;
1701
1702 if ((error = filt_timervalidate(kev, params: &params)) != 0) {
1703 knote_set_error(kn, error);
1704 return 0;
1705 }
1706
1707 callout = thread_call_allocate_with_options(func: filt_timerexpire,
1708 param0: (thread_call_param_t)kn, pri: THREAD_CALL_PRIORITY_HIGH,
1709 options: THREAD_CALL_OPTIONS_ONCE);
1710
1711 if (NULL == callout) {
1712 knote_set_error(kn, ENOMEM);
1713 return 0;
1714 }
1715
1716 filt_timer_set_params(kn, params: &params);
1717 kn->kn_thcall = callout;
1718 kn->kn_flags |= EV_CLEAR;
1719 os_atomic_store(&kn->kn_hook32, TIMER_IDLE, relaxed);
1720
1721 /* NOTE_ABSOLUTE implies EV_ONESHOT */
1722 if (kn->kn_sfflags & NOTE_ABSOLUTE) {
1723 kn->kn_flags |= EV_ONESHOT;
1724 }
1725
1726 if (filt_timer_is_ready(kn)) {
1727 os_atomic_store(&kn->kn_hook32, TIMER_IMMEDIATE, relaxed);
1728 return FILTER_ACTIVE;
1729 } else {
1730 filt_timerarm(kn);
1731 return 0;
1732 }
1733}
1734
1735/*
1736 * Shut down the timer if it's running, and free the callout.
1737 */
1738static void
1739filt_timerdetach(struct knote *kn)
1740{
1741 __assert_only boolean_t freed;
1742
1743 /*
1744 * Unconditionally cancel to make sure there can't be any filt_timerexpire()
1745 * running anymore.
1746 */
1747 thread_call_cancel_wait(call: kn->kn_thcall);
1748 freed = thread_call_free(call: kn->kn_thcall);
1749 assert(freed);
1750}
1751
1752/*
1753 * filt_timertouch - update timer knote with new user input
1754 *
1755 * Cancel and restart the timer based on new user data. When
1756 * the user picks up a knote, clear the count of how many timer
1757 * pops have gone off (in kn_data).
1758 */
1759static int
1760filt_timertouch(struct knote *kn, struct kevent_qos_s *kev)
1761{
1762 struct filt_timer_params params;
1763 uint32_t changed_flags = (kn->kn_sfflags ^ kev->fflags);
1764 int error;
1765
1766 if (kev->qos && (knote_get_kq(kn)->kq_state & KQ_WORKLOOP) &&
1767 !_pthread_priority_thread_qos(pp: kev->qos)) {
1768 /* validate usage of FILTER_UPDATE_REQ_QOS */
1769 kev->flags |= EV_ERROR;
1770 kev->data = ERANGE;
1771 return 0;
1772 }
1773
1774 if (changed_flags & NOTE_ABSOLUTE) {
1775 kev->flags |= EV_ERROR;
1776 kev->data = EINVAL;
1777 return 0;
1778 }
1779
1780 if ((error = filt_timervalidate(kev, params: &params)) != 0) {
1781 kev->flags |= EV_ERROR;
1782 kev->data = error;
1783 return 0;
1784 }
1785
1786 /* capture the new values used to compute deadline */
1787 filt_timer_set_params(kn, params: &params);
1788 kn->kn_sfflags = kev->fflags;
1789
1790 if (filt_timer_is_ready(kn)) {
1791 filt_timerfire_immediate(kn);
1792 return FILTER_ACTIVE | FILTER_UPDATE_REQ_QOS;
1793 } else {
1794 filt_timerarm(kn);
1795 return FILTER_UPDATE_REQ_QOS;
1796 }
1797}
1798
1799/*
1800 * filt_timerprocess - query state of knote and snapshot event data
1801 *
1802 * Determine if the timer has fired in the past, snapshot the state
1803 * of the kevent for returning to user-space, and clear pending event
1804 * counters for the next time.
1805 */
1806static int
1807filt_timerprocess(struct knote *kn, struct kevent_qos_s *kev)
1808{
1809 uint32_t state = os_atomic_load(&kn->kn_hook32, relaxed);
1810
1811 /*
1812 * filt_timerprocess is serialized with any filter routine except for
1813 * filt_timerexpire which atomically does a TIMER_ARMED -> TIMER_FIRED
1814 * transition, and on success, activates the knote.
1815 *
1816 * Hence, we don't need atomic modifications of the state, only to peek at
1817 * whether we see any of the "FIRED" state, and if we do, it is safe to
1818 * do simple state machine transitions.
1819 */
1820 switch (state & TIMER_STATE_MASK) {
1821 case TIMER_IDLE:
1822 case TIMER_ARMED:
1823 /*
1824 * This can happen if a touch resets a timer that had fired
1825 * without being processed
1826 */
1827 return 0;
1828 }
1829
1830 os_atomic_store(&kn->kn_hook32, state & ~TIMER_STATE_MASK, relaxed);
1831
1832 /*
1833 * Copy out the interesting kevent state,
1834 * but don't leak out the raw time calculations.
1835 *
1836 * TODO: potential enhancements - tell the user about:
1837 * - deadline to which this timer thought it was expiring
1838 * - return kn_sfflags in the fflags field so the client can know
1839 * under what flags the timer fired
1840 */
1841 knote_fill_kevent(kn, kev, data: 1);
1842 kev->ext[0] = 0;
1843 /* kev->ext[1] = 0; JMM - shouldn't we hide this too? */
1844
1845 if (kn->kn_sdata != 0) {
1846 /*
1847 * This is a 'repeating' timer, so we have to emit
1848 * how many intervals expired between the arm
1849 * and the process.
1850 *
1851 * A very strange style of interface, because
1852 * this could easily be done in the client...
1853 */
1854
1855 uint64_t now;
1856
1857 if (kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) {
1858 now = mach_continuous_time();
1859 } else {
1860 now = mach_absolute_time();
1861 }
1862
1863 uint64_t first_deadline = kn->kn_ext[0];
1864 uint64_t interval_abs = kn->kn_sdata;
1865 uint64_t orig_arm_time = first_deadline - interval_abs;
1866
1867 assert(now > orig_arm_time);
1868 assert(now > first_deadline);
1869
1870 uint64_t elapsed = now - orig_arm_time;
1871
1872 uint64_t num_fired = elapsed / interval_abs;
1873
1874 /*
1875 * To reach this code, we must have seen the timer pop
1876 * and be in repeating mode, so therefore it must have been
1877 * more than 'interval' time since the attach or last
1878 * successful touch.
1879 */
1880 assert(num_fired > 0);
1881
1882 /* report how many intervals have elapsed to the user */
1883 kev->data = (int64_t)num_fired;
1884
1885 /* We only need to re-arm the timer if it's not about to be destroyed */
1886 if ((kn->kn_flags & EV_ONESHOT) == 0) {
1887 /* fire at the end of the next interval */
1888 uint64_t new_deadline = first_deadline + num_fired * interval_abs;
1889
1890 assert(new_deadline > now);
1891
1892 kn->kn_ext[0] = new_deadline;
1893
1894 /*
1895 * This can't shortcut setting up the thread call, because
1896 * knote_process deactivates EV_CLEAR knotes unconditionnally.
1897 */
1898 filt_timerarm(kn);
1899 }
1900 }
1901
1902 return FILTER_ACTIVE;
1903}
1904
1905SECURITY_READ_ONLY_EARLY(static struct filterops) timer_filtops = {
1906 .f_extended_codes = true,
1907 .f_attach = filt_timerattach,
1908 .f_detach = filt_timerdetach,
1909 .f_event = filt_bad_event,
1910 .f_touch = filt_timertouch,
1911 .f_process = filt_timerprocess,
1912};
1913
1914#pragma mark user_filtops
1915
1916static int
1917filt_userattach(struct knote *kn, __unused struct kevent_qos_s *kev)
1918{
1919 if (kn->kn_sfflags & NOTE_TRIGGER) {
1920 kn->kn_hook32 = FILTER_ACTIVE;
1921 } else {
1922 kn->kn_hook32 = 0;
1923 }
1924 return kn->kn_hook32;
1925}
1926
1927static int
1928filt_usertouch(struct knote *kn, struct kevent_qos_s *kev)
1929{
1930 uint32_t ffctrl;
1931 int fflags;
1932
1933 ffctrl = kev->fflags & NOTE_FFCTRLMASK;
1934 fflags = kev->fflags & NOTE_FFLAGSMASK;
1935 switch (ffctrl) {
1936 case NOTE_FFNOP:
1937 break;
1938 case NOTE_FFAND:
1939 kn->kn_sfflags &= fflags;
1940 break;
1941 case NOTE_FFOR:
1942 kn->kn_sfflags |= fflags;
1943 break;
1944 case NOTE_FFCOPY:
1945 kn->kn_sfflags = fflags;
1946 break;
1947 }
1948 kn->kn_sdata = kev->data;
1949
1950 if (kev->fflags & NOTE_TRIGGER) {
1951 kn->kn_hook32 = FILTER_ACTIVE;
1952 }
1953 return (int)kn->kn_hook32;
1954}
1955
1956static int
1957filt_userprocess(struct knote *kn, struct kevent_qos_s *kev)
1958{
1959 int result = (int)kn->kn_hook32;
1960
1961 if (result) {
1962 /* EVFILT_USER returns the data that was passed in */
1963 knote_fill_kevent_with_sdata(kn, kev);
1964 kev->fflags = kn->kn_sfflags;
1965 if (kn->kn_flags & EV_CLEAR) {
1966 /* knote_fill_kevent cleared kn_fflags */
1967 kn->kn_hook32 = 0;
1968 }
1969 }
1970
1971 return result;
1972}
1973
1974SECURITY_READ_ONLY_EARLY(static struct filterops) user_filtops = {
1975 .f_extended_codes = true,
1976 .f_attach = filt_userattach,
1977 .f_detach = filt_no_detach,
1978 .f_event = filt_bad_event,
1979 .f_touch = filt_usertouch,
1980 .f_process = filt_userprocess,
1981};
1982
1983#pragma mark workloop_filtops
1984
1985#define EPREEMPTDISABLED (-1)
1986
1987static inline void
1988filt_wllock(struct kqworkloop *kqwl)
1989{
1990 lck_spin_lock(lck: &kqwl->kqwl_statelock);
1991}
1992
1993static inline void
1994filt_wlunlock(struct kqworkloop *kqwl)
1995{
1996 lck_spin_unlock(lck: &kqwl->kqwl_statelock);
1997}
1998
1999/*
2000 * Returns true when the interlock for the turnstile is the workqueue lock
2001 *
2002 * When this is the case, all turnstiles operations are delegated
2003 * to the workqueue subsystem.
2004 *
2005 * This is required because kqueue_threadreq_bind_prepost only holds the
2006 * workqueue lock but needs to move the inheritor from the workloop turnstile
2007 * away from the creator thread, so that this now fulfilled request cannot be
2008 * picked anymore by other threads.
2009 */
2010static inline bool
2011filt_wlturnstile_interlock_is_workq(struct kqworkloop *kqwl)
2012{
2013 return kqr_thread_requested_pending(kqr: &kqwl->kqwl_request);
2014}
2015
2016static void
2017filt_wlupdate_inheritor(struct kqworkloop *kqwl, struct turnstile *ts,
2018 turnstile_update_flags_t flags)
2019{
2020 turnstile_inheritor_t inheritor = TURNSTILE_INHERITOR_NULL;
2021 workq_threadreq_t kqr = &kqwl->kqwl_request;
2022
2023 /*
2024 * binding to the workq should always happen through
2025 * workq_kern_threadreq_update_inheritor()
2026 */
2027 assert(!filt_wlturnstile_interlock_is_workq(kqwl));
2028
2029 if ((inheritor = kqwl->kqwl_owner)) {
2030 flags |= TURNSTILE_INHERITOR_THREAD;
2031 } else if ((inheritor = kqr_thread(kqr))) {
2032 flags |= TURNSTILE_INHERITOR_THREAD;
2033 }
2034
2035 turnstile_update_inheritor(turnstile: ts, new_inheritor: inheritor, flags);
2036}
2037
2038#define EVFILT_WORKLOOP_EFAULT_RETRY_COUNT 100
2039#define FILT_WLATTACH 0
2040#define FILT_WLTOUCH 1
2041#define FILT_WLDROP 2
2042
2043__result_use_check
2044static int
2045filt_wlupdate(struct kqworkloop *kqwl, struct knote *kn,
2046 struct kevent_qos_s *kev, kq_index_t qos_index, int op)
2047{
2048 user_addr_t uaddr = CAST_USER_ADDR_T(kev->ext[EV_EXTIDX_WL_ADDR]);
2049 workq_threadreq_t kqr = &kqwl->kqwl_request;
2050 thread_t cur_owner, new_owner, extra_thread_ref = THREAD_NULL;
2051 kq_index_t cur_override = THREAD_QOS_UNSPECIFIED;
2052 int efault_retry = EVFILT_WORKLOOP_EFAULT_RETRY_COUNT;
2053 int action = KQWL_UTQ_NONE, error = 0;
2054 bool wl_inheritor_updated = false, needs_wake = false;
2055 uint64_t kdata = kev->ext[EV_EXTIDX_WL_VALUE];
2056 uint64_t mask = kev->ext[EV_EXTIDX_WL_MASK];
2057 uint64_t udata = 0;
2058 struct turnstile *ts = TURNSTILE_NULL;
2059
2060 filt_wllock(kqwl);
2061
2062again:
2063 new_owner = cur_owner = kqwl->kqwl_owner;
2064
2065 /*
2066 * Phase 1:
2067 *
2068 * If asked, load the uint64 value at the user provided address and compare
2069 * it against the passed in mask and expected value.
2070 *
2071 * If NOTE_WL_DISCOVER_OWNER is specified, translate the loaded name as
2072 * a thread reference.
2073 *
2074 * If NOTE_WL_END_OWNERSHIP is specified and the currently known owner is
2075 * the current thread, then end ownership.
2076 *
2077 * Lastly decide whether we need to perform a QoS update.
2078 */
2079 if (uaddr) {
2080 /*
2081 * Until <rdar://problem/24999882> exists,
2082 * disabling preemption copyin forces any
2083 * vm_fault we encounter to fail.
2084 */
2085 error = copyin_atomic64(user_addr: uaddr, u64: &udata);
2086
2087 /*
2088 * If we get EFAULT, drop locks, and retry.
2089 * If we still get an error report it,
2090 * else assume the memory has been faulted
2091 * and attempt to copyin under lock again.
2092 */
2093 switch (error) {
2094 case 0:
2095 break;
2096 case EFAULT:
2097 if (efault_retry-- > 0) {
2098 filt_wlunlock(kqwl);
2099 error = copyin_atomic64(user_addr: uaddr, u64: &udata);
2100 filt_wllock(kqwl);
2101 if (error == 0) {
2102 goto again;
2103 }
2104 }
2105 OS_FALLTHROUGH;
2106 default:
2107 goto out;
2108 }
2109
2110 /* Update state as copied in. */
2111 kev->ext[EV_EXTIDX_WL_VALUE] = udata;
2112
2113 if ((udata & mask) != (kdata & mask)) {
2114 error = ESTALE;
2115 } else if (kev->fflags & NOTE_WL_DISCOVER_OWNER) {
2116 /*
2117 * Decipher the owner port name, and translate accordingly.
2118 * The low 2 bits were borrowed for other flags, so mask them off.
2119 *
2120 * Then attempt translation to a thread reference or fail.
2121 */
2122 mach_port_name_t name = (mach_port_name_t)udata & ~0x3;
2123 if (name != MACH_PORT_NULL) {
2124 name = ipc_entry_name_mask(name);
2125 extra_thread_ref = port_name_to_thread(port_name: name,
2126 options: PORT_INTRANS_THREAD_IN_CURRENT_TASK);
2127 if (extra_thread_ref == THREAD_NULL) {
2128 error = EOWNERDEAD;
2129 goto out;
2130 }
2131 new_owner = extra_thread_ref;
2132 }
2133 }
2134 }
2135
2136 if ((kev->fflags & NOTE_WL_END_OWNERSHIP) && new_owner == current_thread()) {
2137 new_owner = THREAD_NULL;
2138 }
2139
2140 if (error == 0) {
2141 if ((kev->fflags & NOTE_WL_THREAD_REQUEST) && (kev->flags & EV_DELETE)) {
2142 action = KQWL_UTQ_SET_QOS_INDEX;
2143 } else if (qos_index && kqr->tr_kq_qos_index != qos_index) {
2144 action = KQWL_UTQ_SET_QOS_INDEX;
2145 }
2146
2147 if (op == FILT_WLTOUCH) {
2148 /*
2149 * Save off any additional fflags/data we just accepted
2150 * But only keep the last round of "update" bits we acted on which helps
2151 * debugging a lot.
2152 */
2153 kn->kn_sfflags &= ~NOTE_WL_UPDATES_MASK;
2154 kn->kn_sfflags |= kev->fflags;
2155 if (kev->fflags & NOTE_WL_SYNC_WAKE) {
2156 needs_wake = (kn->kn_thread != THREAD_NULL);
2157 }
2158 } else if (op == FILT_WLDROP) {
2159 if ((kn->kn_sfflags & (NOTE_WL_SYNC_WAIT | NOTE_WL_SYNC_WAKE)) ==
2160 NOTE_WL_SYNC_WAIT) {
2161 /*
2162 * When deleting a SYNC_WAIT knote that hasn't been woken up
2163 * explicitly, issue a wake up.
2164 */
2165 kn->kn_sfflags |= NOTE_WL_SYNC_WAKE;
2166 needs_wake = (kn->kn_thread != THREAD_NULL);
2167 }
2168 }
2169 }
2170
2171 /*
2172 * Phase 2:
2173 *
2174 * Commit ownership and QoS changes if any, possibly wake up waiters
2175 */
2176
2177 if (cur_owner == new_owner && action == KQWL_UTQ_NONE && !needs_wake) {
2178 goto out;
2179 }
2180
2181 kqlock(kqu: kqwl);
2182
2183 /* If already tracked as servicer, don't track as owner */
2184 if (new_owner == kqr_thread(kqr)) {
2185 new_owner = THREAD_NULL;
2186 }
2187
2188 if (cur_owner != new_owner) {
2189 kqwl->kqwl_owner = new_owner;
2190 if (new_owner == extra_thread_ref) {
2191 /* we just transfered this ref to kqwl_owner */
2192 extra_thread_ref = THREAD_NULL;
2193 }
2194 cur_override = kqworkloop_override(kqwl);
2195
2196 if (new_owner) {
2197 /* override it before we drop the old */
2198 if (cur_override != THREAD_QOS_UNSPECIFIED) {
2199 thread_add_kevent_override(thread: new_owner, qos_override: cur_override);
2200 }
2201 if (kqr_thread_requested_pending(kqr)) {
2202 if (action == KQWL_UTQ_NONE) {
2203 action = KQWL_UTQ_REDRIVE_EVENTS;
2204 }
2205 }
2206 } else if (action == KQWL_UTQ_NONE &&
2207 !kqr_thread_requested(kqr) &&
2208 kqwl->kqwl_wakeup_qos) {
2209 action = KQWL_UTQ_REDRIVE_EVENTS;
2210 }
2211 }
2212
2213 if (action != KQWL_UTQ_NONE) {
2214 kqworkloop_update_threads_qos(kqwl, op: action, qos: qos_index);
2215 }
2216
2217 ts = kqwl->kqwl_turnstile;
2218 if (cur_owner != new_owner && ts) {
2219 if (action == KQWL_UTQ_REDRIVE_EVENTS) {
2220 /*
2221 * Note that when action is KQWL_UTQ_REDRIVE_EVENTS,
2222 * the code went through workq_kern_threadreq_initiate()
2223 * and the workqueue has set the inheritor already
2224 */
2225 assert(filt_wlturnstile_interlock_is_workq(kqwl));
2226 } else if (filt_wlturnstile_interlock_is_workq(kqwl)) {
2227 workq_kern_threadreq_lock(p: kqwl->kqwl_p);
2228 workq_kern_threadreq_update_inheritor(p: kqwl->kqwl_p, kqr, owner: new_owner,
2229 ts, flags: TURNSTILE_IMMEDIATE_UPDATE);
2230 workq_kern_threadreq_unlock(p: kqwl->kqwl_p);
2231 if (!filt_wlturnstile_interlock_is_workq(kqwl)) {
2232 /*
2233 * If the workq is no longer the interlock, then
2234 * workq_kern_threadreq_update_inheritor() has finished a bind
2235 * and we need to fallback to the regular path.
2236 */
2237 filt_wlupdate_inheritor(kqwl, ts, flags: TURNSTILE_IMMEDIATE_UPDATE);
2238 }
2239 wl_inheritor_updated = true;
2240 } else {
2241 filt_wlupdate_inheritor(kqwl, ts, flags: TURNSTILE_IMMEDIATE_UPDATE);
2242 wl_inheritor_updated = true;
2243 }
2244
2245 /*
2246 * We need a turnstile reference because we are dropping the interlock
2247 * and the caller has not called turnstile_prepare.
2248 */
2249 if (wl_inheritor_updated) {
2250 turnstile_reference(turnstile: ts);
2251 }
2252 }
2253
2254 if (needs_wake && ts) {
2255 waitq_wakeup64_thread(waitq: &ts->ts_waitq, wake_event: knote_filt_wev64(kn),
2256 thread: kn->kn_thread, THREAD_AWAKENED);
2257 if (op == FILT_WLATTACH || op == FILT_WLTOUCH) {
2258 disable_preemption();
2259 error = EPREEMPTDISABLED;
2260 }
2261 }
2262
2263 kqunlock(kqu: kqwl);
2264
2265out:
2266 /*
2267 * Phase 3:
2268 *
2269 * Unlock and cleanup various lingering references and things.
2270 */
2271 filt_wlunlock(kqwl);
2272
2273#if CONFIG_WORKLOOP_DEBUG
2274 KQWL_HISTORY_WRITE_ENTRY(kqwl, {
2275 .updater = current_thread(),
2276 .servicer = kqr_thread(kqr), /* Note: racy */
2277 .old_owner = cur_owner,
2278 .new_owner = new_owner,
2279
2280 .kev_ident = kev->ident,
2281 .error = (int16_t)error,
2282 .kev_flags = kev->flags,
2283 .kev_fflags = kev->fflags,
2284
2285 .kev_mask = mask,
2286 .kev_value = kdata,
2287 .in_value = udata,
2288 });
2289#endif // CONFIG_WORKLOOP_DEBUG
2290
2291 if (wl_inheritor_updated) {
2292 turnstile_update_inheritor_complete(turnstile: ts, flags: TURNSTILE_INTERLOCK_NOT_HELD);
2293 turnstile_deallocate_safe(turnstile: ts);
2294 }
2295
2296 if (cur_owner && new_owner != cur_owner) {
2297 if (cur_override != THREAD_QOS_UNSPECIFIED) {
2298 thread_drop_kevent_override(thread: cur_owner);
2299 }
2300 thread_deallocate_safe(thread: cur_owner);
2301 }
2302 if (extra_thread_ref) {
2303 thread_deallocate_safe(thread: extra_thread_ref);
2304 }
2305 return error;
2306}
2307
2308/*
2309 * Remembers the last updated that came in from userspace for debugging reasons.
2310 * - fflags is mirrored from the userspace kevent
2311 * - ext[i, i != VALUE] is mirrored from the userspace kevent
2312 * - ext[VALUE] is set to what the kernel loaded atomically
2313 * - data is set to the error if any
2314 */
2315static inline void
2316filt_wlremember_last_update(struct knote *kn, struct kevent_qos_s *kev,
2317 int error)
2318{
2319 kn->kn_fflags = kev->fflags;
2320 kn->kn_sdata = error;
2321 memcpy(dst: kn->kn_ext, src: kev->ext, n: sizeof(kev->ext));
2322}
2323
2324static int
2325filt_wlupdate_sync_ipc(struct kqworkloop *kqwl, struct knote *kn,
2326 struct kevent_qos_s *kev, int op)
2327{
2328 user_addr_t uaddr = (user_addr_t) kev->ext[EV_EXTIDX_WL_ADDR];
2329 uint64_t kdata = kev->ext[EV_EXTIDX_WL_VALUE];
2330 uint64_t mask = kev->ext[EV_EXTIDX_WL_MASK];
2331 uint64_t udata = 0;
2332 int efault_retry = EVFILT_WORKLOOP_EFAULT_RETRY_COUNT;
2333 int error = 0;
2334
2335 if (op == FILT_WLATTACH) {
2336 (void)kqueue_alloc_turnstile(&kqwl->kqwl_kqueue);
2337 } else if (uaddr == 0) {
2338 return 0;
2339 }
2340
2341 filt_wllock(kqwl);
2342
2343again:
2344
2345 /*
2346 * Do the debounce thing, the lock serializing the state is the knote lock.
2347 */
2348 if (uaddr) {
2349 /*
2350 * Until <rdar://problem/24999882> exists,
2351 * disabling preemption copyin forces any
2352 * vm_fault we encounter to fail.
2353 */
2354 error = copyin_atomic64(user_addr: uaddr, u64: &udata);
2355
2356 /*
2357 * If we get EFAULT, drop locks, and retry.
2358 * If we still get an error report it,
2359 * else assume the memory has been faulted
2360 * and attempt to copyin under lock again.
2361 */
2362 switch (error) {
2363 case 0:
2364 break;
2365 case EFAULT:
2366 if (efault_retry-- > 0) {
2367 filt_wlunlock(kqwl);
2368 error = copyin_atomic64(user_addr: uaddr, u64: &udata);
2369 filt_wllock(kqwl);
2370 if (error == 0) {
2371 goto again;
2372 }
2373 }
2374 OS_FALLTHROUGH;
2375 default:
2376 goto out;
2377 }
2378
2379 kev->ext[EV_EXTIDX_WL_VALUE] = udata;
2380 kn->kn_ext[EV_EXTIDX_WL_VALUE] = udata;
2381
2382 if ((udata & mask) != (kdata & mask)) {
2383 error = ESTALE;
2384 goto out;
2385 }
2386 }
2387
2388 if (op == FILT_WLATTACH) {
2389 error = filt_wlattach_sync_ipc(kn);
2390 if (error == 0) {
2391 disable_preemption();
2392 error = EPREEMPTDISABLED;
2393 }
2394 }
2395
2396out:
2397 filt_wlunlock(kqwl);
2398 return error;
2399}
2400
2401static int
2402filt_wlattach(struct knote *kn, struct kevent_qos_s *kev)
2403{
2404 struct kqueue *kq = knote_get_kq(kn);
2405 struct kqworkloop *kqwl = (struct kqworkloop *)kq;
2406 int error = 0, result = 0;
2407 kq_index_t qos_index = 0;
2408
2409 if (__improbable((kq->kq_state & KQ_WORKLOOP) == 0)) {
2410 error = ENOTSUP;
2411 goto out;
2412 }
2413
2414 uint32_t command = (kn->kn_sfflags & NOTE_WL_COMMANDS_MASK);
2415 switch (command) {
2416 case NOTE_WL_THREAD_REQUEST:
2417 if (kn->kn_id != kqwl->kqwl_dynamicid) {
2418 error = EINVAL;
2419 goto out;
2420 }
2421 qos_index = _pthread_priority_thread_qos(pp: kn->kn_qos);
2422 if (qos_index == THREAD_QOS_UNSPECIFIED) {
2423 error = ERANGE;
2424 goto out;
2425 }
2426 if (kqwl->kqwl_request.tr_kq_qos_index) {
2427 /*
2428 * There already is a thread request, and well, you're only allowed
2429 * one per workloop, so fail the attach.
2430 */
2431 error = EALREADY;
2432 goto out;
2433 }
2434 break;
2435 case NOTE_WL_SYNC_WAIT:
2436 case NOTE_WL_SYNC_WAKE:
2437 if (kn->kn_id == kqwl->kqwl_dynamicid) {
2438 error = EINVAL;
2439 goto out;
2440 }
2441 if ((kn->kn_flags & EV_DISABLE) == 0) {
2442 error = EINVAL;
2443 goto out;
2444 }
2445 if (kn->kn_sfflags & NOTE_WL_END_OWNERSHIP) {
2446 error = EINVAL;
2447 goto out;
2448 }
2449 break;
2450
2451 case NOTE_WL_SYNC_IPC:
2452 if ((kn->kn_flags & EV_DISABLE) == 0) {
2453 error = EINVAL;
2454 goto out;
2455 }
2456 if (kn->kn_sfflags & (NOTE_WL_UPDATE_QOS | NOTE_WL_DISCOVER_OWNER)) {
2457 error = EINVAL;
2458 goto out;
2459 }
2460 break;
2461 default:
2462 error = EINVAL;
2463 goto out;
2464 }
2465
2466 if (command == NOTE_WL_SYNC_IPC) {
2467 error = filt_wlupdate_sync_ipc(kqwl, kn, kev, FILT_WLATTACH);
2468 } else {
2469 error = filt_wlupdate(kqwl, kn, kev, qos_index, FILT_WLATTACH);
2470 }
2471
2472 if (error == EPREEMPTDISABLED) {
2473 error = 0;
2474 result = FILTER_THREADREQ_NODEFEER;
2475 }
2476out:
2477 if (error) {
2478 /* If userland wants ESTALE to be hidden, fail the attach anyway */
2479 if (error == ESTALE && (kn->kn_sfflags & NOTE_WL_IGNORE_ESTALE)) {
2480 error = 0;
2481 }
2482 knote_set_error(kn, error);
2483 return result;
2484 }
2485 if (command == NOTE_WL_SYNC_WAIT) {
2486 return kevent_register_wait_prepare(kn, kev, result);
2487 }
2488 /* Just attaching the thread request successfully will fire it */
2489 if (command == NOTE_WL_THREAD_REQUEST) {
2490 /*
2491 * Thread Request knotes need an explicit touch to be active again,
2492 * so delivering an event needs to also consume it.
2493 */
2494 kn->kn_flags |= EV_CLEAR;
2495 return result | FILTER_ACTIVE;
2496 }
2497 return result;
2498}
2499
2500static void __dead2
2501filt_wlwait_continue(void *parameter, wait_result_t wr)
2502{
2503 struct _kevent_register *cont_args = parameter;
2504 struct kqworkloop *kqwl = cont_args->kqwl;
2505
2506 kqlock(kqu: kqwl);
2507 if (filt_wlturnstile_interlock_is_workq(kqwl)) {
2508 workq_kern_threadreq_lock(p: kqwl->kqwl_p);
2509 turnstile_complete(proprietor: (uintptr_t)kqwl, tstore: &kqwl->kqwl_turnstile, NULL, type: TURNSTILE_WORKLOOPS);
2510 workq_kern_threadreq_unlock(p: kqwl->kqwl_p);
2511 } else {
2512 turnstile_complete(proprietor: (uintptr_t)kqwl, tstore: &kqwl->kqwl_turnstile, NULL, type: TURNSTILE_WORKLOOPS);
2513 }
2514 kqunlock(kqu: kqwl);
2515
2516 turnstile_cleanup();
2517
2518 if (wr == THREAD_INTERRUPTED) {
2519 cont_args->kev.flags |= EV_ERROR;
2520 cont_args->kev.data = EINTR;
2521 } else if (wr != THREAD_AWAKENED) {
2522 panic("Unexpected wait result: %d", wr);
2523 }
2524
2525 kevent_register_wait_return(cont_args);
2526}
2527
2528/*
2529 * Called with the workloop mutex held, most of the time never returns as it
2530 * calls filt_wlwait_continue through a continuation.
2531 */
2532static void __dead2
2533filt_wlpost_register_wait(struct uthread *uth, struct knote *kn,
2534 struct _kevent_register *cont_args)
2535{
2536 struct kqworkloop *kqwl = cont_args->kqwl;
2537 workq_threadreq_t kqr = &kqwl->kqwl_request;
2538 struct turnstile *ts;
2539 bool workq_locked = false;
2540
2541 kqlock_held(kqu: kqwl);
2542
2543 if (filt_wlturnstile_interlock_is_workq(kqwl)) {
2544 workq_kern_threadreq_lock(p: kqwl->kqwl_p);
2545 workq_locked = true;
2546 }
2547
2548 ts = turnstile_prepare(proprietor: (uintptr_t)kqwl, tstore: &kqwl->kqwl_turnstile,
2549 TURNSTILE_NULL, type: TURNSTILE_WORKLOOPS);
2550
2551 if (workq_locked) {
2552 workq_kern_threadreq_update_inheritor(p: kqwl->kqwl_p,
2553 kqr: &kqwl->kqwl_request, owner: kqwl->kqwl_owner, ts,
2554 flags: TURNSTILE_DELAYED_UPDATE);
2555 if (!filt_wlturnstile_interlock_is_workq(kqwl)) {
2556 /*
2557 * if the interlock is no longer the workqueue lock,
2558 * then we don't need to hold it anymore.
2559 */
2560 workq_kern_threadreq_unlock(p: kqwl->kqwl_p);
2561 workq_locked = false;
2562 }
2563 }
2564 if (!workq_locked) {
2565 /*
2566 * If the interlock is the workloop's, then it's our responsibility to
2567 * call update_inheritor, so just do it.
2568 */
2569 filt_wlupdate_inheritor(kqwl, ts, flags: TURNSTILE_DELAYED_UPDATE);
2570 }
2571
2572 thread_set_pending_block_hint(thread: get_machthread(uth), block_hint: kThreadWaitWorkloopSyncWait);
2573 waitq_assert_wait64(waitq: &ts->ts_waitq, wait_event: knote_filt_wev64(kn),
2574 THREAD_ABORTSAFE, TIMEOUT_WAIT_FOREVER);
2575
2576 if (workq_locked) {
2577 workq_kern_threadreq_unlock(p: kqwl->kqwl_p);
2578 }
2579
2580 thread_t thread = kqwl->kqwl_owner ?: kqr_thread(kqr);
2581 if (thread) {
2582 thread_reference(thread);
2583 }
2584
2585 kevent_register_wait_block(ts, handoff_thread: thread, cont: filt_wlwait_continue, cont_args);
2586}
2587
2588/* called in stackshot context to report the thread responsible for blocking this thread */
2589void
2590kdp_workloop_sync_wait_find_owner(__assert_only thread_t thread,
2591 event64_t event, thread_waitinfo_t *waitinfo)
2592{
2593 struct knote *kn = (struct knote *)event;
2594
2595 zone_require(zone: knote_zone, addr: kn);
2596
2597 assert(kn->kn_thread == thread);
2598
2599 struct kqueue *kq = knote_get_kq(kn);
2600
2601 zone_require(zone: kqworkloop_zone, addr: kq);
2602 assert(kq->kq_state & KQ_WORKLOOP);
2603
2604 struct kqworkloop *kqwl = (struct kqworkloop *)kq;
2605 workq_threadreq_t kqr = &kqwl->kqwl_request;
2606
2607 thread_t kqwl_owner = kqwl->kqwl_owner;
2608
2609 if (kqwl_owner != THREAD_NULL) {
2610 thread_require(thread: kqwl_owner);
2611 waitinfo->owner = thread_tid(thread: kqwl->kqwl_owner);
2612 } else if ((kqr->tr_state >= WORKQ_TR_STATE_BINDING) && (kqr->tr_thread != NULL)) {
2613 thread_require(thread: kqr->tr_thread);
2614 waitinfo->owner = thread_tid(thread: kqr->tr_thread);
2615 } else if (kqr_thread_requested_pending(kqr)) { /* > idle, < bound */
2616 waitinfo->owner = STACKSHOT_WAITOWNER_THREQUESTED;
2617 } else {
2618 waitinfo->owner = 0;
2619 }
2620
2621 waitinfo->context = kqwl->kqwl_dynamicid;
2622}
2623
2624static void
2625filt_wldetach(struct knote *kn)
2626{
2627 if (kn->kn_sfflags & NOTE_WL_SYNC_IPC) {
2628 filt_wldetach_sync_ipc(kn);
2629 } else if (kn->kn_thread) {
2630 kevent_register_wait_cleanup(kn);
2631 }
2632}
2633
2634static int
2635filt_wlvalidate_kev_flags(struct knote *kn, struct kevent_qos_s *kev,
2636 thread_qos_t *qos_index)
2637{
2638 uint32_t new_commands = kev->fflags & NOTE_WL_COMMANDS_MASK;
2639 uint32_t sav_commands = kn->kn_sfflags & NOTE_WL_COMMANDS_MASK;
2640
2641 if ((kev->fflags & NOTE_WL_DISCOVER_OWNER) && (kev->flags & EV_DELETE)) {
2642 return EINVAL;
2643 }
2644 if (kev->fflags & NOTE_WL_UPDATE_QOS) {
2645 if (kev->flags & EV_DELETE) {
2646 return EINVAL;
2647 }
2648 if (sav_commands != NOTE_WL_THREAD_REQUEST) {
2649 return EINVAL;
2650 }
2651 if (!(*qos_index = _pthread_priority_thread_qos(pp: kev->qos))) {
2652 return ERANGE;
2653 }
2654 }
2655
2656 switch (new_commands) {
2657 case NOTE_WL_THREAD_REQUEST:
2658 /* thread requests can only update themselves */
2659 if (sav_commands != NOTE_WL_THREAD_REQUEST) {
2660 return EINVAL;
2661 }
2662 break;
2663
2664 case NOTE_WL_SYNC_WAIT:
2665 if (kev->fflags & NOTE_WL_END_OWNERSHIP) {
2666 return EINVAL;
2667 }
2668 goto sync_checks;
2669
2670 case NOTE_WL_SYNC_WAKE:
2671sync_checks:
2672 if (!(sav_commands & (NOTE_WL_SYNC_WAIT | NOTE_WL_SYNC_WAKE))) {
2673 return EINVAL;
2674 }
2675 if ((kev->flags & (EV_ENABLE | EV_DELETE)) == EV_ENABLE) {
2676 return EINVAL;
2677 }
2678 break;
2679
2680 case NOTE_WL_SYNC_IPC:
2681 if (sav_commands != NOTE_WL_SYNC_IPC) {
2682 return EINVAL;
2683 }
2684 if ((kev->flags & (EV_ENABLE | EV_DELETE)) == EV_ENABLE) {
2685 return EINVAL;
2686 }
2687 break;
2688
2689 default:
2690 return EINVAL;
2691 }
2692 return 0;
2693}
2694
2695static int
2696filt_wltouch(struct knote *kn, struct kevent_qos_s *kev)
2697{
2698 struct kqworkloop *kqwl = (struct kqworkloop *)knote_get_kq(kn);
2699 thread_qos_t qos_index = THREAD_QOS_UNSPECIFIED;
2700 int result = 0;
2701
2702 int error = filt_wlvalidate_kev_flags(kn, kev, qos_index: &qos_index);
2703 if (error) {
2704 goto out;
2705 }
2706
2707 uint32_t command = kev->fflags & NOTE_WL_COMMANDS_MASK;
2708 if (command == NOTE_WL_SYNC_IPC) {
2709 error = filt_wlupdate_sync_ipc(kqwl, kn, kev, FILT_WLTOUCH);
2710 } else {
2711 error = filt_wlupdate(kqwl, kn, kev, qos_index, FILT_WLTOUCH);
2712 filt_wlremember_last_update(kn, kev, error);
2713 }
2714 if (error == EPREEMPTDISABLED) {
2715 error = 0;
2716 result = FILTER_THREADREQ_NODEFEER;
2717 }
2718
2719out:
2720 if (error) {
2721 if (error == ESTALE && (kev->fflags & NOTE_WL_IGNORE_ESTALE)) {
2722 /* If userland wants ESTALE to be hidden, do not activate */
2723 return result;
2724 }
2725 kev->flags |= EV_ERROR;
2726 kev->data = error;
2727 return result;
2728 }
2729 if (command == NOTE_WL_SYNC_WAIT && !(kn->kn_sfflags & NOTE_WL_SYNC_WAKE)) {
2730 return kevent_register_wait_prepare(kn, kev, result);
2731 }
2732 /* Just touching the thread request successfully will fire it */
2733 if (command == NOTE_WL_THREAD_REQUEST) {
2734 if (kev->fflags & NOTE_WL_UPDATE_QOS) {
2735 result |= FILTER_UPDATE_REQ_QOS;
2736 }
2737 result |= FILTER_ACTIVE;
2738 }
2739 return result;
2740}
2741
2742static bool
2743filt_wlallow_drop(struct knote *kn, struct kevent_qos_s *kev)
2744{
2745 struct kqworkloop *kqwl = (struct kqworkloop *)knote_get_kq(kn);
2746
2747 int error = filt_wlvalidate_kev_flags(kn, kev, NULL);
2748 if (error) {
2749 goto out;
2750 }
2751
2752 uint32_t command = (kev->fflags & NOTE_WL_COMMANDS_MASK);
2753 if (command == NOTE_WL_SYNC_IPC) {
2754 error = filt_wlupdate_sync_ipc(kqwl, kn, kev, FILT_WLDROP);
2755 } else {
2756 error = filt_wlupdate(kqwl, kn, kev, qos_index: 0, FILT_WLDROP);
2757 filt_wlremember_last_update(kn, kev, error);
2758 }
2759 assert(error != EPREEMPTDISABLED);
2760
2761out:
2762 if (error) {
2763 if (error == ESTALE && (kev->fflags & NOTE_WL_IGNORE_ESTALE)) {
2764 return false;
2765 }
2766 kev->flags |= EV_ERROR;
2767 kev->data = error;
2768 return false;
2769 }
2770 return true;
2771}
2772
2773static int
2774filt_wlprocess(struct knote *kn, struct kevent_qos_s *kev)
2775{
2776 struct kqworkloop *kqwl = (struct kqworkloop *)knote_get_kq(kn);
2777 int rc = 0;
2778
2779 assert(kn->kn_sfflags & NOTE_WL_THREAD_REQUEST);
2780
2781 kqlock(kqu: kqwl);
2782
2783 if (kqwl->kqwl_owner) {
2784 /*
2785 * <rdar://problem/33584321> userspace sometimes due to events being
2786 * delivered but not triggering a drain session can cause a process
2787 * of the thread request knote.
2788 *
2789 * When that happens, the automatic deactivation due to process
2790 * would swallow the event, so we have to activate the knote again.
2791 */
2792 knote_activate(kqu: kqwl, kn, FILTER_ACTIVE);
2793 } else {
2794#if DEBUG || DEVELOPMENT
2795 if (kevent_debug_flags & KEVENT_PANIC_ON_NON_ENQUEUED_PROCESS) {
2796 /*
2797 * see src/queue_internal.h in libdispatch
2798 */
2799#define DISPATCH_QUEUE_ENQUEUED 0x1ull
2800 user_addr_t addr = CAST_USER_ADDR_T(kn->kn_ext[EV_EXTIDX_WL_ADDR]);
2801 task_t t = current_task();
2802 uint64_t val;
2803 if (addr && task_is_active(t) && !task_is_halting(t) &&
2804 copyin_atomic64(addr, &val) == 0 &&
2805 val && (val & DISPATCH_QUEUE_ENQUEUED) == 0 &&
2806 (val >> 48) != 0xdead && (val >> 48) != 0 && (val >> 48) != 0xffff) {
2807 panic("kevent: workloop %#016llx is not enqueued "
2808 "(kn:%p dq_state:%#016llx kev.dq_state:%#016llx)",
2809 kn->kn_udata, kn, val, kn->kn_ext[EV_EXTIDX_WL_VALUE]);
2810 }
2811 }
2812#endif
2813 knote_fill_kevent(kn, kev, data: 0);
2814 kev->fflags = kn->kn_sfflags;
2815 rc |= FILTER_ACTIVE;
2816 }
2817
2818 kqunlock(kqu: kqwl);
2819
2820 if (rc & FILTER_ACTIVE) {
2821 workq_thread_set_max_qos(p: kqwl->kqwl_p, kqr: &kqwl->kqwl_request);
2822 }
2823 return rc;
2824}
2825
2826SECURITY_READ_ONLY_EARLY(static struct filterops) workloop_filtops = {
2827 .f_extended_codes = true,
2828 .f_attach = filt_wlattach,
2829 .f_detach = filt_wldetach,
2830 .f_event = filt_bad_event,
2831 .f_touch = filt_wltouch,
2832 .f_process = filt_wlprocess,
2833 .f_allow_drop = filt_wlallow_drop,
2834 .f_post_register_wait = filt_wlpost_register_wait,
2835};
2836
2837#pragma mark - kqueues allocation and deallocation
2838
2839OS_NOINLINE
2840static void
2841kqworkloop_dealloc(struct kqworkloop *, bool hash_remove);
2842
2843static inline bool
2844kqworkloop_try_retain(struct kqworkloop *kqwl)
2845{
2846 return os_ref_retain_try_raw(&kqwl->kqwl_retains, NULL);
2847}
2848
2849static inline void
2850kqworkloop_retain(struct kqworkloop *kqwl)
2851{
2852 return os_ref_retain_raw(&kqwl->kqwl_retains, NULL);
2853}
2854
2855OS_ALWAYS_INLINE
2856static inline void
2857kqueue_retain(kqueue_t kqu)
2858{
2859 if (kqu.kq->kq_state & KQ_DYNAMIC) {
2860 kqworkloop_retain(kqwl: kqu.kqwl);
2861 }
2862}
2863
2864OS_ALWAYS_INLINE
2865static inline void
2866kqworkloop_release_live(struct kqworkloop *kqwl)
2867{
2868 os_ref_release_live_raw(&kqwl->kqwl_retains, NULL);
2869}
2870
2871OS_ALWAYS_INLINE
2872static inline void
2873kqueue_release_live(kqueue_t kqu)
2874{
2875 if (kqu.kq->kq_state & KQ_DYNAMIC) {
2876 kqworkloop_release_live(kqwl: kqu.kqwl);
2877 }
2878}
2879
2880OS_ALWAYS_INLINE
2881static inline void
2882kqworkloop_release(struct kqworkloop *kqwl)
2883{
2884 if (os_ref_release_raw(&kqwl->kqwl_retains, NULL) == 0) {
2885 kqworkloop_dealloc(kqwl, true);
2886 }
2887}
2888
2889OS_ALWAYS_INLINE
2890static inline void
2891kqueue_release(kqueue_t kqu)
2892{
2893 if (kqu.kq->kq_state & KQ_DYNAMIC) {
2894 kqworkloop_release(kqwl: kqu.kqwl);
2895 }
2896}
2897
2898/*!
2899 * @function kqueue_destroy
2900 *
2901 * @brief
2902 * Common part to all kqueue dealloc functions.
2903 */
2904OS_NOINLINE
2905static void
2906kqueue_destroy(kqueue_t kqu, zone_t zone)
2907{
2908 lck_spin_destroy(lck: &kqu.kq->kq_lock, grp: &kq_lck_grp);
2909
2910 zfree(zone, kqu.kq);
2911}
2912
2913/*!
2914 * @function kqueue_init
2915 *
2916 * @brief
2917 * Common part to all kqueue alloc functions.
2918 */
2919static kqueue_t
2920kqueue_init(kqueue_t kqu)
2921{
2922 lck_spin_init(lck: &kqu.kq->kq_lock, grp: &kq_lck_grp, LCK_ATTR_NULL);
2923 return kqu;
2924}
2925
2926#pragma mark kqfile allocation and deallocation
2927
2928/*!
2929 * @function kqueue_dealloc
2930 *
2931 * @brief
2932 * Detach all knotes from a kqfile and free it.
2933 *
2934 * @discussion
2935 * We walk each list looking for knotes referencing this
2936 * this kqueue. If we find one, we try to drop it. But
2937 * if we fail to get a drop reference, that will wait
2938 * until it is dropped. So, we can just restart again
2939 * safe in the assumption that the list will eventually
2940 * not contain any more references to this kqueue (either
2941 * we dropped them all, or someone else did).
2942 *
2943 * Assumes no new events are being added to the kqueue.
2944 * Nothing locked on entry or exit.
2945 */
2946void
2947kqueue_dealloc(struct kqueue *kq)
2948{
2949 KNOTE_LOCK_CTX(knlc);
2950 struct proc *p = kq->kq_p;
2951 struct filedesc *fdp = &p->p_fd;
2952 struct knote *kn;
2953
2954 assert(kq && (kq->kq_state & (KQ_WORKLOOP | KQ_WORKQ)) == 0);
2955
2956 proc_fdlock(p);
2957 for (int i = 0; i < fdp->fd_knlistsize; i++) {
2958 kn = SLIST_FIRST(&fdp->fd_knlist[i]);
2959 while (kn != NULL) {
2960 if (kq == knote_get_kq(kn)) {
2961 kqlock(kqu: kq);
2962 proc_fdunlock(p);
2963 if (knote_lock(kqu: kq, kn, knlc: &knlc, kqlocking: KNOTE_KQ_LOCK_ON_SUCCESS)) {
2964 knote_drop(kqu: kq, kn, knlc: &knlc);
2965 }
2966 proc_fdlock(p);
2967 /* start over at beginning of list */
2968 kn = SLIST_FIRST(&fdp->fd_knlist[i]);
2969 continue;
2970 }
2971 kn = SLIST_NEXT(kn, kn_link);
2972 }
2973 }
2974
2975 knhash_lock(fdp);
2976 proc_fdunlock(p);
2977
2978 if (fdp->fd_knhashmask != 0) {
2979 for (int i = 0; i < (int)fdp->fd_knhashmask + 1; i++) {
2980 kn = SLIST_FIRST(&fdp->fd_knhash[i]);
2981 while (kn != NULL) {
2982 if (kq == knote_get_kq(kn)) {
2983 kqlock(kqu: kq);
2984 knhash_unlock(fdp);
2985 if (knote_lock(kqu: kq, kn, knlc: &knlc, kqlocking: KNOTE_KQ_LOCK_ON_SUCCESS)) {
2986 knote_drop(kqu: kq, kn, knlc: &knlc);
2987 }
2988 knhash_lock(fdp);
2989 /* start over at beginning of list */
2990 kn = SLIST_FIRST(&fdp->fd_knhash[i]);
2991 continue;
2992 }
2993 kn = SLIST_NEXT(kn, kn_link);
2994 }
2995 }
2996 }
2997 knhash_unlock(fdp);
2998
2999 kqueue_destroy(kqu: kq, zone: kqfile_zone);
3000}
3001
3002/*!
3003 * @function kqueue_alloc
3004 *
3005 * @brief
3006 * Allocate a kqfile.
3007 */
3008struct kqueue *
3009kqueue_alloc(struct proc *p)
3010{
3011 struct kqfile *kqf;
3012
3013 /*
3014 * kqfiles are created with kqueue() so we need to wait for
3015 * the first kevent syscall to know which bit among
3016 * KQ_KEV_{32,64,QOS} will be set in kqf_state
3017 */
3018 kqf = zalloc_flags(kqfile_zone, Z_WAITOK | Z_ZERO);
3019 kqf->kqf_p = p;
3020 TAILQ_INIT_AFTER_BZERO(&kqf->kqf_queue);
3021 TAILQ_INIT_AFTER_BZERO(&kqf->kqf_suppressed);
3022
3023 return kqueue_init(kqu: kqf).kq;
3024}
3025
3026/*!
3027 * @function kqueue_internal
3028 *
3029 * @brief
3030 * Core implementation for kqueue and guarded_kqueue_np()
3031 */
3032int
3033kqueue_internal(struct proc *p, fp_initfn_t fp_init, void *initarg, int32_t *retval)
3034{
3035 struct kqueue *kq;
3036 struct fileproc *fp;
3037 int fd, error;
3038
3039 error = falloc_withinit(p, p_cred: current_cached_proc_cred(p),
3040 ctx: vfs_context_current(), resultfp: &fp, resultfd: &fd, fp_init, initarg);
3041 if (error) {
3042 return error;
3043 }
3044
3045 kq = kqueue_alloc(p);
3046 if (kq == NULL) {
3047 fp_free(p, fd, fp);
3048 return ENOMEM;
3049 }
3050
3051 fp->fp_flags |= FP_CLOEXEC | FP_CLOFORK;
3052 fp->f_flag = FREAD | FWRITE;
3053 fp->f_ops = &kqueueops;
3054 fp_set_data(fp, fg_data: kq);
3055 fp->f_lflags |= FG_CONFINED;
3056
3057 proc_fdlock(p);
3058 procfdtbl_releasefd(p, fd, NULL);
3059 fp_drop(p, fd, fp, locked: 1);
3060 proc_fdunlock(p);
3061
3062 *retval = fd;
3063 return error;
3064}
3065
3066/*!
3067 * @function kqueue
3068 *
3069 * @brief
3070 * The kqueue syscall.
3071 */
3072int
3073kqueue(struct proc *p, __unused struct kqueue_args *uap, int32_t *retval)
3074{
3075 return kqueue_internal(p, NULL, NULL, retval);
3076}
3077
3078#pragma mark kqworkq allocation and deallocation
3079
3080/*!
3081 * @function kqworkq_dealloc
3082 *
3083 * @brief
3084 * Deallocates a workqueue kqueue.
3085 *
3086 * @discussion
3087 * This only happens at process death, or for races with concurrent
3088 * kevent_get_kqwq calls, hence we don't have to care about knotes referencing
3089 * this kqueue, either there are none, or someone else took care of them.
3090 */
3091void
3092kqworkq_dealloc(struct kqworkq *kqwq)
3093{
3094 kqueue_destroy(kqu: kqwq, zone: kqworkq_zone);
3095}
3096
3097/*!
3098 * @function kqworkq_alloc
3099 *
3100 * @brief
3101 * Allocates a workqueue kqueue.
3102 *
3103 * @discussion
3104 * This is the slow path of kevent_get_kqwq.
3105 * This takes care of making sure procs have a single workq kqueue.
3106 */
3107OS_NOINLINE
3108static struct kqworkq *
3109kqworkq_alloc(struct proc *p, unsigned int flags)
3110{
3111 struct kqworkq *kqwq, *tmp;
3112
3113 kqwq = zalloc_flags(kqworkq_zone, Z_WAITOK | Z_ZERO);
3114
3115 assert((flags & KEVENT_FLAG_LEGACY32) == 0);
3116 if (flags & KEVENT_FLAG_LEGACY64) {
3117 kqwq->kqwq_state = KQ_WORKQ | KQ_KEV64;
3118 } else {
3119 kqwq->kqwq_state = KQ_WORKQ | KQ_KEV_QOS;
3120 }
3121 kqwq->kqwq_p = p;
3122
3123 for (int i = 0; i < KQWQ_NBUCKETS; i++) {
3124 TAILQ_INIT_AFTER_BZERO(&kqwq->kqwq_queue[i]);
3125 TAILQ_INIT_AFTER_BZERO(&kqwq->kqwq_suppressed[i]);
3126 }
3127 for (int i = 0; i < KQWQ_NBUCKETS; i++) {
3128 /*
3129 * Because of how the bucketized system works, we mix overcommit
3130 * sources with not overcommit: each time we move a knote from
3131 * one bucket to the next due to overrides, we'd had to track
3132 * overcommitness, and it's really not worth it in the workloop
3133 * enabled world that track this faithfully.
3134 *
3135 * Incidentally, this behaves like the original manager-based
3136 * kqwq where event delivery always happened (hence is
3137 * "overcommit")
3138 */
3139 kqwq->kqwq_request[i].tr_state = WORKQ_TR_STATE_IDLE;
3140 kqwq->kqwq_request[i].tr_flags = WORKQ_TR_FLAG_KEVENT;
3141 if (i != KQWQ_QOS_MANAGER) {
3142 kqwq->kqwq_request[i].tr_flags |= WORKQ_TR_FLAG_OVERCOMMIT;
3143 }
3144 kqwq->kqwq_request[i].tr_kq_qos_index = (kq_index_t)i + 1;
3145 }
3146
3147 kqueue_init(kqu: kqwq);
3148
3149 if (!os_atomic_cmpxchgv(&p->p_fd.fd_wqkqueue, NULL, kqwq, &tmp, release)) {
3150 kqworkq_dealloc(kqwq);
3151 return tmp;
3152 }
3153
3154 return kqwq;
3155}
3156
3157#pragma mark kqworkloop allocation and deallocation
3158
3159#define KQ_HASH(val, mask) (((val) ^ (val >> 8)) & (mask))
3160#define CONFIG_KQ_HASHSIZE CONFIG_KN_HASHSIZE
3161
3162OS_ALWAYS_INLINE
3163static inline void
3164kqhash_lock(struct filedesc *fdp)
3165{
3166 lck_mtx_lock_spin_always(lck: &fdp->fd_kqhashlock);
3167}
3168
3169OS_ALWAYS_INLINE
3170static inline void
3171kqhash_unlock(struct filedesc *fdp)
3172{
3173 lck_mtx_unlock(lck: &fdp->fd_kqhashlock);
3174}
3175
3176OS_ALWAYS_INLINE
3177static inline void
3178kqworkloop_hash_insert_locked(struct filedesc *fdp, kqueue_id_t id,
3179 struct kqworkloop *kqwl)
3180{
3181 struct kqwllist *list = &fdp->fd_kqhash[KQ_HASH(id, fdp->fd_kqhashmask)];
3182 LIST_INSERT_HEAD(list, kqwl, kqwl_hashlink);
3183}
3184
3185OS_ALWAYS_INLINE
3186static inline struct kqworkloop *
3187kqworkloop_hash_lookup_locked(struct filedesc *fdp, kqueue_id_t id)
3188{
3189 struct kqwllist *list = &fdp->fd_kqhash[KQ_HASH(id, fdp->fd_kqhashmask)];
3190 struct kqworkloop *kqwl;
3191
3192 LIST_FOREACH(kqwl, list, kqwl_hashlink) {
3193 if (kqwl->kqwl_dynamicid == id) {
3194 return kqwl;
3195 }
3196 }
3197 return NULL;
3198}
3199
3200static struct kqworkloop *
3201kqworkloop_hash_lookup_and_retain(struct filedesc *fdp, kqueue_id_t kq_id)
3202{
3203 struct kqworkloop *kqwl = NULL;
3204
3205 kqhash_lock(fdp);
3206 if (__probable(fdp->fd_kqhash)) {
3207 kqwl = kqworkloop_hash_lookup_locked(fdp, id: kq_id);
3208 if (kqwl && !kqworkloop_try_retain(kqwl)) {
3209 kqwl = NULL;
3210 }
3211 }
3212 kqhash_unlock(fdp);
3213 return kqwl;
3214}
3215
3216OS_NOINLINE
3217static void
3218kqworkloop_hash_init(struct filedesc *fdp)
3219{
3220 struct kqwllist *alloc_hash;
3221 u_long alloc_mask;
3222
3223 kqhash_unlock(fdp);
3224 alloc_hash = hashinit(CONFIG_KQ_HASHSIZE, M_KQUEUE, hashmask: &alloc_mask);
3225 kqhash_lock(fdp);
3226
3227 /* See if we won the race */
3228 if (__probable(fdp->fd_kqhashmask == 0)) {
3229 fdp->fd_kqhash = alloc_hash;
3230 fdp->fd_kqhashmask = alloc_mask;
3231 } else {
3232 kqhash_unlock(fdp);
3233 hashdestroy(alloc_hash, M_KQUEUE, hashmask: alloc_mask);
3234 kqhash_lock(fdp);
3235 }
3236}
3237
3238/*
3239 * kqueue iotier override is only supported for kqueue that has
3240 * only one port as a mach port source. Updating the iotier
3241 * override on the mach port source will update the override
3242 * on kqueue as well. Since kqueue with iotier override will
3243 * only have one port attached, there is no logic for saturation
3244 * like qos override, the iotier override of mach port source
3245 * would be reflected in kevent iotier override.
3246 */
3247void
3248kqueue_set_iotier_override(kqueue_t kqu, uint8_t iotier_override)
3249{
3250 if (!(kqu.kq->kq_state & KQ_WORKLOOP)) {
3251 return;
3252 }
3253
3254 struct kqworkloop *kqwl = kqu.kqwl;
3255 os_atomic_store(&kqwl->kqwl_iotier_override, iotier_override, relaxed);
3256}
3257
3258uint8_t
3259kqueue_get_iotier_override(kqueue_t kqu)
3260{
3261 if (!(kqu.kq->kq_state & KQ_WORKLOOP)) {
3262 return THROTTLE_LEVEL_END;
3263 }
3264
3265 struct kqworkloop *kqwl = kqu.kqwl;
3266 return os_atomic_load(&kqwl->kqwl_iotier_override, relaxed);
3267}
3268
3269#if CONFIG_PREADOPT_TG
3270/*
3271 * This function is called with a borrowed reference on the thread group without
3272 * kq lock held with the mqueue lock held. It may or may not have the knote lock
3273 * (called from both fevent as well as fattach/ftouch). Upon success, an
3274 * additional reference on the TG is taken
3275 */
3276void
3277kqueue_set_preadopted_thread_group(kqueue_t kqu, struct thread_group *tg, thread_qos_t qos)
3278{
3279 if (!(kqu.kq->kq_state & KQ_WORKLOOP)) {
3280 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_PREADOPT_NA),
3281 (uintptr_t)thread_tid(current_thread()), 0, 0, 0);
3282 return;
3283 }
3284
3285 struct kqworkloop *kqwl = kqu.kqwl;
3286
3287 assert(qos < THREAD_QOS_LAST);
3288
3289 thread_group_retain(tg);
3290
3291 thread_group_qos_t old_tg; thread_group_qos_t new_tg;
3292 int ret = os_atomic_rmw_loop(&kqwl->kqwl_preadopt_tg, old_tg, new_tg, relaxed, {
3293 if (!KQWL_CAN_ADOPT_PREADOPT_TG(old_tg)) {
3294 os_atomic_rmw_loop_give_up(break);
3295 }
3296
3297 if (old_tg != KQWL_PREADOPTED_TG_NULL) {
3298 /*
3299 * Note that old_tg could be a NULL TG pointer but with a QoS
3300 * set. See also workq_thread_reset_pri.
3301 *
3302 * Compare the QoS of existing preadopted tg with new one and
3303 * only overwrite the thread group if we have one with a higher
3304 * QoS.
3305 */
3306 thread_qos_t existing_qos = KQWL_GET_PREADOPTED_TG_QOS(old_tg);
3307 if (existing_qos >= qos) {
3308 os_atomic_rmw_loop_give_up(break);
3309 }
3310 }
3311
3312 // Transfer the ref taken earlier in the function to the kqwl
3313 new_tg = KQWL_ENCODE_PREADOPTED_TG_QOS(tg, qos);
3314 });
3315
3316 if (ret) {
3317 KQWL_PREADOPT_TG_HISTORY_WRITE_ENTRY(kqwl, KQWL_PREADOPT_OP_INCOMING_IPC, old_tg, tg);
3318
3319 if (KQWL_HAS_VALID_PREADOPTED_TG(old_tg)) {
3320 thread_group_deallocate_safe(KQWL_GET_PREADOPTED_TG(old_tg));
3321 }
3322
3323 os_atomic_store(&kqwl->kqwl_preadopt_tg_needs_redrive, KQWL_PREADOPT_TG_NEEDS_REDRIVE, release);
3324 } else {
3325 // We failed to write to the kqwl_preadopt_tg, drop the ref we took
3326 // earlier in the function
3327 thread_group_deallocate_safe(tg);
3328 }
3329}
3330
3331/*
3332 * Called from fprocess of EVFILT_MACHPORT without the kqueue lock held.
3333 */
3334bool
3335kqueue_process_preadopt_thread_group(thread_t thread, struct kqueue *kq, struct thread_group *tg)
3336{
3337 bool success = false;
3338 if (kq->kq_state & KQ_WORKLOOP) {
3339 struct kqworkloop *kqwl = (struct kqworkloop *) kq;
3340 thread_group_qos_t old_tg;
3341 success = os_atomic_cmpxchgv(&kqwl->kqwl_preadopt_tg,
3342 KQWL_PREADOPTED_TG_SENTINEL, KQWL_PREADOPTED_TG_PROCESSED,
3343 &old_tg, relaxed);
3344 if (success) {
3345 thread_set_preadopt_thread_group(t: thread, tg);
3346 } else if (KQWL_HAS_PERMANENT_PREADOPTED_TG(old_tg)) {
3347 /*
3348 * Technically the following set_preadopt should be a no-op since this
3349 * servicer thread preadopts kqwl's permanent tg at bind time.
3350 * See kqueue_threadreq_bind.
3351 */
3352 thread_set_preadopt_thread_group(t: thread, KQWL_GET_PREADOPTED_TG(old_tg));
3353 } else {
3354 assert(old_tg == KQWL_PREADOPTED_TG_PROCESSED ||
3355 old_tg == KQWL_PREADOPTED_TG_NEVER);
3356 }
3357 }
3358 return success;
3359}
3360#endif
3361
3362/*!
3363 * @function kqworkloop_dealloc
3364 *
3365 * @brief
3366 * Deallocates a workloop kqueue.
3367 *
3368 * @discussion
3369 * Knotes hold references on the workloop, so we can't really reach this
3370 * function unless all of these are already gone.
3371 *
3372 * Nothing locked on entry or exit.
3373 *
3374 * @param hash_remove
3375 * Whether to remove the workloop from its hash table.
3376 */
3377static void
3378kqworkloop_dealloc(struct kqworkloop *kqwl, bool hash_remove)
3379{
3380 thread_t cur_owner;
3381
3382 cur_owner = kqwl->kqwl_owner;
3383 if (cur_owner) {
3384 if (kqworkloop_override(kqwl) != THREAD_QOS_UNSPECIFIED) {
3385 thread_drop_kevent_override(thread: cur_owner);
3386 }
3387 thread_deallocate(thread: cur_owner);
3388 kqwl->kqwl_owner = THREAD_NULL;
3389 }
3390
3391 if (kqwl->kqwl_state & KQ_HAS_TURNSTILE) {
3392 struct turnstile *ts;
3393 turnstile_complete(proprietor: (uintptr_t)kqwl, tstore: &kqwl->kqwl_turnstile,
3394 turnstile: &ts, type: TURNSTILE_WORKLOOPS);
3395 turnstile_cleanup();
3396 turnstile_deallocate(turnstile: ts);
3397 }
3398
3399 if (hash_remove) {
3400 struct filedesc *fdp = &kqwl->kqwl_p->p_fd;
3401
3402 kqhash_lock(fdp);
3403 LIST_REMOVE(kqwl, kqwl_hashlink);
3404#if CONFIG_PROC_RESOURCE_LIMITS
3405 fdp->num_kqwls--;
3406#endif
3407 kqhash_unlock(fdp);
3408 }
3409
3410#if CONFIG_PREADOPT_TG
3411 thread_group_qos_t tg = os_atomic_load(&kqwl->kqwl_preadopt_tg, relaxed);
3412 if (KQWL_HAS_VALID_PREADOPTED_TG(tg)) {
3413 thread_group_release(KQWL_GET_PREADOPTED_TG(tg));
3414 }
3415#endif
3416
3417 assert(TAILQ_EMPTY(&kqwl->kqwl_suppressed));
3418 assert(kqwl->kqwl_owner == THREAD_NULL);
3419 assert(kqwl->kqwl_turnstile == TURNSTILE_NULL);
3420
3421 lck_spin_destroy(lck: &kqwl->kqwl_statelock, grp: &kq_lck_grp);
3422 kqueue_destroy(kqu: kqwl, zone: kqworkloop_zone);
3423}
3424
3425/*!
3426 * @function kqworkloop_init
3427 *
3428 * @brief
3429 * Initializes an allocated kqworkloop.
3430 */
3431static void
3432kqworkloop_init(struct kqworkloop *kqwl, proc_t p,
3433 kqueue_id_t id, workq_threadreq_param_t *trp
3434#if CONFIG_PREADOPT_TG
3435 , struct thread_group *trp_permanent_preadopt_tg
3436#endif
3437 )
3438{
3439 kqwl->kqwl_state = KQ_WORKLOOP | KQ_DYNAMIC | KQ_KEV_QOS;
3440 os_ref_init_raw(&kqwl->kqwl_retains, NULL);
3441 kqwl->kqwl_dynamicid = id;
3442 kqwl->kqwl_p = p;
3443 if (trp) {
3444 kqwl->kqwl_params = trp->trp_value;
3445 }
3446
3447 workq_tr_flags_t tr_flags = WORKQ_TR_FLAG_WORKLOOP;
3448 if (trp) {
3449 if (trp->trp_flags & TRP_PRIORITY) {
3450 tr_flags |= WORKQ_TR_FLAG_WL_OUTSIDE_QOS;
3451 }
3452 if (trp->trp_flags) {
3453 tr_flags |= WORKQ_TR_FLAG_WL_PARAMS;
3454 }
3455 }
3456 kqwl->kqwl_request.tr_state = WORKQ_TR_STATE_IDLE;
3457 kqwl->kqwl_request.tr_flags = tr_flags;
3458 os_atomic_store(&kqwl->kqwl_iotier_override, (uint8_t)THROTTLE_LEVEL_END, relaxed);
3459#if CONFIG_PREADOPT_TG
3460 if (trp_permanent_preadopt_tg) {
3461 /*
3462 * This kqwl is permanently configured with a thread group.
3463 * By using THREAD_QOS_LAST, we make sure kqueue_set_preadopted_thread_group
3464 * has no effect on kqwl_preadopt_tg. At this point, +1 ref on
3465 * trp_permanent_preadopt_tg is transferred to the kqwl.
3466 */
3467 thread_group_qos_t kqwl_preadopt_tg;
3468 kqwl_preadopt_tg = KQWL_ENCODE_PERMANENT_PREADOPTED_TG(trp_permanent_preadopt_tg);
3469 os_atomic_store(&kqwl->kqwl_preadopt_tg, kqwl_preadopt_tg, relaxed);
3470 } else if (task_is_app(task: current_task())) {
3471 /*
3472 * Not a specially preconfigured kqwl so it is open to participate in sync IPC
3473 * thread group preadoption; but, apps will never adopt a thread group that
3474 * is not their own. This is a gross hack to simulate the post-process that
3475 * is done in the voucher subsystem today for thread groups.
3476 */
3477 os_atomic_store(&kqwl->kqwl_preadopt_tg, KQWL_PREADOPTED_TG_NEVER, relaxed);
3478 }
3479#endif
3480
3481 for (int i = 0; i < KQWL_NBUCKETS; i++) {
3482 TAILQ_INIT_AFTER_BZERO(&kqwl->kqwl_queue[i]);
3483 }
3484 TAILQ_INIT_AFTER_BZERO(&kqwl->kqwl_suppressed);
3485
3486 lck_spin_init(lck: &kqwl->kqwl_statelock, grp: &kq_lck_grp, LCK_ATTR_NULL);
3487
3488 kqueue_init(kqu: kqwl);
3489}
3490
3491#if CONFIG_PROC_RESOURCE_LIMITS
3492void
3493kqworkloop_check_limit_exceeded(struct filedesc *fdp)
3494{
3495 int num_kqwls = fdp->num_kqwls;
3496 if (!kqwl_above_soft_limit_notified(fdp) && fdp->kqwl_dyn_soft_limit > 0 &&
3497 num_kqwls > fdp->kqwl_dyn_soft_limit) {
3498 kqwl_above_soft_limit_send_notification(fdp);
3499 act_set_astproc_resource(current_thread());
3500 } else if (!kqwl_above_hard_limit_notified(fdp) && fdp->kqwl_dyn_hard_limit > 0
3501 && num_kqwls > fdp->kqwl_dyn_hard_limit) {
3502 kqwl_above_hard_limit_send_notification(fdp);
3503 act_set_astproc_resource(current_thread());
3504 }
3505}
3506#endif
3507
3508/*!
3509 * @function kqworkloop_get_or_create
3510 *
3511 * @brief
3512 * Wrapper around kqworkloop_init that handles the uniquing of workloops.
3513 *
3514 * @returns
3515 * 0: success
3516 * EINVAL: invalid parameters
3517 * EEXIST: KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST is set and a collision exists.
3518 * ENOENT: KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST is set and the entry wasn't found.
3519 * ENOMEM: allocation failed
3520 */
3521static int
3522kqworkloop_get_or_create(struct proc *p, kqueue_id_t id,
3523 workq_threadreq_param_t *trp,
3524#if CONFIG_PREADOPT_TG
3525 struct thread_group *trp_permanent_preadopt_tg,
3526#endif
3527 unsigned int flags, struct kqworkloop **kqwlp)
3528{
3529 struct filedesc *fdp = &p->p_fd;
3530 struct kqworkloop *alloc_kqwl = NULL;
3531 struct kqworkloop *kqwl = NULL;
3532 int error = 0;
3533
3534 assert(!trp || (flags & KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST));
3535
3536 if (id == 0 || id == (kqueue_id_t)-1) {
3537 return EINVAL;
3538 }
3539
3540 for (;;) {
3541 kqhash_lock(fdp);
3542 if (__improbable(fdp->fd_kqhash == NULL)) {
3543 kqworkloop_hash_init(fdp);
3544 }
3545
3546 kqwl = kqworkloop_hash_lookup_locked(fdp, id);
3547 if (kqwl) {
3548 if (__improbable(flags & KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST)) {
3549 /*
3550 * If MUST_NOT_EXIST was passed, even if we would have failed
3551 * the try_retain, it could have gone the other way, and
3552 * userspace can't tell. Let'em fix their race.
3553 */
3554 error = EEXIST;
3555 break;
3556 }
3557
3558 if (__probable(kqworkloop_try_retain(kqwl))) {
3559 /*
3560 * This is a valid live workloop !
3561 */
3562 *kqwlp = kqwl;
3563 error = 0;
3564 break;
3565 }
3566 }
3567
3568 if (__improbable(flags & KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST)) {
3569 error = ENOENT;
3570 break;
3571 }
3572
3573 /*
3574 * We didn't find what we were looking for.
3575 *
3576 * If this is the second time we reach this point (alloc_kqwl != NULL),
3577 * then we're done.
3578 *
3579 * If this is the first time we reach this point (alloc_kqwl == NULL),
3580 * then try to allocate one without blocking.
3581 */
3582 if (__probable(alloc_kqwl == NULL)) {
3583 alloc_kqwl = zalloc_flags(kqworkloop_zone, Z_NOWAIT | Z_ZERO);
3584 }
3585 if (__probable(alloc_kqwl)) {
3586#if CONFIG_PROC_RESOURCE_LIMITS
3587 fdp->num_kqwls++;
3588 kqworkloop_check_limit_exceeded(fdp);
3589#endif
3590 kqworkloop_init(kqwl: alloc_kqwl, p, id, trp
3591#if CONFIG_PREADOPT_TG
3592 , trp_permanent_preadopt_tg
3593#endif
3594 );
3595 kqworkloop_hash_insert_locked(fdp, id, kqwl: alloc_kqwl);
3596 kqhash_unlock(fdp);
3597 *kqwlp = alloc_kqwl;
3598 return 0;
3599 }
3600
3601 /*
3602 * We have to block to allocate a workloop, drop the lock,
3603 * allocate one, but then we need to retry lookups as someone
3604 * else could race with us.
3605 */
3606 kqhash_unlock(fdp);
3607
3608 alloc_kqwl = zalloc_flags(kqworkloop_zone, Z_WAITOK | Z_ZERO);
3609 }
3610
3611 kqhash_unlock(fdp);
3612
3613 if (__improbable(alloc_kqwl)) {
3614 zfree(kqworkloop_zone, alloc_kqwl);
3615 }
3616
3617 return error;
3618}
3619
3620#pragma mark - knotes
3621
3622static int
3623filt_no_attach(struct knote *kn, __unused struct kevent_qos_s *kev)
3624{
3625 knote_set_error(kn, ENOTSUP);
3626 return 0;
3627}
3628
3629static void
3630filt_no_detach(__unused struct knote *kn)
3631{
3632}
3633
3634static int __dead2
3635filt_bad_event(struct knote *kn, long hint)
3636{
3637 panic("%s[%d](%p, %ld)", __func__, kn->kn_filter, kn, hint);
3638}
3639
3640static int __dead2
3641filt_bad_touch(struct knote *kn, struct kevent_qos_s *kev)
3642{
3643 panic("%s[%d](%p, %p)", __func__, kn->kn_filter, kn, kev);
3644}
3645
3646static int __dead2
3647filt_bad_process(struct knote *kn, struct kevent_qos_s *kev)
3648{
3649 panic("%s[%d](%p, %p)", __func__, kn->kn_filter, kn, kev);
3650}
3651
3652/*
3653 * knotes_dealloc - detach all knotes for the process and drop them
3654 *
3655 * Process is in such a state that it will not try to allocate
3656 * any more knotes during this process (stopped for exit or exec).
3657 */
3658void
3659knotes_dealloc(proc_t p)
3660{
3661 struct filedesc *fdp = &p->p_fd;
3662 struct kqueue *kq;
3663 struct knote *kn;
3664 struct klist *kn_hash = NULL;
3665 u_long kn_hashmask;
3666 int i;
3667
3668 proc_fdlock(p);
3669
3670 /* Close all the fd-indexed knotes up front */
3671 if (fdp->fd_knlistsize > 0) {
3672 for (i = 0; i < fdp->fd_knlistsize; i++) {
3673 while ((kn = SLIST_FIRST(&fdp->fd_knlist[i])) != NULL) {
3674 kq = knote_get_kq(kn);
3675 kqlock(kqu: kq);
3676 proc_fdunlock(p);
3677 knote_drop(kqu: kq, kn, NULL);
3678 proc_fdlock(p);
3679 }
3680 }
3681 /* free the table */
3682 kfree_type(struct klist, fdp->fd_knlistsize, fdp->fd_knlist);
3683 }
3684 fdp->fd_knlistsize = 0;
3685
3686 proc_fdunlock(p);
3687
3688 knhash_lock(fdp);
3689
3690 /* Clean out all the hashed knotes as well */
3691 if (fdp->fd_knhashmask != 0) {
3692 for (i = 0; i <= (int)fdp->fd_knhashmask; i++) {
3693 while ((kn = SLIST_FIRST(&fdp->fd_knhash[i])) != NULL) {
3694 kq = knote_get_kq(kn);
3695 kqlock(kqu: kq);
3696 knhash_unlock(fdp);
3697 knote_drop(kqu: kq, kn, NULL);
3698 knhash_lock(fdp);
3699 }
3700 }
3701 kn_hash = fdp->fd_knhash;
3702 kn_hashmask = fdp->fd_knhashmask;
3703 fdp->fd_knhashmask = 0;
3704 fdp->fd_knhash = NULL;
3705 }
3706
3707 knhash_unlock(fdp);
3708
3709 if (kn_hash) {
3710 hashdestroy(kn_hash, M_KQUEUE, hashmask: kn_hashmask);
3711 }
3712}
3713
3714/*
3715 * kqworkloops_dealloc - rebalance retains on kqworkloops created with
3716 * scheduling parameters
3717 *
3718 * Process is in such a state that it will not try to allocate
3719 * any more kqs or knotes during this process (stopped for exit or exec).
3720 */
3721void
3722kqworkloops_dealloc(proc_t p)
3723{
3724 struct filedesc *fdp = &p->p_fd;
3725 struct kqworkloop *kqwl, *kqwln;
3726 struct kqwllist tofree;
3727
3728 if (!fdt_flag_test(fdp, FD_WORKLOOP)) {
3729 return;
3730 }
3731
3732 kqhash_lock(fdp);
3733
3734 if (fdp->fd_kqhashmask == 0) {
3735 kqhash_unlock(fdp);
3736 return;
3737 }
3738
3739 LIST_INIT(&tofree);
3740
3741 for (size_t i = 0; i <= fdp->fd_kqhashmask; i++) {
3742 LIST_FOREACH_SAFE(kqwl, &fdp->fd_kqhash[i], kqwl_hashlink, kqwln) {
3743#if CONFIG_PREADOPT_TG
3744 /*
3745 * kqworkloops that have scheduling parameters have an
3746 * implicit retain from kqueue_workloop_ctl that needs
3747 * to be balanced on process exit.
3748 */
3749 __assert_only thread_group_qos_t preadopt_tg;
3750 preadopt_tg = os_atomic_load(&kqwl->kqwl_preadopt_tg, relaxed);
3751#endif
3752 assert(kqwl->kqwl_params
3753#if CONFIG_PREADOPT_TG
3754 || KQWL_HAS_PERMANENT_PREADOPTED_TG(preadopt_tg)
3755#endif
3756 );
3757
3758 LIST_REMOVE(kqwl, kqwl_hashlink);
3759 LIST_INSERT_HEAD(&tofree, kqwl, kqwl_hashlink);
3760 }
3761 }
3762#if CONFIG_PROC_RESOURCE_LIMITS
3763 fdp->num_kqwls = 0;
3764#endif
3765 kqhash_unlock(fdp);
3766
3767 LIST_FOREACH_SAFE(kqwl, &tofree, kqwl_hashlink, kqwln) {
3768 uint32_t ref = os_ref_get_count_raw(rc: &kqwl->kqwl_retains);
3769 if (ref != 1) {
3770 panic("kq(%p) invalid refcount %d", kqwl, ref);
3771 }
3772 kqworkloop_dealloc(kqwl, false);
3773 }
3774}
3775
3776static int
3777kevent_register_validate_priority(struct kqueue *kq, struct knote *kn,
3778 struct kevent_qos_s *kev)
3779{
3780 /* We don't care about the priority of a disabled or deleted knote */
3781 if (kev->flags & (EV_DISABLE | EV_DELETE)) {
3782 return 0;
3783 }
3784
3785 if (kq->kq_state & KQ_WORKLOOP) {
3786 /*
3787 * Workloops need valid priorities with a QOS (excluding manager) for
3788 * any enabled knote.
3789 *
3790 * When it is pre-existing, just make sure it has a valid QoS as
3791 * kevent_register() will not use the incoming priority (filters who do
3792 * have the responsibility to validate it again, see filt_wltouch).
3793 *
3794 * If the knote is being made, validate the incoming priority.
3795 */
3796 if (!_pthread_priority_thread_qos(pp: kn ? kn->kn_qos : kev->qos)) {
3797 return ERANGE;
3798 }
3799 }
3800
3801 return 0;
3802}
3803
3804/*
3805 * Prepare a filter for waiting after register.
3806 *
3807 * The f_post_register_wait hook will be called later by kevent_register()
3808 * and should call kevent_register_wait_block()
3809 */
3810static int
3811kevent_register_wait_prepare(struct knote *kn, struct kevent_qos_s *kev, int rc)
3812{
3813 thread_t thread = current_thread();
3814
3815 assert(knote_fops(kn)->f_extended_codes);
3816
3817 if (kn->kn_thread == NULL) {
3818 thread_reference(thread);
3819 kn->kn_thread = thread;
3820 } else if (kn->kn_thread != thread) {
3821 /*
3822 * kn_thread may be set from a previous aborted wait
3823 * However, it has to be from the same thread.
3824 */
3825 kev->flags |= EV_ERROR;
3826 kev->data = EXDEV;
3827 return 0;
3828 }
3829
3830 return FILTER_REGISTER_WAIT | rc;
3831}
3832
3833/*
3834 * Cleanup a kevent_register_wait_prepare() effect for threads that have been
3835 * aborted instead of properly woken up with thread_wakeup_thread().
3836 */
3837static void
3838kevent_register_wait_cleanup(struct knote *kn)
3839{
3840 thread_t thread = kn->kn_thread;
3841 kn->kn_thread = NULL;
3842 thread_deallocate(thread);
3843}
3844
3845/*
3846 * Must be called at the end of a f_post_register_wait call from a filter.
3847 */
3848static void
3849kevent_register_wait_block(struct turnstile *ts, thread_t thread,
3850 thread_continue_t cont, struct _kevent_register *cont_args)
3851{
3852 turnstile_update_inheritor_complete(turnstile: ts, flags: TURNSTILE_INTERLOCK_HELD);
3853 kqunlock(kqu: cont_args->kqwl);
3854 cont_args->handoff_thread = thread;
3855 thread_handoff_parameter(thread, continuation: cont, parameter: cont_args, THREAD_HANDOFF_NONE);
3856}
3857
3858/*
3859 * Called by Filters using a f_post_register_wait to return from their wait.
3860 */
3861static void
3862kevent_register_wait_return(struct _kevent_register *cont_args)
3863{
3864 struct kqworkloop *kqwl = cont_args->kqwl;
3865 struct kevent_qos_s *kev = &cont_args->kev;
3866 int error = 0;
3867
3868 if (cont_args->handoff_thread) {
3869 thread_deallocate(thread: cont_args->handoff_thread);
3870 }
3871
3872 if (kev->flags & (EV_ERROR | EV_RECEIPT)) {
3873 if ((kev->flags & EV_ERROR) == 0) {
3874 kev->flags |= EV_ERROR;
3875 kev->data = 0;
3876 }
3877 error = kevent_modern_copyout(kev, &cont_args->ueventlist);
3878 if (error == 0) {
3879 cont_args->eventout++;
3880 }
3881 }
3882
3883 kqworkloop_release(kqwl);
3884 if (error == 0) {
3885 *(int32_t *)&current_uthread()->uu_rval = cont_args->eventout;
3886 }
3887 unix_syscall_return(error);
3888}
3889
3890/*
3891 * kevent_register - add a new event to a kqueue
3892 *
3893 * Creates a mapping between the event source and
3894 * the kqueue via a knote data structure.
3895 *
3896 * Because many/most the event sources are file
3897 * descriptor related, the knote is linked off
3898 * the filedescriptor table for quick access.
3899 *
3900 * called with nothing locked
3901 * caller holds a reference on the kqueue
3902 */
3903
3904int
3905kevent_register(struct kqueue *kq, struct kevent_qos_s *kev,
3906 struct knote **kn_out)
3907{
3908 struct proc *p = kq->kq_p;
3909 const struct filterops *fops;
3910 struct knote *kn = NULL;
3911 int result = 0, error = 0;
3912 unsigned short kev_flags = kev->flags;
3913 KNOTE_LOCK_CTX(knlc);
3914
3915 if (__probable(kev->filter < 0 && kev->filter + EVFILT_SYSCOUNT >= 0)) {
3916 fops = sysfilt_ops[~kev->filter]; /* to 0-base index */
3917 } else {
3918 error = EINVAL;
3919 goto out;
3920 }
3921
3922 /* restrict EV_VANISHED to adding udata-specific dispatch kevents */
3923 if (__improbable((kev->flags & EV_VANISHED) &&
3924 (kev->flags & (EV_ADD | EV_DISPATCH2)) != (EV_ADD | EV_DISPATCH2))) {
3925 error = EINVAL;
3926 goto out;
3927 }
3928
3929 /* Simplify the flags - delete and disable overrule */
3930 if (kev->flags & EV_DELETE) {
3931 kev->flags &= ~EV_ADD;
3932 }
3933 if (kev->flags & EV_DISABLE) {
3934 kev->flags &= ~EV_ENABLE;
3935 }
3936
3937 if (kq->kq_state & KQ_WORKLOOP) {
3938 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_REGISTER),
3939 ((struct kqworkloop *)kq)->kqwl_dynamicid,
3940 kev->udata, kev->flags, kev->filter);
3941 } else if (kq->kq_state & KQ_WORKQ) {
3942 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWQ_REGISTER),
3943 0, kev->udata, kev->flags, kev->filter);
3944 } else {
3945 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_REGISTER),
3946 VM_KERNEL_UNSLIDE_OR_PERM(kq),
3947 kev->udata, kev->flags, kev->filter);
3948 }
3949
3950restart:
3951 /* find the matching knote from the fd tables/hashes */
3952 kn = kq_find_knote_and_kq_lock(kq, kev, is_fd: fops->f_isfd, p);
3953 error = kevent_register_validate_priority(kq, kn, kev);
3954 result = 0;
3955 if (error) {
3956 if (kn) {
3957 kqunlock(kqu: kq);
3958 }
3959 goto out;
3960 }
3961
3962 if (kn == NULL && (kev->flags & EV_ADD) == 0) {
3963 /*
3964 * No knote found, EV_ADD wasn't specified
3965 */
3966
3967 if ((kev_flags & EV_ADD) && (kev_flags & EV_DELETE) &&
3968 (kq->kq_state & KQ_WORKLOOP)) {
3969 /*
3970 * For workloops, understand EV_ADD|EV_DELETE as a "soft" delete
3971 * that doesn't care about ENOENT, so just pretend the deletion
3972 * happened.
3973 */
3974 } else {
3975 error = ENOENT;
3976 }
3977 goto out;
3978 } else if (kn == NULL) {
3979 /*
3980 * No knote found, need to attach a new one (attach)
3981 */
3982
3983 struct fileproc *knote_fp = NULL;
3984
3985 /* grab a file reference for the new knote */
3986 if (fops->f_isfd) {
3987 if ((error = fp_lookup(p, fd: (int)kev->ident, resultfp: &knote_fp, locked: 0)) != 0) {
3988 goto out;
3989 }
3990 }
3991
3992 kn = knote_alloc();
3993 kn->kn_fp = knote_fp;
3994 kn->kn_is_fd = fops->f_isfd;
3995 kn->kn_kq_packed = VM_PACK_POINTER((vm_offset_t)kq, KNOTE_KQ_PACKED);
3996 kn->kn_status = 0;
3997
3998 /* was vanish support requested */
3999 if (kev->flags & EV_VANISHED) {
4000 kev->flags &= ~EV_VANISHED;
4001 kn->kn_status |= KN_REQVANISH;
4002 }
4003
4004 /* snapshot matching/dispatching protocol flags into knote */
4005 if (kev->flags & EV_DISABLE) {
4006 kn->kn_status |= KN_DISABLED;
4007 }
4008
4009 /*
4010 * copy the kevent state into knote
4011 * protocol is that fflags and data
4012 * are saved off, and cleared before
4013 * calling the attach routine.
4014 *
4015 * - kn->kn_sfflags aliases with kev->xflags
4016 * - kn->kn_sdata aliases with kev->data
4017 * - kn->kn_filter is the top 8 bits of kev->filter
4018 */
4019 kn->kn_kevent = *(struct kevent_internal_s *)kev;
4020 kn->kn_sfflags = kev->fflags;
4021 kn->kn_filtid = (uint8_t)~kev->filter;
4022 kn->kn_fflags = 0;
4023 knote_reset_priority(kqu: kq, kn, pp: kev->qos);
4024
4025 /* Add the knote for lookup thru the fd table */
4026 error = kq_add_knote(kq, kn, knlc: &knlc, p);
4027 if (error) {
4028 knote_free(kn);
4029 if (knote_fp != NULL) {
4030 fp_drop(p, fd: (int)kev->ident, fp: knote_fp, locked: 0);
4031 }
4032
4033 if (error == ERESTART) {
4034 goto restart;
4035 }
4036 goto out;
4037 }
4038
4039 /* fp reference count now applies to knote */
4040
4041 /*
4042 * we can't use filter_call() because f_attach can change the filter ops
4043 * for a filter that supports f_extended_codes, so we need to reload
4044 * knote_fops() and not use `fops`.
4045 */
4046 result = fops->f_attach(kn, kev);
4047 if (result && !knote_fops(kn)->f_extended_codes) {
4048 result = FILTER_ACTIVE;
4049 }
4050
4051 kqlock(kqu: kq);
4052
4053 if (result & FILTER_THREADREQ_NODEFEER) {
4054 enable_preemption();
4055 }
4056
4057 if (kn->kn_flags & EV_ERROR) {
4058 /*
4059 * Failed to attach correctly, so drop.
4060 */
4061 kn->kn_filtid = EVFILTID_DETACHED;
4062 error = (int)kn->kn_sdata;
4063 knote_drop(kqu: kq, kn, knlc: &knlc);
4064 result = 0;
4065 goto out;
4066 }
4067
4068 /*
4069 * end "attaching" phase - now just attached
4070 *
4071 * Mark the thread request overcommit, if appropos
4072 *
4073 * If the attach routine indicated that an
4074 * event is already fired, activate the knote.
4075 */
4076 if ((kn->kn_qos & _PTHREAD_PRIORITY_OVERCOMMIT_FLAG) &&
4077 (kq->kq_state & KQ_WORKLOOP)) {
4078 kqworkloop_set_overcommit(kqwl: (struct kqworkloop *)kq);
4079 }
4080 } else if (!knote_lock(kqu: kq, kn, knlc: &knlc, kqlocking: KNOTE_KQ_LOCK_ON_SUCCESS)) {
4081 /*
4082 * The knote was dropped while we were waiting for the lock,
4083 * we need to re-evaluate entirely
4084 */
4085
4086 goto restart;
4087 } else if (kev->flags & EV_DELETE) {
4088 /*
4089 * Deletion of a knote (drop)
4090 *
4091 * If the filter wants to filter drop events, let it do so.
4092 *
4093 * defer-delete: when trying to delete a disabled EV_DISPATCH2 knote,
4094 * we must wait for the knote to be re-enabled (unless it is being
4095 * re-enabled atomically here).
4096 */
4097
4098 if (knote_fops(kn)->f_allow_drop) {
4099 bool drop;
4100
4101 kqunlock(kqu: kq);
4102 drop = knote_fops(kn)->f_allow_drop(kn, kev);
4103 kqlock(kqu: kq);
4104
4105 if (!drop) {
4106 goto out_unlock;
4107 }
4108 }
4109
4110 if ((kev->flags & EV_ENABLE) == 0 &&
4111 (kn->kn_flags & EV_DISPATCH2) == EV_DISPATCH2 &&
4112 (kn->kn_status & KN_DISABLED) != 0) {
4113 kn->kn_status |= KN_DEFERDELETE;
4114 error = EINPROGRESS;
4115 goto out_unlock;
4116 }
4117
4118 knote_drop(kqu: kq, kn, knlc: &knlc);
4119 goto out;
4120 } else {
4121 /*
4122 * Regular update of a knote (touch)
4123 *
4124 * Call touch routine to notify filter of changes in filter values
4125 * (and to re-determine if any events are fired).
4126 *
4127 * If the knote is in defer-delete, avoid calling the filter touch
4128 * routine (it has delivered its last event already).
4129 *
4130 * If the touch routine had no failure,
4131 * apply the requested side effects to the knote.
4132 */
4133
4134 if (kn->kn_status & (KN_DEFERDELETE | KN_VANISHED)) {
4135 if (kev->flags & EV_ENABLE) {
4136 result = FILTER_ACTIVE;
4137 }
4138 } else {
4139 kqunlock(kqu: kq);
4140 result = filter_call(knote_fops(kn), f_touch(kn, kev));
4141 kqlock(kqu: kq);
4142 if (result & FILTER_THREADREQ_NODEFEER) {
4143 enable_preemption();
4144 }
4145 }
4146
4147 if (kev->flags & EV_ERROR) {
4148 result = 0;
4149 goto out_unlock;
4150 }
4151
4152 if ((kn->kn_flags & EV_UDATA_SPECIFIC) == 0 &&
4153 kn->kn_udata != kev->udata) {
4154 // this allows klist_copy_udata() not to take locks
4155 os_atomic_store_wide(&kn->kn_udata, kev->udata, relaxed);
4156 }
4157 if ((kev->flags & EV_DISABLE) && !(kn->kn_status & KN_DISABLED)) {
4158 kn->kn_status |= KN_DISABLED;
4159 knote_dequeue(kqu: kq, kn);
4160 }
4161 }
4162
4163 /* accept new kevent state */
4164 knote_apply_touch(kqu: kq, kn, kev, result);
4165
4166out_unlock:
4167 /*
4168 * When the filter asked for a post-register wait,
4169 * we leave the kqueue locked for kevent_register()
4170 * to call the filter's f_post_register_wait hook.
4171 */
4172 if (result & FILTER_REGISTER_WAIT) {
4173 knote_unlock(kqu: kq, kn, knlc: &knlc, kqlocking: KNOTE_KQ_LOCK_ALWAYS);
4174 *kn_out = kn;
4175 } else {
4176 knote_unlock(kqu: kq, kn, knlc: &knlc, kqlocking: KNOTE_KQ_UNLOCK);
4177 }
4178
4179out:
4180 /* output local errors through the kevent */
4181 if (error) {
4182 kev->flags |= EV_ERROR;
4183 kev->data = error;
4184 }
4185 return result;
4186}
4187
4188/*
4189 * knote_process - process a triggered event
4190 *
4191 * Validate that it is really still a triggered event
4192 * by calling the filter routines (if necessary). Hold
4193 * a use reference on the knote to avoid it being detached.
4194 *
4195 * If it is still considered triggered, we will have taken
4196 * a copy of the state under the filter lock. We use that
4197 * snapshot to dispatch the knote for future processing (or
4198 * not, if this was a lost event).
4199 *
4200 * Our caller assures us that nobody else can be processing
4201 * events from this knote during the whole operation. But
4202 * others can be touching or posting events to the knote
4203 * interspersed with our processing it.
4204 *
4205 * caller holds a reference on the kqueue.
4206 * kqueue locked on entry and exit - but may be dropped
4207 */
4208static int
4209knote_process(struct knote *kn, kevent_ctx_t kectx,
4210 kevent_callback_t callback)
4211{
4212 struct kevent_qos_s kev;
4213 struct kqueue *kq = knote_get_kq(kn);
4214 KNOTE_LOCK_CTX(knlc);
4215 int result = FILTER_ACTIVE;
4216 int error = 0;
4217 bool drop = false;
4218
4219 /*
4220 * Must be active
4221 * Must be queued and not disabled/suppressed or dropping
4222 */
4223 assert(kn->kn_status & KN_QUEUED);
4224 assert(kn->kn_status & KN_ACTIVE);
4225 assert(!(kn->kn_status & (KN_DISABLED | KN_SUPPRESSED | KN_DROPPING)));
4226
4227 if (kq->kq_state & KQ_WORKLOOP) {
4228 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS),
4229 ((struct kqworkloop *)kq)->kqwl_dynamicid,
4230 kn->kn_udata, kn->kn_status | (kn->kn_id << 32),
4231 kn->kn_filtid);
4232 } else if (kq->kq_state & KQ_WORKQ) {
4233 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWQ_PROCESS),
4234 0, kn->kn_udata, kn->kn_status | (kn->kn_id << 32),
4235 kn->kn_filtid);
4236 } else {
4237 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_PROCESS),
4238 VM_KERNEL_UNSLIDE_OR_PERM(kq), kn->kn_udata,
4239 kn->kn_status | (kn->kn_id << 32), kn->kn_filtid);
4240 }
4241
4242 if (!knote_lock(kqu: kq, kn, knlc: &knlc, kqlocking: KNOTE_KQ_LOCK_ALWAYS)) {
4243 /*
4244 * When the knote is dropping or has dropped,
4245 * then there's nothing we want to process.
4246 */
4247 return EJUSTRETURN;
4248 }
4249
4250 /*
4251 * While waiting for the knote lock, we may have dropped the kq lock.
4252 * and a touch may have disabled and dequeued the knote.
4253 */
4254 if (!(kn->kn_status & KN_QUEUED)) {
4255 knote_unlock(kqu: kq, kn, knlc: &knlc, kqlocking: KNOTE_KQ_LOCK_ALWAYS);
4256 return EJUSTRETURN;
4257 }
4258
4259 /*
4260 * For deferred-drop or vanished events, we just create a fake
4261 * event to acknowledge end-of-life. Otherwise, we call the
4262 * filter's process routine to snapshot the kevent state under
4263 * the filter's locking protocol.
4264 *
4265 * suppress knotes to avoid returning the same event multiple times in
4266 * a single call.
4267 */
4268 knote_suppress(kqu: kq, kn);
4269
4270 if (kn->kn_status & (KN_DEFERDELETE | KN_VANISHED)) {
4271 uint16_t kev_flags = EV_DISPATCH2 | EV_ONESHOT;
4272 if (kn->kn_status & KN_DEFERDELETE) {
4273 kev_flags |= EV_DELETE;
4274 } else {
4275 kev_flags |= EV_VANISHED;
4276 }
4277
4278 /* create fake event */
4279 kev = (struct kevent_qos_s){
4280 .filter = kn->kn_filter,
4281 .ident = kn->kn_id,
4282 .flags = kev_flags,
4283 .udata = kn->kn_udata,
4284 };
4285 } else {
4286 kqunlock(kqu: kq);
4287 kev = (struct kevent_qos_s) { };
4288 result = filter_call(knote_fops(kn), f_process(kn, &kev));
4289 kqlock(kqu: kq);
4290 }
4291
4292 /*
4293 * Determine how to dispatch the knote for future event handling.
4294 * not-fired: just return (do not callout, leave deactivated).
4295 * One-shot: If dispatch2, enter deferred-delete mode (unless this is
4296 * is the deferred delete event delivery itself). Otherwise,
4297 * drop it.
4298 * Dispatch: don't clear state, just mark it disabled.
4299 * Cleared: just leave it deactivated.
4300 * Others: re-activate as there may be more events to handle.
4301 * This will not wake up more handlers right now, but
4302 * at the completion of handling events it may trigger
4303 * more handler threads (TODO: optimize based on more than
4304 * just this one event being detected by the filter).
4305 */
4306 if ((result & FILTER_ACTIVE) == 0) {
4307 if ((kn->kn_status & KN_ACTIVE) == 0) {
4308 /*
4309 * Some knotes (like EVFILT_WORKLOOP) can be reactivated from
4310 * within f_process() but that doesn't necessarily make them
4311 * ready to process, so we should leave them be.
4312 *
4313 * For other knotes, since we will not return an event,
4314 * there's no point keeping the knote suppressed.
4315 */
4316 knote_unsuppress(kqu: kq, kn);
4317 }
4318 knote_unlock(kqu: kq, kn, knlc: &knlc, kqlocking: KNOTE_KQ_LOCK_ALWAYS);
4319 return EJUSTRETURN;
4320 }
4321
4322 if (result & FILTER_ADJUST_EVENT_QOS_BIT) {
4323 knote_adjust_qos(kq, kn, result);
4324 }
4325
4326 if (result & FILTER_ADJUST_EVENT_IOTIER_BIT) {
4327 kqueue_update_iotier_override(kqu: kq);
4328 }
4329
4330 kev.qos = _pthread_priority_combine(base_pp: kn->kn_qos, qos: kn->kn_qos_override);
4331
4332 if (kev.flags & EV_ONESHOT) {
4333 if ((kn->kn_flags & EV_DISPATCH2) == EV_DISPATCH2 &&
4334 (kn->kn_status & KN_DEFERDELETE) == 0) {
4335 /* defer dropping non-delete oneshot dispatch2 events */
4336 kn->kn_status |= KN_DEFERDELETE | KN_DISABLED;
4337 } else {
4338 drop = true;
4339 }
4340 } else if (kn->kn_flags & EV_DISPATCH) {
4341 /* disable all dispatch knotes */
4342 kn->kn_status |= KN_DISABLED;
4343 } else if ((kn->kn_flags & EV_CLEAR) == 0) {
4344 /* re-activate in case there are more events */
4345 knote_activate(kqu: kq, kn, FILTER_ACTIVE);
4346 }
4347
4348 /*
4349 * callback to handle each event as we find it.
4350 * If we have to detach and drop the knote, do
4351 * it while we have the kq unlocked.
4352 */
4353 if (drop) {
4354 knote_drop(kqu: kq, kn, knlc: &knlc);
4355 } else {
4356 knote_unlock(kqu: kq, kn, knlc: &knlc, kqlocking: KNOTE_KQ_UNLOCK);
4357 }
4358
4359 if (kev.flags & EV_VANISHED) {
4360 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KNOTE_VANISHED),
4361 kev.ident, kn->kn_udata, kn->kn_status | (kn->kn_id << 32),
4362 kn->kn_filtid);
4363 }
4364
4365 error = (callback)(&kev, kectx);
4366 kqlock(kqu: kq);
4367 return error;
4368}
4369
4370/*
4371 * Returns -1 if the kqueue was unbound and processing should not happen
4372 */
4373#define KQWQAE_BEGIN_PROCESSING 1
4374#define KQWQAE_END_PROCESSING 2
4375#define KQWQAE_UNBIND 3
4376static int
4377kqworkq_acknowledge_events(struct kqworkq *kqwq, workq_threadreq_t kqr,
4378 int kevent_flags, int kqwqae_op)
4379{
4380 struct knote *kn;
4381 int rc = 0;
4382 bool unbind;
4383 struct kqtailq *suppressq = &kqwq->kqwq_suppressed[kqr->tr_kq_qos_index - 1];
4384 struct kqtailq *queue = &kqwq->kqwq_queue[kqr->tr_kq_qos_index - 1];
4385
4386 kqlock_held(kqu: &kqwq->kqwq_kqueue);
4387
4388 /*
4389 * Return suppressed knotes to their original state.
4390 * For workq kqueues, suppressed ones that are still
4391 * truly active (not just forced into the queue) will
4392 * set flags we check below to see if anything got
4393 * woken up.
4394 */
4395 while ((kn = TAILQ_FIRST(suppressq)) != NULL) {
4396 knote_unsuppress(kqu: kqwq, kn);
4397 }
4398
4399 if (kqwqae_op == KQWQAE_UNBIND) {
4400 unbind = true;
4401 } else if ((kevent_flags & KEVENT_FLAG_PARKING) == 0) {
4402 unbind = false;
4403 } else {
4404 unbind = TAILQ_EMPTY(queue);
4405 }
4406 if (unbind) {
4407 thread_t thread = kqr_thread_fast(kqr);
4408 thread_qos_t old_override;
4409
4410#if DEBUG || DEVELOPMENT
4411 thread_t self = current_thread();
4412 struct uthread *ut = get_bsdthread_info(self);
4413
4414 assert(thread == self);
4415 assert(ut->uu_kqr_bound == kqr);
4416#endif // DEBUG || DEVELOPMENT
4417
4418 old_override = kqworkq_unbind_locked(kqwq, kqr, thread);
4419 if (!TAILQ_EMPTY(queue)) {
4420 /*
4421 * Request a new thread if we didn't process the whole
4422 * queue.
4423 */
4424 kqueue_threadreq_initiate(kq: &kqwq->kqwq_kqueue, kqr,
4425 qos: kqr->tr_kq_qos_index, flags: 0);
4426 }
4427 if (old_override) {
4428 thread_drop_kevent_override(thread);
4429 }
4430 rc = -1;
4431 }
4432
4433 return rc;
4434}
4435
4436/*
4437 * Return 0 to indicate that processing should proceed,
4438 * -1 if there is nothing to process.
4439 *
4440 * Called with kqueue locked and returns the same way,
4441 * but may drop lock temporarily.
4442 */
4443static int
4444kqworkq_begin_processing(struct kqworkq *kqwq, workq_threadreq_t kqr,
4445 int kevent_flags)
4446{
4447 int rc = 0;
4448
4449 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWQ_PROCESS_BEGIN) | DBG_FUNC_START,
4450 0, kqr->tr_kq_qos_index);
4451
4452 rc = kqworkq_acknowledge_events(kqwq, kqr, kevent_flags,
4453 KQWQAE_BEGIN_PROCESSING);
4454
4455 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWQ_PROCESS_BEGIN) | DBG_FUNC_END,
4456 thread_tid(kqr_thread(kqr)),
4457 !TAILQ_EMPTY(&kqwq->kqwq_queue[kqr->tr_kq_qos_index - 1]));
4458
4459 return rc;
4460}
4461
4462static thread_qos_t
4463kqworkloop_acknowledge_events(struct kqworkloop *kqwl)
4464{
4465 kq_index_t qos = THREAD_QOS_UNSPECIFIED;
4466 struct knote *kn, *tmp;
4467
4468 kqlock_held(kqu: kqwl);
4469
4470 TAILQ_FOREACH_SAFE(kn, &kqwl->kqwl_suppressed, kn_tqe, tmp) {
4471 /*
4472 * If a knote that can adjust QoS is disabled because of the automatic
4473 * behavior of EV_DISPATCH, the knotes should stay suppressed so that
4474 * further overrides keep pushing.
4475 */
4476 if (knote_fops(kn)->f_adjusts_qos &&
4477 (kn->kn_status & KN_DISABLED) != 0 &&
4478 (kn->kn_status & KN_DROPPING) == 0 &&
4479 (kn->kn_flags & (EV_DISPATCH | EV_DISABLE)) == EV_DISPATCH) {
4480 qos = MAX(qos, kn->kn_qos_override);
4481 continue;
4482 }
4483 knote_unsuppress(kqu: kqwl, kn);
4484 }
4485
4486 return qos;
4487}
4488
4489static int
4490kqworkloop_begin_processing(struct kqworkloop *kqwl, unsigned int kevent_flags)
4491{
4492 workq_threadreq_t kqr = &kqwl->kqwl_request;
4493 struct kqueue *kq = &kqwl->kqwl_kqueue;
4494 int rc = 0, op = KQWL_UTQ_NONE;
4495
4496 kqlock_held(kqu: kq);
4497
4498 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_BEGIN) | DBG_FUNC_START,
4499 kqwl->kqwl_dynamicid, 0, 0);
4500
4501 /* nobody else should still be processing */
4502 assert((kq->kq_state & KQ_PROCESSING) == 0);
4503
4504 kq->kq_state |= KQ_PROCESSING;
4505
4506 if (kevent_flags & KEVENT_FLAG_PARKING) {
4507 /*
4508 * When "parking" we want to process events and if no events are found
4509 * unbind.
4510 *
4511 * However, non overcommit threads sometimes park even when they have
4512 * more work so that the pool can narrow. For these, we need to unbind
4513 * early, so that calling kqworkloop_update_threads_qos() can ask the
4514 * workqueue subsystem whether the thread should park despite having
4515 * pending events.
4516 */
4517 if (kqr->tr_flags & WORKQ_TR_FLAG_OVERCOMMIT) {
4518 op = KQWL_UTQ_PARKING;
4519 } else {
4520 op = KQWL_UTQ_UNBINDING;
4521 }
4522 } else if (!TAILQ_EMPTY(&kqwl->kqwl_suppressed)) {
4523 op = KQWL_UTQ_RESET_WAKEUP_OVERRIDE;
4524 }
4525
4526 if (op != KQWL_UTQ_NONE) {
4527 thread_qos_t qos_override;
4528 thread_t thread = kqr_thread_fast(kqr);
4529
4530 qos_override = kqworkloop_acknowledge_events(kqwl);
4531
4532 if (op == KQWL_UTQ_UNBINDING) {
4533 kqworkloop_unbind_locked(kwql: kqwl, thread,
4534 how: KQWL_OVERRIDE_DROP_IMMEDIATELY);
4535 kqworkloop_release_live(kqwl);
4536 }
4537 kqworkloop_update_threads_qos(kqwl, op, qos: qos_override);
4538 if (op == KQWL_UTQ_PARKING &&
4539 (!kqwl->kqwl_count || kqwl->kqwl_owner)) {
4540 kqworkloop_unbind_locked(kwql: kqwl, thread,
4541 how: KQWL_OVERRIDE_DROP_DELAYED);
4542 kqworkloop_release_live(kqwl);
4543 rc = -1;
4544 } else if (op == KQWL_UTQ_UNBINDING &&
4545 kqr_thread(kqr) != thread) {
4546 rc = -1;
4547 }
4548
4549 if (rc == -1) {
4550 kq->kq_state &= ~KQ_PROCESSING;
4551 kqworkloop_unbind_delayed_override_drop(thread);
4552 }
4553 }
4554
4555 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_BEGIN) | DBG_FUNC_END,
4556 kqwl->kqwl_dynamicid, 0, 0);
4557
4558 return rc;
4559}
4560
4561/*
4562 * Return 0 to indicate that processing should proceed,
4563 * -1 if there is nothing to process.
4564 * EBADF if the kqueue is draining
4565 *
4566 * Called with kqueue locked and returns the same way,
4567 * but may drop lock temporarily.
4568 * May block.
4569 */
4570static int
4571kqfile_begin_processing(struct kqfile *kq)
4572{
4573 kqlock_held(kqu: kq);
4574
4575 assert((kq->kqf_state & (KQ_WORKQ | KQ_WORKLOOP)) == 0);
4576 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN) | DBG_FUNC_START,
4577 VM_KERNEL_UNSLIDE_OR_PERM(kq), 0);
4578
4579 /* wait to become the exclusive processing thread */
4580 while ((kq->kqf_state & (KQ_PROCESSING | KQ_DRAIN)) == KQ_PROCESSING) {
4581 kq->kqf_state |= KQ_PROCWAIT;
4582 lck_spin_sleep(lck: &kq->kqf_lock, lck_sleep_action: LCK_SLEEP_DEFAULT,
4583 event: &kq->kqf_suppressed, THREAD_UNINT | THREAD_WAIT_NOREPORT);
4584 }
4585
4586 if (kq->kqf_state & KQ_DRAIN) {
4587 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN) | DBG_FUNC_END,
4588 VM_KERNEL_UNSLIDE_OR_PERM(kq), 2);
4589 return EBADF;
4590 }
4591
4592 /* Nobody else processing */
4593
4594 /* anything left to process? */
4595 if (kq->kqf_count == 0) {
4596 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN) | DBG_FUNC_END,
4597 VM_KERNEL_UNSLIDE_OR_PERM(kq), 1);
4598 return -1;
4599 }
4600
4601 /* convert to processing mode */
4602 kq->kqf_state |= KQ_PROCESSING;
4603
4604 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN) | DBG_FUNC_END,
4605 VM_KERNEL_UNSLIDE_OR_PERM(kq), 0);
4606 return 0;
4607}
4608
4609/*
4610 * Try to end the processing, only called when a workq thread is attempting to
4611 * park (KEVENT_FLAG_PARKING is set).
4612 *
4613 * When returning -1, the kqworkq is setup again so that it is ready to be
4614 * processed.
4615 */
4616static int
4617kqworkq_end_processing(struct kqworkq *kqwq, workq_threadreq_t kqr,
4618 int kevent_flags)
4619{
4620 if (kevent_flags & KEVENT_FLAG_PARKING) {
4621 /*
4622 * if acknowledge events "succeeds" it means there are events,
4623 * which is a failure condition for end_processing.
4624 */
4625 int rc = kqworkq_acknowledge_events(kqwq, kqr, kevent_flags,
4626 KQWQAE_END_PROCESSING);
4627 if (rc == 0) {
4628 return -1;
4629 }
4630 }
4631
4632 return 0;
4633}
4634
4635/*
4636 * Try to end the processing, only called when a workq thread is attempting to
4637 * park (KEVENT_FLAG_PARKING is set).
4638 *
4639 * When returning -1, the kqworkq is setup again so that it is ready to be
4640 * processed (as if kqworkloop_begin_processing had just been called).
4641 *
4642 * If successful and KEVENT_FLAG_PARKING was set in the kevent_flags,
4643 * the kqworkloop is unbound from its servicer as a side effect.
4644 */
4645static int
4646kqworkloop_end_processing(struct kqworkloop *kqwl, int flags, int kevent_flags)
4647{
4648 struct kqueue *kq = &kqwl->kqwl_kqueue;
4649 workq_threadreq_t kqr = &kqwl->kqwl_request;
4650 int rc = 0;
4651
4652 kqlock_held(kqu: kq);
4653
4654 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_END) | DBG_FUNC_START,
4655 kqwl->kqwl_dynamicid, 0, 0);
4656
4657 if (kevent_flags & KEVENT_FLAG_PARKING) {
4658 thread_t thread = kqr_thread_fast(kqr);
4659 thread_qos_t qos_override;
4660
4661 /*
4662 * When KEVENT_FLAG_PARKING is set, we need to attempt
4663 * an unbind while still under the lock.
4664 *
4665 * So we do everything kqworkloop_unbind() would do, but because
4666 * we're inside kqueue_process(), if the workloop actually
4667 * received events while our locks were dropped, we have
4668 * the opportunity to fail the end processing and loop again.
4669 *
4670 * This avoids going through the process-wide workqueue lock
4671 * hence scales better.
4672 */
4673 assert(flags & KQ_PROCESSING);
4674 qos_override = kqworkloop_acknowledge_events(kqwl);
4675 kqworkloop_update_threads_qos(kqwl, op: KQWL_UTQ_PARKING, qos: qos_override);
4676
4677 if (kqwl->kqwl_wakeup_qos && !kqwl->kqwl_owner) {
4678 rc = -1;
4679 } else {
4680 kqworkloop_unbind_locked(kwql: kqwl, thread, how: KQWL_OVERRIDE_DROP_DELAYED);
4681 kqworkloop_release_live(kqwl);
4682 kq->kq_state &= ~flags;
4683 kqworkloop_unbind_delayed_override_drop(thread);
4684 }
4685 } else {
4686 kq->kq_state &= ~flags;
4687 kq->kq_state |= KQ_R2K_ARMED;
4688 kqworkloop_update_threads_qos(kqwl, op: KQWL_UTQ_RECOMPUTE_WAKEUP_QOS, qos: 0);
4689 }
4690
4691 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_END) | DBG_FUNC_END,
4692 kqwl->kqwl_dynamicid, 0, 0);
4693
4694 return rc;
4695}
4696
4697/*
4698 * Called with kqueue lock held.
4699 *
4700 * 0: no more events
4701 * -1: has more events
4702 * EBADF: kqueue is in draining mode
4703 */
4704static int
4705kqfile_end_processing(struct kqfile *kq)
4706{
4707 struct knote *kn;
4708 int procwait;
4709
4710 kqlock_held(kqu: kq);
4711
4712 assert((kq->kqf_state & (KQ_WORKQ | KQ_WORKLOOP)) == 0);
4713
4714 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_END),
4715 VM_KERNEL_UNSLIDE_OR_PERM(kq), 0);
4716
4717 /*
4718 * Return suppressed knotes to their original state.
4719 */
4720 while ((kn = TAILQ_FIRST(&kq->kqf_suppressed)) != NULL) {
4721 knote_unsuppress(kqu: kq, kn);
4722 }
4723
4724 procwait = (kq->kqf_state & KQ_PROCWAIT);
4725 kq->kqf_state &= ~(KQ_PROCESSING | KQ_PROCWAIT);
4726
4727 if (procwait) {
4728 /* first wake up any thread already waiting to process */
4729 thread_wakeup(&kq->kqf_suppressed);
4730 }
4731
4732 if (kq->kqf_state & KQ_DRAIN) {
4733 return EBADF;
4734 }
4735 return kq->kqf_count != 0 ? -1 : 0;
4736}
4737
4738static int
4739kqueue_workloop_ctl_internal(proc_t p, uintptr_t cmd, uint64_t __unused options,
4740 struct kqueue_workloop_params *params, int *retval)
4741{
4742 int error = 0;
4743 struct kqworkloop *kqwl;
4744 struct filedesc *fdp = &p->p_fd;
4745 workq_threadreq_param_t trp = { };
4746#if CONFIG_PREADOPT_TG
4747 struct thread_group *trp_permanent_preadopt_tg = NULL;
4748 integer_t trp_preadopt_priority = 0;
4749 integer_t trp_preadopt_policy = 0;
4750#endif /* CONFIG_PREADOPT_TG */
4751
4752 switch (cmd) {
4753 case KQ_WORKLOOP_CREATE:
4754 if (!params->kqwlp_flags) {
4755 error = EINVAL;
4756 break;
4757 }
4758
4759 if ((params->kqwlp_flags & KQ_WORKLOOP_CREATE_SCHED_PRI) &&
4760 (params->kqwlp_sched_pri < 1 ||
4761 params->kqwlp_sched_pri > 63 /* MAXPRI_USER */)) {
4762 error = EINVAL;
4763 break;
4764 }
4765
4766 if ((params->kqwlp_flags & KQ_WORKLOOP_CREATE_SCHED_POL) &&
4767 invalid_policy(params->kqwlp_sched_pol)) {
4768 error = EINVAL;
4769 break;
4770 }
4771
4772 if ((params->kqwlp_flags & KQ_WORKLOOP_CREATE_CPU_PERCENT) &&
4773 (params->kqwlp_cpu_percent <= 0 ||
4774 params->kqwlp_cpu_percent > 100 ||
4775 params->kqwlp_cpu_refillms <= 0 ||
4776 params->kqwlp_cpu_refillms > 0x00ffffff)) {
4777 error = EINVAL;
4778 break;
4779 }
4780
4781 if (params->kqwlp_flags & KQ_WORKLOOP_CREATE_WORK_INTERVAL) {
4782#if CONFIG_PREADOPT_TG
4783 kern_return_t kr;
4784 kr = kern_work_interval_get_policy_from_port(port_name: params->kqwl_wi_port,
4785 policy: &trp_preadopt_policy,
4786 priority: &trp_preadopt_priority,
4787 tg: &trp_permanent_preadopt_tg);
4788 if (kr != KERN_SUCCESS) {
4789 error = EINVAL;
4790 break;
4791 }
4792 /* The work interval comes with scheduling policy. */
4793 if (trp_preadopt_policy) {
4794 trp.trp_flags |= TRP_POLICY;
4795 trp.trp_pol = (uint8_t)trp_preadopt_policy;
4796
4797 trp.trp_flags |= TRP_PRIORITY;
4798 trp.trp_pri = (uint8_t)trp_preadopt_priority;
4799 }
4800 /*
4801 * We take +1 ref on a thread group backing this work interval
4802 * via kern_work_interval_get_policy_from_port and pass it on to kqwl.
4803 * If, for whatever reasons, kqworkloop_get_or_create fails, we
4804 * get back this ref.
4805 */
4806#else
4807 error = ENOTSUP;
4808 break;
4809#endif /* CONFIG_PREADOPT_TG */
4810 }
4811
4812 if (!(trp.trp_flags & (TRP_POLICY | TRP_PRIORITY))) {
4813 /*
4814 * We always prefer scheduling policy + priority that comes with
4815 * a work interval. It it does not exist, we fallback to what the user
4816 * has asked.
4817 */
4818 if (params->kqwlp_flags & KQ_WORKLOOP_CREATE_SCHED_PRI) {
4819 trp.trp_flags |= TRP_PRIORITY;
4820 trp.trp_pri = (uint8_t)params->kqwlp_sched_pri;
4821 }
4822 if (params->kqwlp_flags & KQ_WORKLOOP_CREATE_SCHED_POL) {
4823 trp.trp_flags |= TRP_POLICY;
4824 trp.trp_pol = (uint8_t)params->kqwlp_sched_pol;
4825 }
4826 if (params->kqwlp_flags & KQ_WORKLOOP_CREATE_CPU_PERCENT) {
4827 trp.trp_flags |= TRP_CPUPERCENT;
4828 trp.trp_cpupercent = (uint8_t)params->kqwlp_cpu_percent;
4829 trp.trp_refillms = params->kqwlp_cpu_refillms;
4830 }
4831 }
4832
4833 error = kqworkloop_get_or_create(p, id: params->kqwlp_id, trp: &trp,
4834#if CONFIG_PREADOPT_TG
4835 trp_permanent_preadopt_tg,
4836#endif /* CONFIG_PREADOPT_TG */
4837 KEVENT_FLAG_DYNAMIC_KQUEUE | KEVENT_FLAG_WORKLOOP |
4838 KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST, kqwlp: &kqwl);
4839 if (error) {
4840#if CONFIG_PREADOPT_TG
4841 /* In case of success, kqwl consumes this +1 ref. */
4842 if (trp_permanent_preadopt_tg) {
4843 thread_group_release(tg: trp_permanent_preadopt_tg);
4844 }
4845#endif
4846 break;
4847 }
4848
4849 if (!fdt_flag_test(fdp, FD_WORKLOOP)) {
4850 /* FD_WORKLOOP indicates we've ever created a workloop
4851 * via this syscall but its only ever added to a process, never
4852 * removed.
4853 */
4854 proc_fdlock(p);
4855 fdt_flag_set(fdp, FD_WORKLOOP);
4856 proc_fdunlock(p);
4857 }
4858 break;
4859 case KQ_WORKLOOP_DESTROY:
4860 error = kqworkloop_get_or_create(p, id: params->kqwlp_id, NULL,
4861#if CONFIG_PREADOPT_TG
4862 NULL,
4863#endif /* CONFIG_PREADOPT_TG */
4864 KEVENT_FLAG_DYNAMIC_KQUEUE | KEVENT_FLAG_WORKLOOP |
4865 KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST, kqwlp: &kqwl);
4866 if (error) {
4867 break;
4868 }
4869 kqlock(kqu: kqwl);
4870 trp.trp_value = kqwl->kqwl_params;
4871 if (trp.trp_flags && !(trp.trp_flags & TRP_RELEASED)) {
4872 trp.trp_flags |= TRP_RELEASED;
4873 kqwl->kqwl_params = trp.trp_value;
4874 kqworkloop_release_live(kqwl);
4875 } else {
4876 error = EINVAL;
4877 }
4878 kqunlock(kqu: kqwl);
4879 kqworkloop_release(kqwl);
4880 break;
4881 }
4882 *retval = 0;
4883 return error;
4884}
4885
4886int
4887kqueue_workloop_ctl(proc_t p, struct kqueue_workloop_ctl_args *uap, int *retval)
4888{
4889 struct kqueue_workloop_params params = {
4890 .kqwlp_id = 0,
4891 };
4892 if (uap->sz < sizeof(params.kqwlp_version)) {
4893 return EINVAL;
4894 }
4895
4896 size_t copyin_sz = MIN(sizeof(params), uap->sz);
4897 int rv = copyin(uap->addr, &params, copyin_sz);
4898 if (rv) {
4899 return rv;
4900 }
4901
4902 if (params.kqwlp_version != (int)uap->sz) {
4903 return EINVAL;
4904 }
4905
4906 return kqueue_workloop_ctl_internal(p, cmd: uap->cmd, options: uap->options, params: &params,
4907 retval);
4908}
4909
4910static int
4911kqueue_select(struct fileproc *fp, int which, void *wql, __unused vfs_context_t ctx)
4912{
4913 struct kqfile *kq = (struct kqfile *)fp_get_data(fp);
4914 int retnum = 0;
4915
4916 assert((kq->kqf_state & (KQ_WORKLOOP | KQ_WORKQ)) == 0);
4917
4918 if (which == FREAD) {
4919 kqlock(kqu: kq);
4920 if (kqfile_begin_processing(kq) == 0) {
4921 retnum = kq->kqf_count;
4922 kqfile_end_processing(kq);
4923 } else if ((kq->kqf_state & KQ_DRAIN) == 0) {
4924 selrecord(selector: kq->kqf_p, &kq->kqf_sel, wql);
4925 }
4926 kqunlock(kqu: kq);
4927 }
4928 return retnum;
4929}
4930
4931/*
4932 * kqueue_close -
4933 */
4934static int
4935kqueue_close(struct fileglob *fg, __unused vfs_context_t ctx)
4936{
4937 struct kqfile *kqf = fg_get_data(fg);
4938
4939 assert((kqf->kqf_state & (KQ_WORKLOOP | KQ_WORKQ)) == 0);
4940 kqlock(kqu: kqf);
4941 selthreadclear(&kqf->kqf_sel);
4942 kqunlock(kqu: kqf);
4943 kqueue_dealloc(kq: &kqf->kqf_kqueue);
4944 fg_set_data(fg, NULL);
4945 return 0;
4946}
4947
4948/*
4949 * Max depth of the nested kq path that can be created.
4950 * Note that this has to be less than the size of kq_level
4951 * to avoid wrapping around and mislabeling the level. We also
4952 * want to be aggressive about this so that we don't overflow the
4953 * kernel stack while posting kevents
4954 */
4955#define MAX_NESTED_KQ 10
4956
4957/*
4958 * The callers has taken a use-count reference on this kqueue and will donate it
4959 * to the kqueue we are being added to. This keeps the kqueue from closing until
4960 * that relationship is torn down.
4961 */
4962static int
4963kqueue_kqfilter(struct fil