1/*
2 * Copyright (c) 2014 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#ifndef _WORKQUEUE_INTERNAL_H_
30#define _WORKQUEUE_INTERNAL_H_
31
32// Sometimes something gets passed a bucket number and we need a way to express
33// that it's actually the event manager. Use the (0)th bucket for that.
34#define WORKQ_THREAD_QOS_MIN (THREAD_QOS_MAINTENANCE)
35#define WORKQ_THREAD_QOS_MAX (THREAD_QOS_LAST)
36#define WORKQ_THREAD_QOS_CLEANUP (THREAD_QOS_LEGACY)
37#define WORKQ_THREAD_QOS_ABOVEUI (THREAD_QOS_LAST)
38#define WORKQ_THREAD_QOS_MANAGER (THREAD_QOS_LAST + 1) // outside of MIN/MAX
39
40#define WORKQ_NUM_QOS_BUCKETS (WORKQ_THREAD_QOS_MAX - 1) // MT/BG shared
41#define WORKQ_NUM_BUCKETS (WORKQ_NUM_QOS_BUCKETS + 1) // + mgr
42
43/* These definitions are only available to the kext, to avoid bleeding
44 * constants and types across the boundary to the userspace library.
45 */
46#ifdef KERNEL
47#pragma mark wq structs
48
49/* These defines come from kern/thread.h but are XNU_KERNEL_PRIVATE so do not get
50 * exported to kernel extensions.
51 */
52#define SCHED_CALL_BLOCK 0x1
53#define SCHED_CALL_UNBLOCK 0x2
54
55/* old workq priority scheme */
56
57#define WORKQUEUE_HIGH_PRIOQUEUE 0 /* high priority queue */
58#define WORKQUEUE_DEFAULT_PRIOQUEUE 1 /* default priority queue */
59#define WORKQUEUE_LOW_PRIOQUEUE 2 /* low priority queue */
60#define WORKQUEUE_BG_PRIOQUEUE 3 /* background priority queue */
61
62/* wq_max_constrained_threads = max(64, N_CPU * WORKQUEUE_CONSTRAINED_FACTOR)
63 * This used to be WORKQ_NUM_BUCKETS + 1 when NUM_BUCKETS was 4, yielding
64 * N_CPU * 5. When NUM_BUCKETS changed, we decided that the limit should
65 * not change. So the factor is now always 5.
66 */
67#define WORKQUEUE_CONSTRAINED_FACTOR 5
68
69#if BSD_KERNEL_PRIVATE
70#include <kern/mpsc_queue.h>
71#include <kern/priority_queue.h>
72#include <kern/thread_call.h>
73#include <kern/turnstile.h>
74#include <mach/kern_return.h>
75#include <sys/queue.h>
76#include <sys/kernel_types.h>
77
78/* struct uthread::uu_workq_flags */
79#define UT_WORKQ_NEW 0x01 /* First return to userspace */
80#define UT_WORKQ_RUNNING 0x02 /* On thrunlist, not parked. */
81#define UT_WORKQ_DYING 0x04 /* Thread is being killed */
82#define UT_WORKQ_OVERCOMMIT 0x08 /* Overcommit thread. */
83#define UT_WORKQ_OUTSIDE_QOS 0x10 /* Thread should avoid send QoS changes to kernel */
84#define UT_WORKQ_IDLE_CLEANUP 0x20 /* Thread is removing its voucher or stack */
85#define UT_WORKQ_EARLY_BOUND 0x40 /* Thread has been bound early */
86#define UT_WORKQ_CPUPERCENT 0x80 /* Thread has CPU percent policy active */
87#define UT_WORKQ_COOPERATIVE 0x100 /* Thread is part of cooperative pool */
88
89typedef union workq_threadreq_param_s {
90 struct {
91 uint16_t trp_flags;
92 uint8_t trp_pri;
93 uint8_t trp_pol;
94 uint32_t trp_cpupercent: 8,
95 trp_refillms: 24;
96 };
97 uint64_t trp_value;
98} workq_threadreq_param_t;
99
100#define TRP_PRIORITY 0x1
101#define TRP_POLICY 0x2
102#define TRP_CPUPERCENT 0x4
103#define TRP_RELEASED 0x8000
104
105/*!
106 * @enum workq_tr_state_t
107 *
108 * @brief
109 * This enum represents the state of a workq thread request.
110 *
111 * @discussion
112 * The states are used and set by both kevent and the workq subsystem under very
113 * precise locking domains.
114 *
115 * When for kevent requests, this structure is embedded on the kqueue itself,
116 * for non kevent related thread requests, it is allocated.
117 *
118 * Only the BINDING state isn't set under the kqlock, but then only QUEUED could
119 * be read by kqueue in its stead.
120 *
121 * @const WORKQ_TR_STATE_IDLE
122 * This thread request is idle.
123 * The state is only transient for non kevent thread requests.
124 * Set under the kqlock (kevent) or after allocation (workq).
125 *
126 * tr_entry/tr_thread are unused.
127 *
128 * @const WORKQ_TR_STATE_NEW
129 * This thread request is being initialized. This state is transient.
130 * Set workq lock for all kinds, set under the kqlock to for kevent requests.
131 *
132 * tr_entry is initialized, tr_thread is unused.
133 *
134 * @const WORKQ_TR_STATE_QUEUED
135 * This thread request has been pended, waiting for a thread to be bound.
136 * Set workq lock for all kinds, set under the kqlock to for kevent requests.
137 *
138 * tr_entry is used as linkage in a workq priority queue, tr_thread is unused.
139 *
140 * @const WORKQ_TR_STATE_CANCELED
141 * When the process exits, Queued thread requests are marked canceled.
142 * This happens under the workqueue lock.
143 *
144 * @const WORKQ_TR_STATE_BINDING (kevent only)
145 * A thread was found to bind to the thread request.
146 * The bind is preposted this way under the workq lock and will be
147 * acknowledged by the kevent subsystem.
148 *
149 * tr_entry is unused, tr_thread is the thread we're binding to.
150 *
151 * @const WORKQ_TR_STATE_BOUND (kevent only)
152 * A thread bind has been acknowledged by the kevent subsystem.
153 * This is always set under the kqlock, sometimes also under the workq lock.
154 *
155 * tr_entry is unused, tr_thread is the thread we're bound to.
156 */
157__enum_decl(workq_tr_state_t, uint8_t, {
158 WORKQ_TR_STATE_IDLE = 0, /* request isn't in flight */
159 WORKQ_TR_STATE_NEW = 1, /* request is being initiated */
160 WORKQ_TR_STATE_QUEUED = 2, /* request is being queued */
161 WORKQ_TR_STATE_CANCELED = 3, /* request is canceled */
162 WORKQ_TR_STATE_BINDING = 4, /* request is preposted for bind */
163 WORKQ_TR_STATE_BOUND = 5, /* request is bound to a thread */
164});
165
166__options_decl(workq_tr_flags_t, uint8_t, {
167 WORKQ_TR_FLAG_KEVENT = 0x01,
168 WORKQ_TR_FLAG_WORKLOOP = 0x02,
169 WORKQ_TR_FLAG_OVERCOMMIT = 0x04,
170 WORKQ_TR_FLAG_WL_PARAMS = 0x08,
171 WORKQ_TR_FLAG_WL_OUTSIDE_QOS = 0x10,
172 WORKQ_TR_FLAG_COOPERATIVE = 0x20,
173});
174
175typedef struct workq_threadreq_s {
176 union {
177 struct priority_queue_entry_sched tr_entry;
178 STAILQ_ENTRY(workq_threadreq_s) tr_link;
179 thread_t tr_thread;
180 };
181 uint16_t tr_count;
182 workq_tr_flags_t tr_flags;
183 workq_tr_state_t tr_state;
184 thread_qos_t tr_qos; /* qos for the thread request */
185
186 /* kqueue states, modified under the kqlock */
187 kq_index_t tr_kq_override_index; /* highest wakeup override index */
188 kq_index_t tr_kq_qos_index; /* QoS for the servicer */
189} workq_threadreq_s, *workq_threadreq_t;
190
191STAILQ_HEAD(workq_threadreq_tailq, workq_threadreq_s);
192
193#if defined(__LP64__)
194typedef unsigned __int128 wq_thactive_t;
195#else
196typedef uint64_t wq_thactive_t;
197#endif
198
199__options_decl(workq_state_flags_t, uint32_t, {
200 WQ_EXITING = 0x0001,
201 WQ_PROC_SUSPENDED = 0x0002,
202 WQ_DEATH_CALL_SCHEDULED = 0x0004,
203
204 WQ_DELAYED_CALL_SCHEDULED = 0x0010,
205 WQ_DELAYED_CALL_PENDED = 0x0020,
206 WQ_IMMEDIATE_CALL_SCHEDULED = 0x0040,
207 WQ_IMMEDIATE_CALL_PENDED = 0x0080,
208});
209
210TAILQ_HEAD(workq_uthread_head, uthread);
211
212struct workqueue {
213 thread_call_t wq_delayed_call;
214 thread_call_t wq_immediate_call;
215 thread_call_t wq_death_call;
216
217 union {
218 struct turnstile *wq_turnstile;
219 struct mpsc_queue_chain wq_destroy_link;
220 };
221
222 lck_ticket_t wq_lock;
223
224 uint64_t wq_thread_call_last_run;
225 struct os_refcnt wq_refcnt;
226 workq_state_flags_t _Atomic wq_flags;
227 uint32_t wq_fulfilled;
228 uint32_t wq_creations;
229 uint32_t wq_timer_interval;
230 uint32_t wq_event_manager_priority;
231 uint32_t wq_reqcount; /* number of elements on the wq_*_reqlists */
232 uint16_t wq_thdying_count;
233 uint16_t wq_threads_scheduled;
234 uint16_t wq_constrained_threads_scheduled;
235 uint16_t wq_nthreads;
236 uint16_t wq_thidlecount;
237 uint16_t wq_thscheduled_count[WORKQ_NUM_BUCKETS]; // incl. manager
238
239 workq_threadreq_t wq_event_manager_threadreq;
240
241 _Atomic wq_thactive_t wq_thactive;
242 _Atomic uint64_t wq_lastblocked_ts[WORKQ_NUM_QOS_BUCKETS];
243
244 struct proc *wq_proc;
245 struct uthread *wq_creator;
246 turnstile_inheritor_t wq_inheritor;
247 thread_t wq_turnstile_updater; // thread doing a turnstile_update_ineritor
248 struct workq_uthread_head wq_thrunlist;
249 struct workq_uthread_head wq_thnewlist;
250 struct workq_uthread_head wq_thidlelist;
251
252 struct priority_queue_sched_max wq_overcommit_queue;
253 struct priority_queue_sched_max wq_constrained_queue;
254 struct priority_queue_sched_max wq_special_queue;
255
256 // BG/MT, UT, DEF, IN, UI, AUI. No manager bucket for cooperative pool
257 uint8_t wq_cooperative_queue_scheduled_count[WORKQ_NUM_QOS_BUCKETS];
258 uint16_t wq_cooperative_queue_best_req_qos: 3, /* UN means no request, returns BG for BG/MT bucket */
259 wq_cooperative_queue_has_limited_max_size:1, /* if set, max size of cooperative pool per QoS is 1 */
260 unused:12;
261 struct workq_threadreq_tailq wq_cooperative_queue[WORKQ_NUM_QOS_BUCKETS];
262};
263
264#define WORKQUEUE_MAXTHREADS 512
265#define WQ_STALLED_WINDOW_USECS 200
266#define WQ_REDUCE_POOL_WINDOW_USECS 5000000
267#define WQ_MAX_TIMER_INTERVAL_USECS 50000
268
269#pragma mark definitions
270
271struct workq_threadreq_s;
272uint32_t _get_pwq_state_kdp(proc_t p);
273
274void workq_exit(struct proc *p);
275void workq_mark_exiting(struct proc *p);
276
277bool workq_is_exiting(struct proc *p);
278
279void workq_thread_set_max_qos(struct proc *p, struct workq_threadreq_s *kqr);
280
281void workq_thread_terminate(struct proc *p, struct uthread *uth);
282
283__options_decl(workq_kern_threadreq_flags_t, uint32_t, {
284 WORKQ_THREADREQ_NONE = 0x00,
285 WORKQ_THREADREQ_SET_AST_ON_FAILURE = 0x01,
286 WORKQ_THREADREQ_ATTEMPT_REBIND = 0x02,
287 WORKQ_THREADREQ_CAN_CREATE_THREADS = 0x04,
288 WORKQ_THREADREQ_MAKE_OVERCOMMIT = 0x08,
289#if CONFIG_PREADOPT_TG
290 WORKQ_THREADREQ_REEVALUATE_PREADOPT_TG = 0x10,
291#endif
292});
293
294// called with the kq req lock held
295bool workq_kern_threadreq_initiate(struct proc *p, struct workq_threadreq_s *kqr,
296 struct turnstile *ts, thread_qos_t qos, workq_kern_threadreq_flags_t flags);
297
298// called with the kq req lock held
299void workq_kern_threadreq_modify(struct proc *p, struct workq_threadreq_s *kqr,
300 thread_qos_t qos, workq_kern_threadreq_flags_t flags);
301
302// called with the kq req lock held
303void workq_kern_threadreq_update_inheritor(struct proc *p, struct workq_threadreq_s *kqr,
304 thread_t owner, struct turnstile *ts, turnstile_update_flags_t flags);
305
306void workq_kern_threadreq_lock(struct proc *p);
307void workq_kern_threadreq_unlock(struct proc *p);
308
309void workq_kern_threadreq_redrive(struct proc *p, workq_kern_threadreq_flags_t flags);
310
311void workq_kern_quantum_expiry_reevaluate(struct proc *p, thread_t thread);
312bool bsdthread_part_of_cooperative_workqueue(struct uthread *uth);
313
314// This enum matches _pthread_set_flags in libpthread's qos_private.h
315enum workq_set_self_flags {
316 WORKQ_SET_SELF_QOS_FLAG = 0x01,
317 WORKQ_SET_SELF_VOUCHER_FLAG = 0x02,
318 WORKQ_SET_SELF_FIXEDPRIORITY_FLAG = 0x04,
319 WORKQ_SET_SELF_TIMESHARE_FLAG = 0x08,
320 WORKQ_SET_SELF_WQ_KEVENT_UNBIND = 0x10,
321 WORKQ_SET_SELF_QOS_OVERRIDE_FLAG = 0x40,
322};
323
324void workq_proc_suspended(struct proc *p);
325void workq_proc_resumed(struct proc *p);
326struct workqueue *proc_get_wqptr(struct proc *p);
327
328#endif // BSD_KERNEL_PRIVATE
329
330void workq_init(void);
331
332#endif // KERNEL
333
334#endif // _WORKQUEUE_INTERNAL_H_
335