1 | /* |
2 | * Copyright (c) 2014 Apple Computer, Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #ifndef _WORKQUEUE_INTERNAL_H_ |
30 | #define _WORKQUEUE_INTERNAL_H_ |
31 | |
32 | // Sometimes something gets passed a bucket number and we need a way to express |
33 | // that it's actually the event manager. Use the (0)th bucket for that. |
34 | #define WORKQ_THREAD_QOS_MIN (THREAD_QOS_MAINTENANCE) |
35 | #define WORKQ_THREAD_QOS_MAX (THREAD_QOS_LAST) |
36 | #define WORKQ_THREAD_QOS_CLEANUP (THREAD_QOS_LEGACY) |
37 | #define WORKQ_THREAD_QOS_ABOVEUI (THREAD_QOS_LAST) |
38 | #define WORKQ_THREAD_QOS_MANAGER (THREAD_QOS_LAST + 1) // outside of MIN/MAX |
39 | |
40 | #define WORKQ_NUM_QOS_BUCKETS (WORKQ_THREAD_QOS_MAX - 1) // MT/BG shared |
41 | #define WORKQ_NUM_BUCKETS (WORKQ_NUM_QOS_BUCKETS + 1) // + mgr |
42 | |
43 | /* These definitions are only available to the kext, to avoid bleeding |
44 | * constants and types across the boundary to the userspace library. |
45 | */ |
46 | #ifdef KERNEL |
47 | #pragma mark wq structs |
48 | |
49 | /* These defines come from kern/thread.h but are XNU_KERNEL_PRIVATE so do not get |
50 | * exported to kernel extensions. |
51 | */ |
52 | #define SCHED_CALL_BLOCK 0x1 |
53 | #define SCHED_CALL_UNBLOCK 0x2 |
54 | |
55 | /* old workq priority scheme */ |
56 | |
57 | #define WORKQUEUE_HIGH_PRIOQUEUE 0 /* high priority queue */ |
58 | #define WORKQUEUE_DEFAULT_PRIOQUEUE 1 /* default priority queue */ |
59 | #define WORKQUEUE_LOW_PRIOQUEUE 2 /* low priority queue */ |
60 | #define WORKQUEUE_BG_PRIOQUEUE 3 /* background priority queue */ |
61 | |
62 | /* wq_max_constrained_threads = max(64, N_CPU * WORKQUEUE_CONSTRAINED_FACTOR) |
63 | * This used to be WORKQ_NUM_BUCKETS + 1 when NUM_BUCKETS was 4, yielding |
64 | * N_CPU * 5. When NUM_BUCKETS changed, we decided that the limit should |
65 | * not change. So the factor is now always 5. |
66 | */ |
67 | #define WORKQUEUE_CONSTRAINED_FACTOR 5 |
68 | |
69 | #if BSD_KERNEL_PRIVATE |
70 | #include <kern/mpsc_queue.h> |
71 | #include <kern/priority_queue.h> |
72 | #include <kern/thread_call.h> |
73 | #include <kern/turnstile.h> |
74 | #include <mach/kern_return.h> |
75 | #include <sys/queue.h> |
76 | #include <sys/kernel_types.h> |
77 | |
78 | /* struct uthread::uu_workq_flags */ |
79 | #define UT_WORKQ_NEW 0x01 /* First return to userspace */ |
80 | #define UT_WORKQ_RUNNING 0x02 /* On thrunlist, not parked. */ |
81 | #define UT_WORKQ_DYING 0x04 /* Thread is being killed */ |
82 | #define UT_WORKQ_OVERCOMMIT 0x08 /* Overcommit thread. */ |
83 | #define UT_WORKQ_OUTSIDE_QOS 0x10 /* Thread should avoid send QoS changes to kernel */ |
84 | #define UT_WORKQ_IDLE_CLEANUP 0x20 /* Thread is removing its voucher or stack */ |
85 | #define UT_WORKQ_EARLY_BOUND 0x40 /* Thread has been bound early */ |
86 | #define UT_WORKQ_CPUPERCENT 0x80 /* Thread has CPU percent policy active */ |
87 | #define UT_WORKQ_COOPERATIVE 0x100 /* Thread is part of cooperative pool */ |
88 | |
89 | typedef union workq_threadreq_param_s { |
90 | struct { |
91 | uint16_t trp_flags; |
92 | uint8_t trp_pri; |
93 | uint8_t trp_pol; |
94 | uint32_t trp_cpupercent: 8, |
95 | trp_refillms: 24; |
96 | }; |
97 | uint64_t trp_value; |
98 | } workq_threadreq_param_t; |
99 | |
100 | #define TRP_PRIORITY 0x1 |
101 | #define TRP_POLICY 0x2 |
102 | #define TRP_CPUPERCENT 0x4 |
103 | #define TRP_RELEASED 0x8000 |
104 | |
105 | /*! |
106 | * @enum workq_tr_state_t |
107 | * |
108 | * @brief |
109 | * This enum represents the state of a workq thread request. |
110 | * |
111 | * @discussion |
112 | * The states are used and set by both kevent and the workq subsystem under very |
113 | * precise locking domains. |
114 | * |
115 | * When for kevent requests, this structure is embedded on the kqueue itself, |
116 | * for non kevent related thread requests, it is allocated. |
117 | * |
118 | * Only the BINDING state isn't set under the kqlock, but then only QUEUED could |
119 | * be read by kqueue in its stead. |
120 | * |
121 | * @const WORKQ_TR_STATE_IDLE |
122 | * This thread request is idle. |
123 | * The state is only transient for non kevent thread requests. |
124 | * Set under the kqlock (kevent) or after allocation (workq). |
125 | * |
126 | * tr_entry/tr_thread are unused. |
127 | * |
128 | * @const WORKQ_TR_STATE_NEW |
129 | * This thread request is being initialized. This state is transient. |
130 | * Set workq lock for all kinds, set under the kqlock to for kevent requests. |
131 | * |
132 | * tr_entry is initialized, tr_thread is unused. |
133 | * |
134 | * @const WORKQ_TR_STATE_QUEUED |
135 | * This thread request has been pended, waiting for a thread to be bound. |
136 | * Set workq lock for all kinds, set under the kqlock to for kevent requests. |
137 | * |
138 | * tr_entry is used as linkage in a workq priority queue, tr_thread is unused. |
139 | * |
140 | * @const WORKQ_TR_STATE_CANCELED |
141 | * When the process exits, Queued thread requests are marked canceled. |
142 | * This happens under the workqueue lock. |
143 | * |
144 | * @const WORKQ_TR_STATE_BINDING (kevent only) |
145 | * A thread was found to bind to the thread request. |
146 | * The bind is preposted this way under the workq lock and will be |
147 | * acknowledged by the kevent subsystem. |
148 | * |
149 | * tr_entry is unused, tr_thread is the thread we're binding to. |
150 | * |
151 | * @const WORKQ_TR_STATE_BOUND (kevent only) |
152 | * A thread bind has been acknowledged by the kevent subsystem. |
153 | * This is always set under the kqlock, sometimes also under the workq lock. |
154 | * |
155 | * tr_entry is unused, tr_thread is the thread we're bound to. |
156 | */ |
157 | __enum_decl(workq_tr_state_t, uint8_t, { |
158 | WORKQ_TR_STATE_IDLE = 0, /* request isn't in flight */ |
159 | WORKQ_TR_STATE_NEW = 1, /* request is being initiated */ |
160 | WORKQ_TR_STATE_QUEUED = 2, /* request is being queued */ |
161 | WORKQ_TR_STATE_CANCELED = 3, /* request is canceled */ |
162 | WORKQ_TR_STATE_BINDING = 4, /* request is preposted for bind */ |
163 | WORKQ_TR_STATE_BOUND = 5, /* request is bound to a thread */ |
164 | }); |
165 | |
166 | __options_decl(workq_tr_flags_t, uint8_t, { |
167 | WORKQ_TR_FLAG_KEVENT = 0x01, |
168 | WORKQ_TR_FLAG_WORKLOOP = 0x02, |
169 | WORKQ_TR_FLAG_OVERCOMMIT = 0x04, |
170 | WORKQ_TR_FLAG_WL_PARAMS = 0x08, |
171 | WORKQ_TR_FLAG_WL_OUTSIDE_QOS = 0x10, |
172 | WORKQ_TR_FLAG_COOPERATIVE = 0x20, |
173 | }); |
174 | |
175 | typedef struct workq_threadreq_s { |
176 | union { |
177 | struct priority_queue_entry_sched tr_entry; |
178 | STAILQ_ENTRY(workq_threadreq_s) tr_link; |
179 | thread_t tr_thread; |
180 | }; |
181 | uint16_t tr_count; |
182 | workq_tr_flags_t tr_flags; |
183 | workq_tr_state_t tr_state; |
184 | thread_qos_t tr_qos; /* qos for the thread request */ |
185 | |
186 | /* kqueue states, modified under the kqlock */ |
187 | kq_index_t tr_kq_override_index; /* highest wakeup override index */ |
188 | kq_index_t tr_kq_qos_index; /* QoS for the servicer */ |
189 | } workq_threadreq_s, *workq_threadreq_t; |
190 | |
191 | STAILQ_HEAD(workq_threadreq_tailq, workq_threadreq_s); |
192 | |
193 | #if defined(__LP64__) |
194 | typedef unsigned __int128 wq_thactive_t; |
195 | #else |
196 | typedef uint64_t wq_thactive_t; |
197 | #endif |
198 | |
199 | __options_decl(workq_state_flags_t, uint32_t, { |
200 | WQ_EXITING = 0x0001, |
201 | WQ_PROC_SUSPENDED = 0x0002, |
202 | WQ_DEATH_CALL_SCHEDULED = 0x0004, |
203 | |
204 | WQ_DELAYED_CALL_SCHEDULED = 0x0010, |
205 | WQ_DELAYED_CALL_PENDED = 0x0020, |
206 | WQ_IMMEDIATE_CALL_SCHEDULED = 0x0040, |
207 | WQ_IMMEDIATE_CALL_PENDED = 0x0080, |
208 | }); |
209 | |
210 | TAILQ_HEAD(workq_uthread_head, uthread); |
211 | |
212 | struct workqueue { |
213 | thread_call_t wq_delayed_call; |
214 | thread_call_t wq_immediate_call; |
215 | thread_call_t wq_death_call; |
216 | |
217 | union { |
218 | struct turnstile *wq_turnstile; |
219 | struct mpsc_queue_chain wq_destroy_link; |
220 | }; |
221 | |
222 | lck_ticket_t wq_lock; |
223 | |
224 | uint64_t wq_thread_call_last_run; |
225 | struct os_refcnt wq_refcnt; |
226 | workq_state_flags_t _Atomic wq_flags; |
227 | uint32_t wq_fulfilled; |
228 | uint32_t wq_creations; |
229 | uint32_t wq_timer_interval; |
230 | uint32_t wq_event_manager_priority; |
231 | uint32_t wq_reqcount; /* number of elements on the wq_*_reqlists */ |
232 | uint16_t wq_thdying_count; |
233 | uint16_t wq_threads_scheduled; |
234 | uint16_t wq_constrained_threads_scheduled; |
235 | uint16_t wq_nthreads; |
236 | uint16_t wq_thidlecount; |
237 | uint16_t wq_thscheduled_count[WORKQ_NUM_BUCKETS]; // incl. manager |
238 | |
239 | workq_threadreq_t wq_event_manager_threadreq; |
240 | |
241 | _Atomic wq_thactive_t wq_thactive; |
242 | _Atomic uint64_t wq_lastblocked_ts[WORKQ_NUM_QOS_BUCKETS]; |
243 | |
244 | struct proc *wq_proc; |
245 | struct uthread *wq_creator; |
246 | turnstile_inheritor_t wq_inheritor; |
247 | thread_t wq_turnstile_updater; // thread doing a turnstile_update_ineritor |
248 | struct workq_uthread_head wq_thrunlist; |
249 | struct workq_uthread_head wq_thnewlist; |
250 | struct workq_uthread_head wq_thidlelist; |
251 | |
252 | struct priority_queue_sched_max wq_overcommit_queue; |
253 | struct priority_queue_sched_max wq_constrained_queue; |
254 | struct priority_queue_sched_max wq_special_queue; |
255 | |
256 | // BG/MT, UT, DEF, IN, UI, AUI. No manager bucket for cooperative pool |
257 | uint8_t wq_cooperative_queue_scheduled_count[WORKQ_NUM_QOS_BUCKETS]; |
258 | uint16_t wq_cooperative_queue_best_req_qos: 3, /* UN means no request, returns BG for BG/MT bucket */ |
259 | wq_cooperative_queue_has_limited_max_size:1, /* if set, max size of cooperative pool per QoS is 1 */ |
260 | unused:12; |
261 | struct workq_threadreq_tailq wq_cooperative_queue[WORKQ_NUM_QOS_BUCKETS]; |
262 | }; |
263 | |
264 | #define WORKQUEUE_MAXTHREADS 512 |
265 | #define WQ_STALLED_WINDOW_USECS 200 |
266 | #define WQ_REDUCE_POOL_WINDOW_USECS 5000000 |
267 | #define WQ_MAX_TIMER_INTERVAL_USECS 50000 |
268 | |
269 | #pragma mark definitions |
270 | |
271 | struct workq_threadreq_s; |
272 | uint32_t _get_pwq_state_kdp(proc_t p); |
273 | |
274 | void workq_exit(struct proc *p); |
275 | void workq_mark_exiting(struct proc *p); |
276 | |
277 | bool workq_is_exiting(struct proc *p); |
278 | |
279 | void workq_thread_set_max_qos(struct proc *p, struct workq_threadreq_s *kqr); |
280 | |
281 | void workq_thread_terminate(struct proc *p, struct uthread *uth); |
282 | |
283 | __options_decl(workq_kern_threadreq_flags_t, uint32_t, { |
284 | WORKQ_THREADREQ_NONE = 0x00, |
285 | WORKQ_THREADREQ_SET_AST_ON_FAILURE = 0x01, |
286 | WORKQ_THREADREQ_ATTEMPT_REBIND = 0x02, |
287 | WORKQ_THREADREQ_CAN_CREATE_THREADS = 0x04, |
288 | WORKQ_THREADREQ_MAKE_OVERCOMMIT = 0x08, |
289 | #if CONFIG_PREADOPT_TG |
290 | WORKQ_THREADREQ_REEVALUATE_PREADOPT_TG = 0x10, |
291 | #endif |
292 | }); |
293 | |
294 | // called with the kq req lock held |
295 | bool workq_kern_threadreq_initiate(struct proc *p, struct workq_threadreq_s *kqr, |
296 | struct turnstile *ts, thread_qos_t qos, workq_kern_threadreq_flags_t flags); |
297 | |
298 | // called with the kq req lock held |
299 | void workq_kern_threadreq_modify(struct proc *p, struct workq_threadreq_s *kqr, |
300 | thread_qos_t qos, workq_kern_threadreq_flags_t flags); |
301 | |
302 | // called with the kq req lock held |
303 | void workq_kern_threadreq_update_inheritor(struct proc *p, struct workq_threadreq_s *kqr, |
304 | thread_t owner, struct turnstile *ts, turnstile_update_flags_t flags); |
305 | |
306 | void workq_kern_threadreq_lock(struct proc *p); |
307 | void workq_kern_threadreq_unlock(struct proc *p); |
308 | |
309 | void workq_kern_threadreq_redrive(struct proc *p, workq_kern_threadreq_flags_t flags); |
310 | |
311 | void workq_kern_quantum_expiry_reevaluate(struct proc *p, thread_t thread); |
312 | bool bsdthread_part_of_cooperative_workqueue(struct uthread *uth); |
313 | |
314 | // This enum matches _pthread_set_flags in libpthread's qos_private.h |
315 | enum workq_set_self_flags { |
316 | WORKQ_SET_SELF_QOS_FLAG = 0x01, |
317 | WORKQ_SET_SELF_VOUCHER_FLAG = 0x02, |
318 | WORKQ_SET_SELF_FIXEDPRIORITY_FLAG = 0x04, |
319 | WORKQ_SET_SELF_TIMESHARE_FLAG = 0x08, |
320 | WORKQ_SET_SELF_WQ_KEVENT_UNBIND = 0x10, |
321 | WORKQ_SET_SELF_QOS_OVERRIDE_FLAG = 0x40, |
322 | }; |
323 | |
324 | void workq_proc_suspended(struct proc *p); |
325 | void workq_proc_resumed(struct proc *p); |
326 | struct workqueue *proc_get_wqptr(struct proc *p); |
327 | |
328 | #endif // BSD_KERNEL_PRIVATE |
329 | |
330 | void workq_init(void); |
331 | |
332 | #endif // KERNEL |
333 | |
334 | #endif // _WORKQUEUE_INTERNAL_H_ |
335 | |