1 | /* |
2 | * Copyright (c) 2000-2017 Apple Computer, Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #ifndef _PTHREAD_PRIORITY_PRIVATE_H_ |
30 | #define _PTHREAD_PRIORITY_PRIVATE_H_ |
31 | |
32 | #if KERNEL |
33 | #define PTHREAD_EXPOSE_LAYOUT 1 |
34 | #else |
35 | #include <TargetConditionals.h> |
36 | #if TARGET_OS_SIMULATOR |
37 | #define PTHREAD_EXPOSE_LAYOUT 0 |
38 | #else |
39 | #define PTHREAD_EXPOSE_LAYOUT 1 |
40 | #endif /* TARGET_OS_SIMULATOR */ |
41 | #endif |
42 | |
43 | /*! |
44 | * @typedef pthread_priority_t |
45 | * |
46 | * @abstract |
47 | * pthread_priority_t is an on opaque integer that is guaranteed to be ordered |
48 | * such that combations of QoS classes and relative priorities are ordered |
49 | * numerically, according to their combined priority. |
50 | * |
51 | * <b>xnu, pthread & libdispatch flags</b> |
52 | * |
53 | * @const _PTHREAD_PRIORITY_OVERCOMMIT_FLAG |
54 | * The thread this priority is applied to is overcommit (affects the workqueue |
55 | * creation policy for this priority). |
56 | * |
57 | * @const _PTHREAD_PRIORITY_COOPERATIVE_FLAG |
58 | * Used to convey that a thread is part of the cooperative pool. This is used |
59 | * both outgoing form kernel and incoming into kernel |
60 | * |
61 | * @const _PTHREAD_PRIORITY_THREAD_TYPE_MASK |
62 | * The set of bits that encode information about the thread type - whether it is |
63 | * overcommit, non-overcommit or cooperative |
64 | * |
65 | * @const _PTHREAD_PRIORITY_FALLBACK_FLAG |
66 | * Indicates that this priority is is used only when incoming events have no |
67 | * priority at all. It is merely used as a fallback (hence the name) instead of |
68 | * a floor. |
69 | * |
70 | * This is usually used with QOS_CLASS_DEFAULT and a 0 relative priority. |
71 | * |
72 | * @const _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG |
73 | * The event manager flag indicates that this thread/request is for a event |
74 | * manager thread. There can only ever be one event manager thread at a time |
75 | * and it is brought up at the highest of all event manager priorities pthread |
76 | * knows about. |
77 | * |
78 | * @const _PTHREAD_PRIORITY_OVERRIDE_QOS_FLAG |
79 | * This flag indicates that the bits extracted using |
80 | * _PTHREAD_PRIORITY_QOS_CLASS_MASK represent { QoS override, req QoS } instead |
81 | * of just req QoS. This is only currently only used as input to the kernel as |
82 | * part of pthread_set_properties_self(). The override field here represents the |
83 | * dispatch workqueue override. |
84 | * |
85 | * @const _PTHREAD_PRIORITY_SCHED_PRI_FLAG |
86 | * @const _PTHREAD_PRIORITY_SCHED_PRI_MASK |
87 | * This flag indicates that the bits extracted using |
88 | * _PTHREAD_PRIORITY_SCHED_PRI_MASK represent a scheduler priority instead of |
89 | * a {qos, relative priority} pair. |
90 | * |
91 | * This flag is used by the pthread kext to indicate to libdispatch that the |
92 | * event manager queue priority is a scheduling priority and not a QoS. When |
93 | * the manager thread's priority is updated due to creation of pthread root |
94 | * queues, libdispatch passed a pthread_priority_t in to kernel with this flag |
95 | * to specify the new sched pri of manager. This flag is never used as an input |
96 | * by anything else and is why it can perform a double duty with |
97 | * _PTHREAD_PRIORITY_ROOTQUEUE_FLAG. |
98 | * |
99 | * @const _PTHREAD_PRIORITY_NEEDS_UNBIND_FLAG |
100 | * This flag is used for the priority of event delivery threads to indicate |
101 | * to libdispatch that this thread is bound to a kqueue. |
102 | * |
103 | * <b>dispatch only flags</b> |
104 | * |
105 | * @const _PTHREAD_PRIORITY_INHERIT_FLAG |
106 | * This flag is meaningful to libdispatch only and has no meaning for the |
107 | * kernel and/or pthread. |
108 | * |
109 | * @const _PTHREAD_PRIORITY_ROOTQUEUE_FLAG |
110 | * This flag is meaningful to libdispatch only and has no meaning for the |
111 | * kernel and/or pthread. |
112 | * |
113 | * @const _PTHREAD_PRIORITY_ENFORCE_FLAG |
114 | * This flag is used to indicate that this priority should be prefered for work |
115 | * submited asynchronously over the intrinsic priority of the queue/thread the |
116 | * work is submitted to. |
117 | * |
118 | * |
119 | * pthread_priority_t encoding - outgoing from kernel: |
120 | * |
121 | * Regular: |
122 | * flags req QoS class Rel pri |
123 | * |---------------------------------|--------------------|--------------------| |
124 | * 22 - 31 8-21 0-7 |
125 | * |
126 | * With _PTHREAD_PRIORITY_SCHED_PRI_FLAG: |
127 | * |
128 | * flags unused sched priority |
129 | * |---------------------------------|----------|------------------------------| |
130 | * 22 - 31 16-21 0-15 |
131 | * |
132 | * pthread_priority_t encoding - incoming to kernel via various syscalls: |
133 | * |
134 | * Regular: |
135 | * |
136 | * flags req QoS class Rel pri |
137 | * |---------------------------------|--------------------|--------------------| |
138 | * 22 - 31 8-21 0-7 |
139 | * |
140 | * With _PTHREAD_PRIORITY_OVERRIDE_QOS_FLAG: |
141 | * |
142 | * flags QoS ovr QoS class Rel pri |
143 | * |---------------------------------|---------|----------|--------------------| |
144 | * 22 - 31 14-21 8-13 0-7 |
145 | * |
146 | * With _PTHREAD_PRIORITY_SCHED_PRI_FLAG: |
147 | * |
148 | * flags unused sched priority |
149 | * |---------------------------------|----------|------------------------------| |
150 | * 22 - 31 16-21 0-15 |
151 | */ |
152 | typedef unsigned long pthread_priority_t; |
153 | |
154 | #define _PTHREAD_PRIORITY_OVERCOMMIT_FLAG 0x80000000u |
155 | #define _PTHREAD_PRIORITY_INHERIT_FLAG 0x40000000u /* dispatch only */ |
156 | #define _PTHREAD_PRIORITY_ROOTQUEUE_FLAG 0x20000000u /* dispatch only */ |
157 | #define _PTHREAD_PRIORITY_SCHED_PRI_FLAG 0x20000000u |
158 | #define _PTHREAD_PRIORITY_ENFORCE_FLAG 0x10000000u /* dispatch only */ |
159 | #define _PTHREAD_PRIORITY_FALLBACK_FLAG 0x04000000u |
160 | #define _PTHREAD_PRIORITY_COOPERATIVE_FLAG 0x08000000u |
161 | #define _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG 0x02000000u |
162 | #define _PTHREAD_PRIORITY_NEEDS_UNBIND_FLAG 0x01000000u |
163 | #define _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG _PTHREAD_PRIORITY_FALLBACK_FLAG // compat |
164 | #define _PTHREAD_PRIORITY_OVERRIDE_QOS_FLAG 0x00800000u |
165 | |
166 | #define _PTHREAD_PRIORITY_THREAD_TYPE_MASK (_PTHREAD_PRIORITY_COOPERATIVE_FLAG | _PTHREAD_PRIORITY_OVERCOMMIT_FLAG) |
167 | |
168 | #if PTHREAD_EXPOSE_LAYOUT || defined(__PTHREAD_EXPOSE_INTERNALS__) |
169 | // Masks for encoding of pthread priority |
170 | #define _PTHREAD_PRIORITY_FLAGS_MASK 0xff000000u |
171 | |
172 | #define _PTHREAD_PRIORITY_SCHED_PRI_MASK 0x0000ffffu |
173 | |
174 | #define _PTHREAD_PRIORITY_QOS_CLASS_MASK 0x003fff00u |
175 | #define _PTHREAD_PRIORITY_QOS_CLASS_SHIFT (8ull) |
176 | #define _PTHREAD_PRIORITY_VALID_QOS_CLASS_MASK 0x00003f00u |
177 | #define _PTHREAD_PRIORITY_VALID_OVERRIDE_QOS_MASK 0x003fc000u |
178 | #define _PTHREAD_PRIORITY_QOS_OVERRIDE_SHIFT (14ull) |
179 | |
180 | #define _PTHREAD_PRIORITY_PRIORITY_MASK 0x000000ffu |
181 | #define _PTHREAD_PRIORITY_PRIORITY_SHIFT (0) |
182 | #endif /* PTHREAD_EXPOSE_LAYOUT */ |
183 | |
184 | #if PRIVATE |
185 | #if XNU_KERNEL_PRIVATE && !defined(__PTHREAD_EXPOSE_INTERNALS__) |
186 | #define __PTHREAD_EXPOSE_INTERNALS__ 1 |
187 | #endif // XNU_KERNEL_PRIVATE |
188 | #ifdef __PTHREAD_EXPOSE_INTERNALS__ |
189 | /* |
190 | * This exposes the encoding used for pthread_priority_t |
191 | * and is meant to be used by pthread and XNU only |
192 | */ |
193 | #include <mach/thread_policy.h> // THREAD_QOS_* |
194 | #include <stdbool.h> |
195 | |
196 | // pthread_priority_t's type is unfortunately 64bits on LP64 |
197 | // so we use this type for people who need to store it in structs |
198 | typedef unsigned int pthread_priority_compact_t; |
199 | |
200 | __attribute__((always_inline, const)) |
201 | static inline bool |
202 | _pthread_priority_has_qos(pthread_priority_t pp) |
203 | { |
204 | return (pp & (_PTHREAD_PRIORITY_SCHED_PRI_FLAG | |
205 | _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG)) == 0 && |
206 | (pp & _PTHREAD_PRIORITY_VALID_QOS_CLASS_MASK) != 0; |
207 | } |
208 | |
209 | __attribute__((always_inline, const)) |
210 | static inline bool |
211 | _pthread_priority_has_sched_pri(pthread_priority_t pp) |
212 | { |
213 | return pp & _PTHREAD_PRIORITY_SCHED_PRI_FLAG; |
214 | } |
215 | |
216 | __attribute__((always_inline, const)) |
217 | static inline bool |
218 | _pthread_priority_has_override_qos(pthread_priority_t pp) |
219 | { |
220 | return (pp & (_PTHREAD_PRIORITY_SCHED_PRI_FLAG | |
221 | _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG)) == 0 && |
222 | (pp & _PTHREAD_PRIORITY_OVERRIDE_QOS_FLAG) != 0 && |
223 | (pp & _PTHREAD_PRIORITY_VALID_OVERRIDE_QOS_MASK) != 0; |
224 | } |
225 | |
226 | __attribute__((always_inline, const)) |
227 | static inline pthread_priority_compact_t |
228 | _pthread_priority_make_from_thread_qos(thread_qos_t qos, int relpri, |
229 | unsigned long flags) |
230 | { |
231 | pthread_priority_compact_t pp = (flags & _PTHREAD_PRIORITY_FLAGS_MASK); |
232 | if (qos && qos < THREAD_QOS_LAST) { |
233 | pp |= (1 << (_PTHREAD_PRIORITY_QOS_CLASS_SHIFT + qos - 1)); |
234 | pp |= ((uint8_t)relpri - 1) & _PTHREAD_PRIORITY_PRIORITY_MASK; |
235 | } |
236 | return pp; |
237 | } |
238 | |
239 | __attribute__((always_inline, const)) |
240 | static inline pthread_priority_compact_t |
241 | _pthread_priority_make_from_sched_pri(int sched_pri, unsigned long flags) |
242 | { |
243 | pthread_priority_compact_t pp = (flags & _PTHREAD_PRIORITY_FLAGS_MASK); |
244 | pp |= _PTHREAD_PRIORITY_SCHED_PRI_FLAG; |
245 | pp |= (pthread_priority_compact_t) sched_pri; |
246 | |
247 | return pp; |
248 | } |
249 | |
250 | __attribute__((always_inline, const)) |
251 | static inline pthread_priority_compact_t |
252 | _pthread_priority_make_from_thread_qos_and_override(thread_qos_t req_qos, |
253 | int relpri, thread_qos_t override_qos, unsigned long flags) |
254 | { |
255 | pthread_priority_compact_t pp; |
256 | pp = _pthread_priority_make_from_thread_qos(qos: req_qos, relpri, flags); |
257 | |
258 | if (override_qos && override_qos < THREAD_QOS_LAST) { |
259 | pp |= (1 << (_PTHREAD_PRIORITY_QOS_OVERRIDE_SHIFT + (override_qos - 1))); |
260 | pp |= _PTHREAD_PRIORITY_OVERRIDE_QOS_FLAG; |
261 | } |
262 | |
263 | return pp; |
264 | } |
265 | |
266 | __attribute__((always_inline, const)) |
267 | static inline pthread_priority_compact_t |
268 | _pthread_event_manager_priority(void) |
269 | { |
270 | return _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG; |
271 | } |
272 | |
273 | __attribute__((always_inline, const)) |
274 | static inline pthread_priority_compact_t |
275 | _pthread_unspecified_priority(void) |
276 | { |
277 | return _pthread_priority_make_from_thread_qos(THREAD_QOS_UNSPECIFIED, relpri: 0, flags: 0); |
278 | } |
279 | |
280 | __attribute__((always_inline, const)) |
281 | static inline pthread_priority_compact_t |
282 | _pthread_default_priority(unsigned long flags) |
283 | { |
284 | return _pthread_priority_make_from_thread_qos(THREAD_QOS_LEGACY, relpri: 0, flags); |
285 | } |
286 | |
287 | __attribute__((always_inline, const)) |
288 | static inline thread_qos_t |
289 | _pthread_priority_thread_qos_fast(pthread_priority_t pp) |
290 | { |
291 | pp &= _PTHREAD_PRIORITY_VALID_QOS_CLASS_MASK; |
292 | pp >>= _PTHREAD_PRIORITY_QOS_CLASS_SHIFT; |
293 | return (thread_qos_t)__builtin_ffs((int)pp); |
294 | } |
295 | |
296 | __attribute__((always_inline, const)) |
297 | static inline thread_qos_t |
298 | _pthread_priority_thread_override_qos_fast(pthread_priority_t pp) |
299 | { |
300 | pp &= _PTHREAD_PRIORITY_VALID_OVERRIDE_QOS_MASK; |
301 | pp >>= _PTHREAD_PRIORITY_QOS_OVERRIDE_SHIFT; |
302 | return (thread_qos_t)__builtin_ffs((int)pp); |
303 | } |
304 | |
305 | __attribute__((always_inline, const)) |
306 | static inline int |
307 | _pthread_priority_sched_pri_fast(pthread_priority_t pp) |
308 | { |
309 | return pp & _PTHREAD_PRIORITY_SCHED_PRI_MASK; |
310 | } |
311 | |
312 | __attribute__((always_inline, const)) |
313 | static inline thread_qos_t |
314 | _pthread_priority_thread_qos(pthread_priority_t pp) |
315 | { |
316 | if (_pthread_priority_has_qos(pp)) { |
317 | return _pthread_priority_thread_qos_fast(pp); |
318 | } |
319 | return THREAD_QOS_UNSPECIFIED; |
320 | } |
321 | |
322 | __attribute__((always_inline, const)) |
323 | static inline thread_qos_t |
324 | _pthread_priority_thread_override_qos(pthread_priority_t pp) |
325 | { |
326 | if (_pthread_priority_has_override_qos(pp)) { |
327 | return _pthread_priority_thread_override_qos_fast(pp); |
328 | } |
329 | return THREAD_QOS_UNSPECIFIED; |
330 | } |
331 | |
332 | __attribute__((always_inline, const)) |
333 | static inline int |
334 | _pthread_priority_sched_pri(pthread_priority_t pp) |
335 | { |
336 | if (_pthread_priority_has_sched_pri(pp)) { |
337 | return _pthread_priority_sched_pri_fast(pp); |
338 | } |
339 | |
340 | return 0; |
341 | } |
342 | |
343 | __attribute__((always_inline, const)) |
344 | static inline int |
345 | _pthread_priority_relpri(pthread_priority_t pp) |
346 | { |
347 | if (_pthread_priority_has_qos(pp)) { |
348 | pp &= _PTHREAD_PRIORITY_PRIORITY_MASK; |
349 | pp >>= _PTHREAD_PRIORITY_PRIORITY_SHIFT; |
350 | return (int8_t)pp + 1; |
351 | } |
352 | return 0; |
353 | } |
354 | |
355 | __attribute__((always_inline, const)) |
356 | static inline bool |
357 | _pthread_priority_is_overcommit(pthread_priority_t pp) |
358 | { |
359 | return pp & _PTHREAD_PRIORITY_OVERCOMMIT_FLAG; |
360 | } |
361 | |
362 | __attribute__((always_inline, const)) |
363 | static inline bool |
364 | _pthread_priority_is_cooperative(pthread_priority_t pp) |
365 | { |
366 | return pp & _PTHREAD_PRIORITY_COOPERATIVE_FLAG; |
367 | } |
368 | |
369 | __attribute__((always_inline, const)) |
370 | static inline bool |
371 | _pthread_priority_is_nonovercommit(pthread_priority_t pp) |
372 | { |
373 | return !_pthread_priority_is_cooperative(pp) && !_pthread_priority_is_overcommit(pp); |
374 | } |
375 | |
376 | #if XNU_KERNEL_PRIVATE |
377 | // Interfaces only used by the kernel and not implemented in userspace. |
378 | |
379 | /* |
380 | * Keep managerness, overcomitness and fallback, discard other flags. |
381 | * Normalize and validate QoS/relpri |
382 | */ |
383 | __attribute__((const)) |
384 | pthread_priority_compact_t |
385 | _pthread_priority_normalize(pthread_priority_t pp); |
386 | |
387 | /* |
388 | * Keep managerness, discard other flags. |
389 | * Normalize and validate QoS/relpri |
390 | */ |
391 | __attribute__((const)) |
392 | pthread_priority_compact_t |
393 | _pthread_priority_normalize_for_ipc(pthread_priority_t pp); |
394 | |
395 | /* |
396 | * Keep the flags from base_pp and return the priority with the maximum priority |
397 | * of base_pp and _pthread_priority_make_from_thread_qos(qos, 0, 0) |
398 | */ |
399 | __attribute__((const)) |
400 | pthread_priority_compact_t |
401 | _pthread_priority_combine(pthread_priority_t base_pp, thread_qos_t qos); |
402 | |
403 | |
404 | #endif // XNU_KERNEL_PRIVATE |
405 | #endif // __PTHREAD_EXPOSE_INTERNALS__ |
406 | #endif // PRIVATE |
407 | #endif // _PTHREAD_PRIORITY_PRIVATE_H_ |
408 | |