1/*
2 * Copyright (c) 2000-2017 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#ifndef _PTHREAD_PRIORITY_PRIVATE_H_
30#define _PTHREAD_PRIORITY_PRIVATE_H_
31
32#if KERNEL
33#define PTHREAD_EXPOSE_LAYOUT 1
34#else
35#include <TargetConditionals.h>
36#if TARGET_OS_SIMULATOR
37#define PTHREAD_EXPOSE_LAYOUT 0
38#else
39#define PTHREAD_EXPOSE_LAYOUT 1
40#endif /* TARGET_OS_SIMULATOR */
41#endif
42
43/*!
44 * @typedef pthread_priority_t
45 *
46 * @abstract
47 * pthread_priority_t is an on opaque integer that is guaranteed to be ordered
48 * such that combations of QoS classes and relative priorities are ordered
49 * numerically, according to their combined priority.
50 *
51 * <b>xnu, pthread & libdispatch flags</b>
52 *
53 * @const _PTHREAD_PRIORITY_OVERCOMMIT_FLAG
54 * The thread this priority is applied to is overcommit (affects the workqueue
55 * creation policy for this priority).
56 *
57 * @const _PTHREAD_PRIORITY_COOPERATIVE_FLAG
58 * Used to convey that a thread is part of the cooperative pool. This is used
59 * both outgoing form kernel and incoming into kernel
60 *
61 * @const _PTHREAD_PRIORITY_THREAD_TYPE_MASK
62 * The set of bits that encode information about the thread type - whether it is
63 * overcommit, non-overcommit or cooperative
64 *
65 * @const _PTHREAD_PRIORITY_FALLBACK_FLAG
66 * Indicates that this priority is is used only when incoming events have no
67 * priority at all. It is merely used as a fallback (hence the name) instead of
68 * a floor.
69 *
70 * This is usually used with QOS_CLASS_DEFAULT and a 0 relative priority.
71 *
72 * @const _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG
73 * The event manager flag indicates that this thread/request is for a event
74 * manager thread. There can only ever be one event manager thread at a time
75 * and it is brought up at the highest of all event manager priorities pthread
76 * knows about.
77 *
78 * @const _PTHREAD_PRIORITY_OVERRIDE_QOS_FLAG
79 * This flag indicates that the bits extracted using
80 * _PTHREAD_PRIORITY_QOS_CLASS_MASK represent { QoS override, req QoS } instead
81 * of just req QoS. This is only currently only used as input to the kernel as
82 * part of pthread_set_properties_self(). The override field here represents the
83 * dispatch workqueue override.
84 *
85 * @const _PTHREAD_PRIORITY_SCHED_PRI_FLAG
86 * @const _PTHREAD_PRIORITY_SCHED_PRI_MASK
87 * This flag indicates that the bits extracted using
88 * _PTHREAD_PRIORITY_SCHED_PRI_MASK represent a scheduler priority instead of
89 * a {qos, relative priority} pair.
90 *
91 * This flag is used by the pthread kext to indicate to libdispatch that the
92 * event manager queue priority is a scheduling priority and not a QoS. When
93 * the manager thread's priority is updated due to creation of pthread root
94 * queues, libdispatch passed a pthread_priority_t in to kernel with this flag
95 * to specify the new sched pri of manager. This flag is never used as an input
96 * by anything else and is why it can perform a double duty with
97 * _PTHREAD_PRIORITY_ROOTQUEUE_FLAG.
98 *
99 * @const _PTHREAD_PRIORITY_NEEDS_UNBIND_FLAG
100 * This flag is used for the priority of event delivery threads to indicate
101 * to libdispatch that this thread is bound to a kqueue.
102 *
103 * <b>dispatch only flags</b>
104 *
105 * @const _PTHREAD_PRIORITY_INHERIT_FLAG
106 * This flag is meaningful to libdispatch only and has no meaning for the
107 * kernel and/or pthread.
108 *
109 * @const _PTHREAD_PRIORITY_ROOTQUEUE_FLAG
110 * This flag is meaningful to libdispatch only and has no meaning for the
111 * kernel and/or pthread.
112 *
113 * @const _PTHREAD_PRIORITY_ENFORCE_FLAG
114 * This flag is used to indicate that this priority should be prefered for work
115 * submited asynchronously over the intrinsic priority of the queue/thread the
116 * work is submitted to.
117 *
118 *
119 * pthread_priority_t encoding - outgoing from kernel:
120 *
121 * Regular:
122 * flags req QoS class Rel pri
123 * |---------------------------------|--------------------|--------------------|
124 * 22 - 31 8-21 0-7
125 *
126 * With _PTHREAD_PRIORITY_SCHED_PRI_FLAG:
127 *
128 * flags unused sched priority
129 * |---------------------------------|----------|------------------------------|
130 * 22 - 31 16-21 0-15
131 *
132 * pthread_priority_t encoding - incoming to kernel via various syscalls:
133 *
134 * Regular:
135 *
136 * flags req QoS class Rel pri
137 * |---------------------------------|--------------------|--------------------|
138 * 22 - 31 8-21 0-7
139 *
140 * With _PTHREAD_PRIORITY_OVERRIDE_QOS_FLAG:
141 *
142 * flags QoS ovr QoS class Rel pri
143 * |---------------------------------|---------|----------|--------------------|
144 * 22 - 31 14-21 8-13 0-7
145 *
146 * With _PTHREAD_PRIORITY_SCHED_PRI_FLAG:
147 *
148 * flags unused sched priority
149 * |---------------------------------|----------|------------------------------|
150 * 22 - 31 16-21 0-15
151 */
152typedef unsigned long pthread_priority_t;
153
154#define _PTHREAD_PRIORITY_OVERCOMMIT_FLAG 0x80000000u
155#define _PTHREAD_PRIORITY_INHERIT_FLAG 0x40000000u /* dispatch only */
156#define _PTHREAD_PRIORITY_ROOTQUEUE_FLAG 0x20000000u /* dispatch only */
157#define _PTHREAD_PRIORITY_SCHED_PRI_FLAG 0x20000000u
158#define _PTHREAD_PRIORITY_ENFORCE_FLAG 0x10000000u /* dispatch only */
159#define _PTHREAD_PRIORITY_FALLBACK_FLAG 0x04000000u
160#define _PTHREAD_PRIORITY_COOPERATIVE_FLAG 0x08000000u
161#define _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG 0x02000000u
162#define _PTHREAD_PRIORITY_NEEDS_UNBIND_FLAG 0x01000000u
163#define _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG _PTHREAD_PRIORITY_FALLBACK_FLAG // compat
164#define _PTHREAD_PRIORITY_OVERRIDE_QOS_FLAG 0x00800000u
165
166#define _PTHREAD_PRIORITY_THREAD_TYPE_MASK (_PTHREAD_PRIORITY_COOPERATIVE_FLAG | _PTHREAD_PRIORITY_OVERCOMMIT_FLAG)
167
168#if PTHREAD_EXPOSE_LAYOUT || defined(__PTHREAD_EXPOSE_INTERNALS__)
169// Masks for encoding of pthread priority
170#define _PTHREAD_PRIORITY_FLAGS_MASK 0xff000000u
171
172#define _PTHREAD_PRIORITY_SCHED_PRI_MASK 0x0000ffffu
173
174#define _PTHREAD_PRIORITY_QOS_CLASS_MASK 0x003fff00u
175#define _PTHREAD_PRIORITY_QOS_CLASS_SHIFT (8ull)
176#define _PTHREAD_PRIORITY_VALID_QOS_CLASS_MASK 0x00003f00u
177#define _PTHREAD_PRIORITY_VALID_OVERRIDE_QOS_MASK 0x003fc000u
178#define _PTHREAD_PRIORITY_QOS_OVERRIDE_SHIFT (14ull)
179
180#define _PTHREAD_PRIORITY_PRIORITY_MASK 0x000000ffu
181#define _PTHREAD_PRIORITY_PRIORITY_SHIFT (0)
182#endif /* PTHREAD_EXPOSE_LAYOUT */
183
184#if PRIVATE
185#if XNU_KERNEL_PRIVATE && !defined(__PTHREAD_EXPOSE_INTERNALS__)
186#define __PTHREAD_EXPOSE_INTERNALS__ 1
187#endif // XNU_KERNEL_PRIVATE
188#ifdef __PTHREAD_EXPOSE_INTERNALS__
189/*
190 * This exposes the encoding used for pthread_priority_t
191 * and is meant to be used by pthread and XNU only
192 */
193#include <mach/thread_policy.h> // THREAD_QOS_*
194#include <stdbool.h>
195
196// pthread_priority_t's type is unfortunately 64bits on LP64
197// so we use this type for people who need to store it in structs
198typedef unsigned int pthread_priority_compact_t;
199
200__attribute__((always_inline, const))
201static inline bool
202_pthread_priority_has_qos(pthread_priority_t pp)
203{
204 return (pp & (_PTHREAD_PRIORITY_SCHED_PRI_FLAG |
205 _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG)) == 0 &&
206 (pp & _PTHREAD_PRIORITY_VALID_QOS_CLASS_MASK) != 0;
207}
208
209__attribute__((always_inline, const))
210static inline bool
211_pthread_priority_has_sched_pri(pthread_priority_t pp)
212{
213 return pp & _PTHREAD_PRIORITY_SCHED_PRI_FLAG;
214}
215
216__attribute__((always_inline, const))
217static inline bool
218_pthread_priority_has_override_qos(pthread_priority_t pp)
219{
220 return (pp & (_PTHREAD_PRIORITY_SCHED_PRI_FLAG |
221 _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG)) == 0 &&
222 (pp & _PTHREAD_PRIORITY_OVERRIDE_QOS_FLAG) != 0 &&
223 (pp & _PTHREAD_PRIORITY_VALID_OVERRIDE_QOS_MASK) != 0;
224}
225
226__attribute__((always_inline, const))
227static inline pthread_priority_compact_t
228_pthread_priority_make_from_thread_qos(thread_qos_t qos, int relpri,
229 unsigned long flags)
230{
231 pthread_priority_compact_t pp = (flags & _PTHREAD_PRIORITY_FLAGS_MASK);
232 if (qos && qos < THREAD_QOS_LAST) {
233 pp |= (1 << (_PTHREAD_PRIORITY_QOS_CLASS_SHIFT + qos - 1));
234 pp |= ((uint8_t)relpri - 1) & _PTHREAD_PRIORITY_PRIORITY_MASK;
235 }
236 return pp;
237}
238
239__attribute__((always_inline, const))
240static inline pthread_priority_compact_t
241_pthread_priority_make_from_sched_pri(int sched_pri, unsigned long flags)
242{
243 pthread_priority_compact_t pp = (flags & _PTHREAD_PRIORITY_FLAGS_MASK);
244 pp |= _PTHREAD_PRIORITY_SCHED_PRI_FLAG;
245 pp |= (pthread_priority_compact_t) sched_pri;
246
247 return pp;
248}
249
250__attribute__((always_inline, const))
251static inline pthread_priority_compact_t
252_pthread_priority_make_from_thread_qos_and_override(thread_qos_t req_qos,
253 int relpri, thread_qos_t override_qos, unsigned long flags)
254{
255 pthread_priority_compact_t pp;
256 pp = _pthread_priority_make_from_thread_qos(qos: req_qos, relpri, flags);
257
258 if (override_qos && override_qos < THREAD_QOS_LAST) {
259 pp |= (1 << (_PTHREAD_PRIORITY_QOS_OVERRIDE_SHIFT + (override_qos - 1)));
260 pp |= _PTHREAD_PRIORITY_OVERRIDE_QOS_FLAG;
261 }
262
263 return pp;
264}
265
266__attribute__((always_inline, const))
267static inline pthread_priority_compact_t
268_pthread_event_manager_priority(void)
269{
270 return _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG;
271}
272
273__attribute__((always_inline, const))
274static inline pthread_priority_compact_t
275_pthread_unspecified_priority(void)
276{
277 return _pthread_priority_make_from_thread_qos(THREAD_QOS_UNSPECIFIED, relpri: 0, flags: 0);
278}
279
280__attribute__((always_inline, const))
281static inline pthread_priority_compact_t
282_pthread_default_priority(unsigned long flags)
283{
284 return _pthread_priority_make_from_thread_qos(THREAD_QOS_LEGACY, relpri: 0, flags);
285}
286
287__attribute__((always_inline, const))
288static inline thread_qos_t
289_pthread_priority_thread_qos_fast(pthread_priority_t pp)
290{
291 pp &= _PTHREAD_PRIORITY_VALID_QOS_CLASS_MASK;
292 pp >>= _PTHREAD_PRIORITY_QOS_CLASS_SHIFT;
293 return (thread_qos_t)__builtin_ffs((int)pp);
294}
295
296__attribute__((always_inline, const))
297static inline thread_qos_t
298_pthread_priority_thread_override_qos_fast(pthread_priority_t pp)
299{
300 pp &= _PTHREAD_PRIORITY_VALID_OVERRIDE_QOS_MASK;
301 pp >>= _PTHREAD_PRIORITY_QOS_OVERRIDE_SHIFT;
302 return (thread_qos_t)__builtin_ffs((int)pp);
303}
304
305__attribute__((always_inline, const))
306static inline int
307_pthread_priority_sched_pri_fast(pthread_priority_t pp)
308{
309 return pp & _PTHREAD_PRIORITY_SCHED_PRI_MASK;
310}
311
312__attribute__((always_inline, const))
313static inline thread_qos_t
314_pthread_priority_thread_qos(pthread_priority_t pp)
315{
316 if (_pthread_priority_has_qos(pp)) {
317 return _pthread_priority_thread_qos_fast(pp);
318 }
319 return THREAD_QOS_UNSPECIFIED;
320}
321
322__attribute__((always_inline, const))
323static inline thread_qos_t
324_pthread_priority_thread_override_qos(pthread_priority_t pp)
325{
326 if (_pthread_priority_has_override_qos(pp)) {
327 return _pthread_priority_thread_override_qos_fast(pp);
328 }
329 return THREAD_QOS_UNSPECIFIED;
330}
331
332__attribute__((always_inline, const))
333static inline int
334_pthread_priority_sched_pri(pthread_priority_t pp)
335{
336 if (_pthread_priority_has_sched_pri(pp)) {
337 return _pthread_priority_sched_pri_fast(pp);
338 }
339
340 return 0;
341}
342
343__attribute__((always_inline, const))
344static inline int
345_pthread_priority_relpri(pthread_priority_t pp)
346{
347 if (_pthread_priority_has_qos(pp)) {
348 pp &= _PTHREAD_PRIORITY_PRIORITY_MASK;
349 pp >>= _PTHREAD_PRIORITY_PRIORITY_SHIFT;
350 return (int8_t)pp + 1;
351 }
352 return 0;
353}
354
355__attribute__((always_inline, const))
356static inline bool
357_pthread_priority_is_overcommit(pthread_priority_t pp)
358{
359 return pp & _PTHREAD_PRIORITY_OVERCOMMIT_FLAG;
360}
361
362__attribute__((always_inline, const))
363static inline bool
364_pthread_priority_is_cooperative(pthread_priority_t pp)
365{
366 return pp & _PTHREAD_PRIORITY_COOPERATIVE_FLAG;
367}
368
369__attribute__((always_inline, const))
370static inline bool
371_pthread_priority_is_nonovercommit(pthread_priority_t pp)
372{
373 return !_pthread_priority_is_cooperative(pp) && !_pthread_priority_is_overcommit(pp);
374}
375
376#if XNU_KERNEL_PRIVATE
377// Interfaces only used by the kernel and not implemented in userspace.
378
379/*
380 * Keep managerness, overcomitness and fallback, discard other flags.
381 * Normalize and validate QoS/relpri
382 */
383__attribute__((const))
384pthread_priority_compact_t
385_pthread_priority_normalize(pthread_priority_t pp);
386
387/*
388 * Keep managerness, discard other flags.
389 * Normalize and validate QoS/relpri
390 */
391__attribute__((const))
392pthread_priority_compact_t
393_pthread_priority_normalize_for_ipc(pthread_priority_t pp);
394
395/*
396 * Keep the flags from base_pp and return the priority with the maximum priority
397 * of base_pp and _pthread_priority_make_from_thread_qos(qos, 0, 0)
398 */
399__attribute__((const))
400pthread_priority_compact_t
401_pthread_priority_combine(pthread_priority_t base_pp, thread_qos_t qos);
402
403
404#endif // XNU_KERNEL_PRIVATE
405#endif // __PTHREAD_EXPOSE_INTERNALS__
406#endif // PRIVATE
407#endif // _PTHREAD_PRIORITY_PRIVATE_H_
408