1/*
2 * Copyright (c) 2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#ifndef _TURNSTILE_H_
30#define _TURNSTILE_H_
31
32#include <mach/mach_types.h>
33#include <mach/kern_return.h>
34#include <sys/cdefs.h>
35
36__BEGIN_DECLS
37
38#if PRIVATE
39#define TURNSTILE_MAX_HOP_DEFAULT (10)
40struct turnstile_stats {
41 uint64_t ts_priority_propagation;
42 uint64_t ts_no_inheritor;
43 uint64_t ts_thread_runnable;
44 uint64_t ts_no_priority_change_required;
45 uint64_t ts_above_ui_pri_change;
46 uint64_t ts_no_turnstile;
47};
48#endif
49
50#ifdef KERNEL_PRIVATE
51#include <kern/queue.h>
52#include <sys/queue.h>
53#include <kern/waitq.h>
54#include <kern/priority_queue.h>
55#include <os/refcnt.h>
56#include <kern/assert.h>
57#include <kern/kern_types.h>
58#include <kern/mpsc_queue.h>
59#include <kern/locks.h>
60
61/*
62 * turnstile_type_t : Indicates the type of primitive the turnstile is associated with
63 * Please populate turnstile_promote_policy array if a new type is added here.
64 */
65typedef enum __attribute__((packed)) turnstile_type {
66 TURNSTILE_NONE = 0,
67 TURNSTILE_KERNEL_MUTEX = 1,
68 TURNSTILE_ULOCK = 2,
69 TURNSTILE_PTHREAD_MUTEX = 3,
70 TURNSTILE_SYNC_IPC = 4,
71 TURNSTILE_WORKLOOPS = 5,
72 TURNSTILE_WORKQS = 6,
73 TURNSTILE_KNOTE = 7,
74 TURNSTILE_SLEEP_INHERITOR = 8,
75 TURNSTILE_EPOCH_KERNEL = 9,
76 TURNSTILE_EPOCH_USER = 10,
77 TURNSTILE_TOTAL_TYPES = 11,
78} turnstile_type_t;
79
80/*
81 * For each type of turnstile, following are the type of
82 * inheritors passed:
83 *
84 * TURNSTILE_KERNEL_MUTEX
85 * Interlock: kernel mutex interlock.
86 * Inheritor: threads.
87 * Lock order: turnstile lock, thread lock.
88 *
89 * TURNSTILE_ULOCK
90 * Interlock: ulocks interlock.
91 * Inheritor: threads.
92 * Lock order: turnstile lock, thread lock.
93 *
94 * TURNSTILE_PTHREAD_MUTEX
95 * Interlock: pthread mtx interlock.
96 * Inheritor: threads.
97 * Lock order: turnstile lock, thread lock.
98 *
99 * TURNSTILE_SYNC_IPC
100 * Interlock: port's mqueue lock
101 * Inheritor: turnstile (of port in which we are enqueued or WL turnstile.
102 * Lock order: Our turnstile, then turnstile of the port we are enqueued in.
103 * Port circularity will make sure there is never a cycle formation
104 * and lock order is maintained.
105 *
106 * TURNSTILE_WORKLOOPS
107 * Interlock:
108 * - kq req lock
109 * - wq lock when "filt_wlworkq_interlock_needed() is true"
110 * Inheritor: thread, turnstile (of workq)
111 * Lock order: turnstile lock, thread lock
112 * WL turnstile lock, Workq turnstile lock
113 *
114 * TURNSTILE_WORKQS
115 * Interlock: workqueue lock
116 * Inheritor: thread
117 * Lock order: turnstile lock, thread lock.
118 *
119 * TURNSTILE_KNOTE
120 * Interlock: the knote lock
121 * Inheritor: WL turnstile
122 *
123 * TURNSTILE_SLEEP_INHERITOR
124 * Interlock: turnstile_htable bucket spinlock.
125 * Inheritor: threads.
126 * Lock order: turnstile lock, thread lock.
127 *
128 * TURNSTILE_EPOCH_KERNEL
129 * Interlock: the epoch sync interlock.
130 * Inheritor: threads.
131 * Lock order: turnstile lock, thread lock.
132 *
133 * TURNSTILE_EPOCH_USER
134 * Interlock: the epoch sync interlock.
135 * Inheritor: threads.
136 * Lock order: turnstile lock, thread lock.
137 */
138
139typedef enum __attribute__((flag_enum)) turnstile_promote_policy {
140 TURNSTILE_PROMOTE_NONE = 0,
141 TURNSTILE_KERNEL_PROMOTE = 0x1,
142 TURNSTILE_USER_PROMOTE = 0x2,
143 TURNSTILE_USER_IPC_PROMOTE = 0x4,
144} turnstile_promote_policy_t;
145
146typedef enum __attribute__((flag_enum)) turnstile_hash_lock_policy {
147 TURNSTILE_HASH_LOCK_POLICY_NONE = 0,
148 TURNSTILE_IRQ_UNSAFE_HASH = 0x1,
149 TURNSTILE_LOCKED_HASH = 0x2,
150} turnstile_hash_lock_policy_t;
151
152/*
153 * Turnstile state flags
154 *
155 * The turnstile state flags represent the current ownership of a turnstile.
156 * The supported flags are:
157 * - TURNSTILE_STATE_THREAD : Turnstile is attached to a thread
158 * - TURNSTILE_STATE_FREELIST : Turnstile is hanging off the freelist of another turnstile
159 * - TURNSTILE_STATE_HASHTABLE : Turnstile is in the global hash table as the turnstile for a primitive
160 * - TURNSTILE_STATE_PROPRIETOR : Turnstile is attached to a proprietor
161 *
162 * The flag updates are done while holding the primitive interlock.
163 * */
164
165#define TURNSTILE_STATE_THREAD 0x1
166#define TURNSTILE_STATE_FREELIST 0x2
167#define TURNSTILE_STATE_HASHTABLE 0x4
168#define TURNSTILE_STATE_PROPRIETOR 0x8
169
170/* Helper macros to set/unset turnstile state flags */
171#if DEVELOPMENT || DEBUG
172
173#define turnstile_state_init(ts, state) \
174MACRO_BEGIN \
175 ts->ts_state = state; \
176MACRO_END
177
178#define turnstile_state_add(ts, state) \
179MACRO_BEGIN \
180 assert((ts->ts_state & (state)) == 0); \
181 ts->ts_state |= state; \
182MACRO_END
183
184#define turnstile_state_remove(ts, state) \
185MACRO_BEGIN \
186 assert(ts->ts_state & (state)); \
187 ts->ts_state &= ~(state); \
188MACRO_END
189
190#else /* DEVELOPMENT || DEBUG */
191
192#define turnstile_state_init(ts, state) \
193MACRO_BEGIN \
194 (void)ts; \
195MACRO_END
196
197#define turnstile_state_add(ts, state) \
198MACRO_BEGIN \
199 (void)ts; \
200MACRO_END
201
202#define turnstile_state_remove(ts, state) \
203MACRO_BEGIN \
204 (void)ts; \
205MACRO_END
206
207#endif /* DEVELOPMENT || DEBUG */
208
209struct knote;
210struct turnstile;
211
212/*
213 * Turnstile update flags
214 *
215 * TURNSTILE_IMMEDIATE_UPDATE
216 * When passed to turnstile_update_inheritor
217 * update the inheritor of the turnstile in
218 * the same call.
219 *
220 * TURNSTILE_DELAYED_UPDATE
221 * When passed to turnstile_update_inheritor
222 * it stashed the inheritor on the thread and
223 * turnstile's inheritor is updated in
224 * assert wait.
225 *
226 * TURNSTILE_INHERITOR_THREAD
227 * The turnstile inheritor is of type thread.
228 *
229 * TURNSTILE_INHERITOR_TURNSTILE
230 * The turnstile inheritor is of type turnstile.
231 *
232 * TURNSTILE_INHERITOR_WORKQ
233 * The turnstile inheritor is of type workqueue
234 *
235 * TURNSTILE_INHERITOR_NEEDS_PRI_UPDATE
236 * The inheritor needs a chain priority update.
237 *
238 * TURNSTILE_NEEDS_PRI_UPDATE
239 * Current turnstile needs a chain priority update.
240 *
241 * Locking order for passing thread and turnstile as inheritor
242 *
243 * Thread as an inheritor:
244 * When thread is passed as an inheritor of a turnstile
245 * turnstile lock is taken and then thread lock.
246 *
247 * Turnstile as in inheritor:
248 * When turnstile (T1) is passed as an inheritor of
249 * a turnstile (T2), turnstile lock of T2 is taken
250 * and then turnstile lock of T1 is taken.
251 *
252 * Caution: While passing turnstile as an inheritor, its
253 * job of the adopter to make sure that there is no
254 * lock inversion.
255 */
256typedef enum __attribute__((flag_enum)) __attribute__((packed)) turnstile_update_flags {
257 TURNSTILE_UPDATE_FLAGS_NONE = 0,
258 TURNSTILE_IMMEDIATE_UPDATE = 0x1,
259 TURNSTILE_DELAYED_UPDATE = 0x2,
260 TURNSTILE_INHERITOR_THREAD = 0x4,
261 TURNSTILE_INHERITOR_TURNSTILE = 0x8,
262 TURNSTILE_INHERITOR_NEEDS_PRI_UPDATE = 0x10,
263 TURNSTILE_NEEDS_PRI_UPDATE = 0x20,
264 TURNSTILE_INHERITOR_WORKQ = 0x40,
265 TURNSTILE_UPDATE_BOOST = 0x80,
266} turnstile_update_flags_t;
267
268#define TURNSTILE_NULL ((struct turnstile *)0)
269
270typedef void * turnstile_inheritor_t;
271
272#define TURNSTILE_INHERITOR_NULL NULL
273
274#ifdef XNU_KERNEL_PRIVATE
275#pragma GCC visibility push(hidden)
276
277/* Turnstile stats update flags
278 *
279 * TSU_TURNSTILE_BLOCK_COUNT
280 * thread blocking on turnstile waitq, increment global
281 * thread block on turnstile count.
282 *
283 * TSU_REGULAR_WAITQ_BLOCK_COUNT
284 * thread blocking on regular waitq, increment global
285 * thread block on regular waitq count.
286 *
287 * TSU_PRI_PROPAGATION
288 * turnstile propagation update stopped at nth hop, update
289 * priority change count for nth element in stats array.
290 *
291 * TSU_NO_INHERITOR
292 * turnstile propagation update stopped due to turnstile
293 * not having an inheritor after nth hop, update the no
294 * inheritor count for nth element in the stats array.
295 *
296 * TSU_NO_TURNSTILE
297 * turnstile propagation update stopped due to thread
298 * not blocked on a turnstile waitq after nth hop, update
299 * the no turnstile count for the nth element in the stats
300 * array.
301 *
302 * TSU_NO_PRI_CHANGE_NEEDED
303 * turnstile propagation update stopped due to thread or
304 * turnstile having the correct priority or not blocked.
305 * update the no priority change count for the nth element
306 * in the stats array.
307 *
308 * TSU_THREAD_RUNNABLE
309 * turnstile propagation update stopped due to thread
310 * being runnable, update the thread runnable count for
311 * the nth element in the stats array.
312 *
313 * TSU_ABOVE_UI_PRI_CHANGE
314 * turnstile propagation caused an above UI priority change.
315 */
316typedef enum __attribute__((flag_enum)) turnstile_stats_update_flags {
317 TSU_FLAGS_NONE = 0,
318 TSU_TURNSTILE_BLOCK_COUNT = 0x1,
319 TSU_REGULAR_WAITQ_BLOCK_COUNT = 0x2,
320 TSU_PRI_PROPAGATION = 0x4,
321 TSU_NO_INHERITOR = 0x8,
322 TSU_NO_TURNSTILE = 0x10,
323 TSU_NO_PRI_CHANGE_NEEDED = 0x20,
324 TSU_THREAD_RUNNABLE = 0x40,
325 TSU_ABOVE_UI_PRI_CHANGE = 0x80,
326 TSU_THREAD_ARG = 0x100,
327 TSU_TURNSTILE_ARG = 0x200,
328 TSU_BOOST_ARG = 0x400,
329} turnstile_stats_update_flags_t;
330
331SLIST_HEAD(turnstile_list, turnstile);
332
333#define CTSID_BITS 20
334#define CTSID_MASK ((1u << CTSID_BITS) - 1)
335#define CTSID_MAX (CTSID_MASK - 1)
336
337struct turnstile {
338 union {
339 /*
340 * The waitq_eventmask field is only used on the global queues.
341 * We hence repurpose all those bits for our own use.
342 */
343#if MACH_KERNEL_PRIVATE
344 WAITQ_FLAGS(ts_waitq
345 , __ts_unused_bits: 7
346 , ts_compact_id: CTSID_BITS);
347#endif
348 struct waitq ts_waitq; /* waitq embedded in turnstile */
349 };
350#define ts_inheritor ts_waitq.waitq_inheritor /* thread/turnstile inheriting the priority (IL, WL) */
351 union {
352 struct turnstile_list ts_free_turnstiles; /* turnstile free list (IL) */
353 SLIST_ENTRY(turnstile) ts_free_elm; /* turnstile free list element (IL) */
354 struct mpsc_queue_chain ts_deallocate_link; /* thread deallocate link */
355 };
356 struct priority_queue_sched_max ts_inheritor_queue; /* Queue of turnstile with us as an inheritor (WL) */
357 struct priority_queue_entry_sched ts_inheritor_links; /* Inheritor queue links */
358 SLIST_ENTRY(turnstile) ts_htable_link; /* linkage for turnstile in global hash table */
359 uintptr_t ts_proprietor; /* hash key lookup turnstile (IL) */
360 os_ref_atomic_t ts_refcount; /* reference count for turnstiles */
361 _Atomic uint32_t ts_type_gencount; /* gen count used for priority chaining (IL), type of turnstile (IL) */
362 uint32_t ts_prim_count; /* counter used by the primitive */
363 turnstile_update_flags_t ts_inheritor_flags; /* flags for turnstile inheritor (IL, WL) */
364 uint8_t ts_priority; /* priority of turnstile (WL) */
365
366#if DEVELOPMENT || DEBUG
367 uint8_t ts_state; /* current state of turnstile (IL) */
368 thread_t ts_thread; /* thread the turnstile is attached to */
369 thread_t ts_prev_thread; /* thread the turnstile was attached before donation */
370#endif
371};
372
373#define waitq_to_turnstile(waitq) __container_of(waitq, struct turnstile, ts_waitq)
374
375/* IL - interlock, WL - turnstile lock i.e. waitq lock */
376
377#define TURNSTILE_PROPRIETOR_NULL 0ul
378
379/*
380 * Name: turnstiles_init
381 *
382 * Description: Initialize turnstile sub system.
383 *
384 * Args: None.
385 *
386 * Returns: None.
387 */
388void
389turnstiles_init(void);
390
391/*
392 * Name: turnstile_alloc
393 *
394 * Description: Allocate a turnstile.
395 *
396 * Args: None.
397 *
398 * Returns:
399 * turnstile on Success.
400 */
401struct turnstile *
402turnstile_alloc(void);
403
404/*
405 * Name: turnstile_compact_id_get()
406 *
407 * Description: Allocate a compact turnstile ID slot.
408 *
409 * Args: None.
410 *
411 * Returns:
412 * A non 0 compact compact turnstile ID.
413 */
414uint32_t
415turnstile_compact_id_get(void);
416
417/*
418 * Name: turnstile_compact_id_put()
419 *
420 * Description: Frees a compact turnstile ID slot.
421 *
422 * Args:
423 * Args1: the compact ID to free.
424 */
425void
426turnstile_compact_id_put(uint32_t cid);
427
428/*
429 * Name: turnstile_get_by_id
430 *
431 * Description: Resolve a turnstile by compact ID
432 *
433 * Args:
434 * Arg1: turnstile compact ID
435 *
436 * Returns: a turnstile
437 */
438struct turnstile *
439turnstile_get_by_id(uint32_t tsid);
440
441/*
442 * Name: turnstile_reference
443 *
444 * Description: Take a reference on the turnstile.
445 *
446 * Arg1: turnstile
447 *
448 * Returns: None.
449 */
450void
451turnstile_reference(struct turnstile *turnstile);
452
453/*
454 * Name: turnstile_deallocate
455 *
456 * Description: Drop a reference on the turnstile.
457 * Destroy the turnstile if the last ref.
458 *
459 * Arg1: turnstile
460 *
461 * Returns: None.
462 */
463void
464turnstile_deallocate(struct turnstile *turnstile);
465
466/*
467 * Name: turnstile_waitq_add_thread_priority_queue
468 *
469 * Description: add thread to the turnstile waitq
470 *
471 * Arg1: waitq
472 * Arg2: thread
473 *
474 * Conditions: waitq locked
475 */
476void
477turnstile_waitq_add_thread_priority_queue(
478 struct waitq* wq,
479 thread_t thread);
480
481/*
482 * Name: turnstile_deallocate_safe
483 *
484 * Description: Drop a reference on the turnstile safely without triggering zfree.
485 *
486 * Arg1: turnstile
487 *
488 * Returns: None.
489 */
490void
491turnstile_deallocate_safe(struct turnstile *turnstile);
492
493/*
494 * Name: turnstile_recompute_priority_locked
495 *
496 * Description: Update turnstile priority based
497 * on highest waiter thread and highest blocking
498 * turnstile.
499 *
500 * Args: turnstile
501 *
502 * Returns: TRUE: if the turnstile priority changed and needs propagation.
503 * FALSE: if the turnstile priority did not change or it does not need propagation.
504 *
505 * Condition: turnstile locked
506 */
507boolean_t
508turnstile_recompute_priority_locked(
509 struct turnstile *turnstile);
510
511/*
512 * Name: turnstile_recompute_priority
513 *
514 * Description: Update turnstile priority based
515 * on highest waiter thread and highest blocking
516 * turnstile.
517 *
518 * Args: turnstile
519 *
520 * Returns: TRUE: if the turnstile priority changed and needs propagation.
521 * FALSE: if the turnstile priority did not change or it does not need propagation.
522 */
523boolean_t
524turnstile_recompute_priority(
525 struct turnstile *turnstile);
526
527/*
528 * Name: turnstile_workq_proprietor_of_max_turnstile
529 *
530 * Description: Returns the highest priority and proprietor of a turnstile
531 * pushing on a workqueue turnstile.
532 *
533 * This will not return waiters that are at priority
534 * MAXPRI_THROTTLE or lower.
535 *
536 * Args: turnstile
537 *
538 * Returns:
539 * Priority of the max entry, or 0
540 * Pointer to the max entry proprietor
541 */
542int
543turnstile_workq_proprietor_of_max_turnstile(
544 struct turnstile *turnstile,
545 uintptr_t *proprietor);
546
547/*
548 * Name: turnstile_workloop_pusher_info
549 *
550 * Description: Returns the priority of the turnstile push for a workloop,
551 * and the thread or knote responsible for this push.
552 *
553 * Args: workloop turnstile
554 *
555 * Returns:
556 * Priority of the push or 0
557 * Thread (with a +1 reference) with that push or THREAD_NULL.
558 * Port (with a +1 reference) with that push, or IP_NULL.
559 * Sync IPC knote with the highest push (or NULL)
560 */
561int
562turnstile_workloop_pusher_info(
563 struct turnstile *turnstile,
564 thread_t *thread,
565 ipc_port_t *port,
566 struct knote **knote_out);
567
568/*
569 * Name: turnstile_cleanup
570 *
571 * Description: Update priority of a turnstile inheritor
572 * if needed.
573 *
574 * Args: inheritor and flags passed on thread struct.
575 *
576 * Returns: None.
577 */
578void
579turnstile_cleanup(void);
580
581/*
582 * Name: turnstile_update_thread_priority_chain
583 *
584 * Description: Priority of a thread blocked on a turnstile
585 * has changed, update the turnstile priority.
586 *
587 * Arg1: thread: thread whose priority has changed.
588 *
589 * Returns: None.
590 */
591void
592turnstile_update_thread_priority_chain(thread_t thread);
593
594/*
595 * Name: turnstile_update_inheritor_locked
596 *
597 * Description: Update the inheritor of the turnstile and boost the
598 * inheritor, called with turnstile locked.
599 *
600 * Args:
601 * Arg1: turnstile
602 * Implicit arg: new inheritor value is stashed in current thread's struct
603 *
604 * Returns:
605 * old inheritor reference is returned on current thread's struct.
606 */
607void
608turnstile_update_inheritor_locked(struct turnstile *turnstile);
609
610/*
611 * Name: thread_get_inheritor_turnstile_base_priority
612 *
613 * Description: Get the max base priority of all the inheritor turnstiles
614 *
615 * Arg1: thread
616 *
617 * Returns: Max base priority of all the inheritor turnstiles.
618 *
619 * Condition: thread locked
620 */
621int
622thread_get_inheritor_turnstile_base_priority(thread_t thread);
623
624/*
625 * Name: thread_get_inheritor_turnstile_sched_priority
626 *
627 * Description: Get the max sched priority of all the inheritor turnstiles
628 *
629 * Arg1: thread
630 *
631 * Returns: Max sched priority of all the inheritor turnstiles.
632 *
633 * Condition: thread locked
634 */
635int
636thread_get_inheritor_turnstile_sched_priority(thread_t thread);
637
638/*
639 * Name: thread_get_waiting_turnstile
640 *
641 * Description: Get the turnstile if the thread is waiting on a turnstile.
642 *
643 * Arg1: thread
644 *
645 * Returns: turnstile: if the thread is blocked on a turnstile.
646 * TURNSTILE_NULL: otherwise.
647 *
648 * Condition: thread locked.
649 */
650struct turnstile *
651thread_get_waiting_turnstile(thread_t thread);
652
653/*
654 * Name: turnstile_lookup_by_proprietor
655 *
656 * Description: Get turnstile for a proprietor from global
657 * turnstile hash.
658 *
659 * Arg1: port
660 * Arg2: turnstile_type_t type
661 *
662 * Returns: turnstile: if the proprietor has a turnstile.
663 * TURNSTILE_NULL: otherwise.
664 *
665 * Condition: proprietor interlock held.
666 */
667struct turnstile *
668turnstile_lookup_by_proprietor(uintptr_t proprietor, turnstile_type_t type);
669
670/*
671 * Name: turnstile_has_waiters
672 *
673 * Description: returns if there are waiters on the turnstile
674 *
675 * Arg1: turnstile: turnstile
676 *
677 * Returns: TRUE if there are waiters, FALSE otherwise.
678 */
679
680boolean_t
681turnstile_has_waiters(struct turnstile *turnstile);
682
683/*
684 * Name: turnstile_stats_update
685 *
686 * Description: Function to update turnstile stats for dev kernel.
687 *
688 * Arg1: hops : number of thread hops in priority propagation
689 * Arg2: flags : turnstile stats update flags
690 * Arg3: inheritor: inheritor
691 *
692 * Returns: Nothing
693 */
694void
695turnstile_stats_update(
696 int hop __assert_only,
697 turnstile_stats_update_flags_t flags __assert_only,
698 turnstile_inheritor_t inheritor __assert_only);
699
700#if DEVELOPMENT || DEBUG
701
702#define SYSCTL_TURNSTILE_TEST_USER_DEFAULT 1
703#define SYSCTL_TURNSTILE_TEST_USER_HASHTABLE 2
704#define SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT 3
705#define SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE 4
706
707/* Functions used by debug test primitive exported by sysctls */
708int
709tstile_test_prim_lock(int val);
710
711int
712tstile_test_prim_unlock(int val);
713
714int
715turnstile_get_boost_stats_sysctl(void *req);
716int
717turnstile_get_unboost_stats_sysctl(void *req);
718#endif /* DEVELOPMENT || DEBUG */
719
720#pragma GCC visibility pop
721#endif /* XNU_KERNEL_PRIVATE */
722
723/* Interface */
724
725/*
726 * Name: turnstile_hash_bucket_lock
727 *
728 * Description: locks the spinlock associated with proprietor's bucket.
729 * if proprietor is specified the index for the hash will be
730 * recomputed and returned in index_proprietor,
731 * otherwise the value save in index_proprietor is used as index.
732 *
733 * Args:
734 * Arg1: proprietor (key) for hashing
735 * Arg2: index for proprietor in the hash
736 * Arg3: turnstile type
737 *
738 * Returns: old value of irq if irq were disabled before acquiring the lock.
739 */
740unsigned
741turnstile_hash_bucket_lock(uintptr_t proprietor, uint32_t *index_proprietor, turnstile_type_t type);
742
743/*
744 * Name: turnstile_hash_bucket_unlock
745 *
746 * Description: unlocks the spinlock associated with proprietor's bucket.
747 * if proprietor is specified the index for the hash will be
748 * recomputed and returned in index_proprietor,
749 * otherwise the value save in index_proprietor is used as index.
750 *
751 * Args:
752 * Arg1: proprietor (key) for hashing
753 * Arg2: index for proprietor in the hash
754 * Arg3: turnstile type
755 * Arg4: irq value returned by turnstile_hash_bucket_lock
756 *
757 */
758void
759turnstile_hash_bucket_unlock(uintptr_t proprietor, uint32_t *index_proprietor, turnstile_type_t type, unsigned s);
760
761/*
762 * Name: turnstile_prepare
763 *
764 * Description: Transfer current thread's turnstile to primitive or it's free turnstile list.
765 * Function is called holding the interlock (spinlock) of the primitive.
766 * The turnstile returned by this function is safe to use until
767 * the thread calls turnstile_complete.
768 * When no turnstile is provided explicitly, the calling thread will not have a turnstile attached to
769 * it until it calls turnstile_complete.
770 *
771 * Args:
772 * Arg1: proprietor
773 * Arg2: pointer in primitive struct to store turnstile
774 * Arg3: turnstile to use instead of taking it from thread.
775 * Arg4: type of primitive
776 *
777 * Returns:
778 * turnstile.
779 */
780struct turnstile *
781turnstile_prepare(
782 uintptr_t proprietor,
783 struct turnstile **tstore,
784 struct turnstile *turnstile,
785 turnstile_type_t type);
786
787/*
788 * Name: turnstile_complete
789 *
790 * Description: Transfer the primitive's turnstile or from it's freelist to current thread.
791 * Function is called holding the interlock (spinlock) of the primitive.
792 * Current thread will have a turnstile attached to it after this call.
793 *
794 * Args:
795 * Arg1: proprietor
796 * Arg2: pointer in primitive struct to update turnstile
797 * Arg3: pointer to store the returned turnstile instead of attaching it to thread
798 * Arg4: type of primitive
799 *
800 * Returns:
801 * None.
802 */
803void
804turnstile_complete(
805 uintptr_t proprietor,
806 struct turnstile **tstore,
807 struct turnstile **turnstile,
808 turnstile_type_t type);
809
810/*
811 * Name: turnstile_prepare_compact_id
812 *
813 * Description: Transfer current thread's turnstile to primitive or it's free turnstile list.
814 * Function is called holding the interlock (spinlock) of the primitive.
815 * The turnstile returned by this function is safe to use until
816 * the thread calls turnstile_complete_compact_id.
817 * The calling thread will not have a turnstile attached to
818 * it until it calls turnstile_complete_compact_id.
819 *
820 * Args:
821 * Arg1: proprietor
822 * Arg2: the current compact ID for the turnstile head
823 * Arg3: type of primitive
824 *
825 * Returns:
826 * turnstile.
827 */
828struct turnstile *
829turnstile_prepare_compact_id(
830 uintptr_t proprietor,
831 uint32_t compact_id,
832 turnstile_type_t type);
833
834/*
835 * Name: turnstile_complete_compact_id
836 *
837 * Description: Transfer the primitive's turnstile or from it's freelist to current thread.
838 * Function is called holding the interlock (spinlock) of the primitive.
839 * Current thread will have a turnstile attached to it after this call.
840 *
841 * Args:
842 * Arg1: proprietor
843 * Arg2: the turnstile pointer that was returned by turnstile_prepare_compact_id()
844 * Arg3: type of primitive
845 *
846 * Returns:
847 * Whether the primitive no longer has a turnstile.
848 */
849bool
850turnstile_complete_compact_id(
851 uintptr_t proprietor,
852 struct turnstile *turnstile,
853 turnstile_type_t type);
854
855/*
856 * Name: turnstile_prepare_hash
857 *
858 * Description: Transfer current thread's turnstile to primitive or it's free turnstile list.
859 * Function is called holding the interlock (spinlock) of the primitive.
860 * The turnstile returned by this function is safe to use until
861 * the thread calls turnstile_complete_hash.
862 * The calling thread will not have a turnstile attached to
863 * it until it calls turnstile_complete_hash.
864 *
865 * The turnstile used for this proprietor will be stored in
866 * a global hash table.
867 *
868 * Args:
869 * Arg1: proprietor
870 * Arg3: type of primitive
871 *
872 * Returns:
873 * turnstile.
874 */
875struct turnstile *
876turnstile_prepare_hash(
877 uintptr_t proprietor,
878 turnstile_type_t type);
879
880/*
881 * Name: turnstile_complete_hash
882 *
883 * Description: Transfer the primitive's turnstile or from it's freelist to current thread.
884 * Function is called holding the interlock (spinlock) of the primitive.
885 * Current thread will have a turnstile attached to it after this call.
886 *
887 * Args:
888 * Arg1: proprietor
889 * Arg3: type of primitive
890 *
891 * Returns:
892 * None.
893 */
894void
895turnstile_complete_hash(
896 uintptr_t proprietor,
897 turnstile_type_t type);
898
899/*
900 * Name: turnstile_update_inheritor
901 *
902 * Description: Update the inheritor of the turnstile and boost the
903 * inheritor. It will take a thread reference on the inheritor.
904 * Called with the interlock of the primitive held.
905 *
906 * Args:
907 * Arg1: turnstile
908 * Arg2: inheritor
909 * Arg3: flags - TURNSTILE_DELAYED_UPDATE - update will happen later in assert_wait
910 *
911 * Returns:
912 * old inheritor reference is stashed on current thread's struct.
913 */
914void
915turnstile_update_inheritor(
916 struct turnstile *turnstile,
917 turnstile_inheritor_t new_inheritor,
918 turnstile_update_flags_t flags);
919
920typedef enum turnstile_update_complete_flags {
921 TURNSTILE_INTERLOCK_NOT_HELD = 0x1,
922 TURNSTILE_INTERLOCK_HELD = 0x2,
923} turnstile_update_complete_flags_t;
924
925/*
926 * Name: turnstile_update_inheritor_complete
927 *
928 * Description: Update turnstile inheritor's priority and propagate the
929 * priority if the inheritor is blocked on a turnstile.
930 * Consumes thread ref of old inheritor returned by
931 * turnstile_update_inheritor. Recursive priority update
932 * will only happen when called with interlock dropped.
933 *
934 * Args:
935 * Arg1: turnstile
936 * Arg2: interlock held
937 *
938 * Returns: None.
939 */
940void
941turnstile_update_inheritor_complete(
942 struct turnstile *turnstile,
943 turnstile_update_complete_flags_t flags);
944
945
946/*
947 * Name: turnstile_kernel_update_inheritor_on_wake_locked
948 *
949 * Description: Set thread as the inheritor of the turnstile and
950 * boost the inheritor.
951 * Args:
952 * Arg1: turnstile
953 * Arg2: new_inheritor
954 * Arg3: flags
955 *
956 * Called with turnstile locked
957 */
958void
959turnstile_kernel_update_inheritor_on_wake_locked(
960 struct turnstile *turnstile,
961 turnstile_inheritor_t new_inheritor,
962 turnstile_update_flags_t flags);
963
964#endif /* KERNEL_PRIVATE */
965#if XNU_KERNEL_PRIVATE
966
967struct workqueue;
968
969/* pthread_workqueue.c */
970extern void workq_reference(struct workqueue *wq);
971extern void workq_deallocate_safe(struct workqueue *wq);
972extern bool workq_is_current_thread_updating_turnstile(struct workqueue *wq);
973extern void workq_schedule_creator_turnstile_redrive(struct workqueue *wq,
974 bool locked);
975
976#endif /* XNU_KERNEL_PRIVATE */
977
978__END_DECLS
979
980#endif /* _TURNSTILE_H_ */
981