1 | /* |
2 | * Copyright (c) 2017 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #ifndef _TURNSTILE_H_ |
30 | #define _TURNSTILE_H_ |
31 | |
32 | #include <mach/mach_types.h> |
33 | #include <mach/kern_return.h> |
34 | #include <sys/cdefs.h> |
35 | |
36 | #if PRIVATE |
37 | #define TURNSTILE_MAX_HOP_DEFAULT (10) |
38 | struct turnstile_stats { |
39 | uint64_t ts_priority_propagation; |
40 | uint64_t ts_no_inheritor; |
41 | uint64_t ts_thread_runnable; |
42 | uint64_t ts_no_priority_change_required; |
43 | uint64_t ts_above_ui_pri_change; |
44 | uint64_t ts_no_turnstile; |
45 | }; |
46 | #endif |
47 | |
48 | #ifdef KERNEL_PRIVATE |
49 | #include <kern/queue.h> |
50 | #include <sys/queue.h> |
51 | #include <kern/waitq.h> |
52 | #include <kern/priority_queue.h> |
53 | #include <os/refcnt.h> |
54 | #include <kern/assert.h> |
55 | #include <kern/kern_types.h> |
56 | |
57 | /* |
58 | * turnstile_type_t : Indicates the type of primitive the turnstile is associated with |
59 | * Please populate turnstile_promote_policy array if a new type is added here. |
60 | */ |
61 | typedef enum __attribute__((packed)) turnstile_type { |
62 | TURNSTILE_NONE = 0, |
63 | TURNSTILE_KERNEL_MUTEX = 1, |
64 | TURNSTILE_ULOCK = 2, |
65 | TURNSTILE_PTHREAD_MUTEX = 3, |
66 | TURNSTILE_SYNC_IPC = 4, |
67 | TURNSTILE_WORKLOOPS = 5, |
68 | TURNSTILE_WORKQS = 6, |
69 | TURNSTILE_KNOTE = 7, |
70 | TURNSTILE_TOTAL_TYPES = 8, |
71 | } turnstile_type_t; |
72 | |
73 | /* |
74 | * For each type of turnstile, following are the type of |
75 | * inheritors passed: |
76 | * |
77 | * TURNSTILE_KERNEL_MUTEX |
78 | * Interlock: kernel mutex interlock. |
79 | * Inheritor: threads. |
80 | * Lock order: turnstile lock, thread lock. |
81 | * |
82 | * TURNSTILE_ULOCK |
83 | * Interlock: ulocks interlock. |
84 | * Inheritor: threads. |
85 | * Lock order: turnstile lock, thread lock. |
86 | * |
87 | * TURNSTILE_PTHREAD_MUTEX |
88 | * Interlock: pthread mtx interlock. |
89 | * Inheritor: threads. |
90 | * Lock order: turnstile lock, thread lock. |
91 | * |
92 | * TURNSTILE_SYNC_IPC |
93 | * Interlock: port's mqueue lock |
94 | * Inheritor: turnstile (of port in which we are enqueued or WL turnstile. |
95 | * Lock order: Our turnstile, then turnstile of the port we are enqueued in. |
96 | * Port circularity will make sure there is never a cycle formation |
97 | * and lock order is maintained. |
98 | * |
99 | * TURNSTILE_WORKLOOPS |
100 | * Interlock: |
101 | * - kq req lock |
102 | * - wq lock when "filt_wlworkq_interlock_needed() is true" |
103 | * Inheritor: thread, turnstile (of workq) |
104 | * Lock order: turnstile lock, thread lock |
105 | * WL turnstile lock, Workq turnstile lock |
106 | * |
107 | * TURNSTILE_WORKQS |
108 | * Interlock: workqueue lock |
109 | * Inheritor: thread |
110 | * Lock order: turnstile lock, thread lock. |
111 | * |
112 | * TURNSTILE_KNOTE |
113 | * Interlock: the knote lock |
114 | * Inheritor: WL turnstile |
115 | */ |
116 | |
117 | typedef enum __attribute__((flag_enum)) turnstile_promote_policy { |
118 | TURNSTILE_PROMOTE_NONE = 0, |
119 | TURNSTILE_KERNEL_PROMOTE = 0x1, |
120 | TURNSTILE_USER_PROMOTE = 0x2, |
121 | TURNSTILE_USER_IPC_PROMOTE = 0x4, |
122 | } turnstile_promote_policy_t; |
123 | |
124 | /* |
125 | * Turnstile state flags |
126 | * |
127 | * The turnstile state flags represent the current ownership of a turnstile. |
128 | * The supported flags are: |
129 | * - TURNSTILE_STATE_THREAD : Turnstile is attached to a thread |
130 | * - TURNSTILE_STATE_FREELIST : Turnstile is hanging off the freelist of another turnstile |
131 | * - TURNSTILE_STATE_HASHTABLE : Turnstile is in the global hash table as the turnstile for a primitive |
132 | * - TURNSTILE_STATE_PROPRIETOR : Turnstile is attached to a proprietor |
133 | * |
134 | * The flag updates are done while holding the primitive interlock. |
135 | * */ |
136 | |
137 | #define TURNSTILE_STATE_THREAD 0x1 |
138 | #define TURNSTILE_STATE_FREELIST 0x2 |
139 | #define TURNSTILE_STATE_HASHTABLE 0x4 |
140 | #define TURNSTILE_STATE_PROPRIETOR 0x8 |
141 | |
142 | /* Helper macros to set/unset turnstile state flags */ |
143 | #if DEVELOPMENT || DEBUG |
144 | |
145 | #define turnstile_state_init(ts, state) \ |
146 | MACRO_BEGIN \ |
147 | ts->ts_state = state; \ |
148 | MACRO_END |
149 | |
150 | #define turnstile_state_add(ts, state) \ |
151 | MACRO_BEGIN \ |
152 | assert((ts->ts_state & (state)) == 0); \ |
153 | ts->ts_state |= state; \ |
154 | MACRO_END |
155 | |
156 | #define turnstile_state_remove(ts, state) \ |
157 | MACRO_BEGIN \ |
158 | assert(ts->ts_state & (state)); \ |
159 | ts->ts_state &= ~(state); \ |
160 | MACRO_END |
161 | |
162 | #else /* DEVELOPMENT || DEBUG */ |
163 | |
164 | #define turnstile_state_init(ts, state) \ |
165 | MACRO_BEGIN \ |
166 | (void)ts; \ |
167 | MACRO_END |
168 | |
169 | #define turnstile_state_add(ts, state) \ |
170 | MACRO_BEGIN \ |
171 | (void)ts; \ |
172 | MACRO_END |
173 | |
174 | #define turnstile_state_remove(ts, state) \ |
175 | MACRO_BEGIN \ |
176 | (void)ts; \ |
177 | MACRO_END |
178 | |
179 | #endif /* DEVELOPMENT || DEBUG */ |
180 | |
181 | /* Foward declaration of turnstile */ |
182 | struct turnstile; |
183 | |
184 | /* |
185 | * Turnstile update flags |
186 | * |
187 | * TURNSTILE_IMMEDIATE_UPDATE |
188 | * When passed to turnstile_update_inheritor |
189 | * update the inheritor of the turnstile in |
190 | * the same call. |
191 | * |
192 | * TURNSTILE_DELAYED_UPDATE |
193 | * When passed to turnstile_update_inheritor |
194 | * it stashed the inheritor on the thread and |
195 | * turnstile's inheritor is updated in |
196 | * assert wait. |
197 | * |
198 | * TURNSTILE_INHERITOR_THREAD |
199 | * The turnstile inheritor is of type thread. |
200 | * |
201 | * TURNSTILE_INHERITOR_TURNSTILE |
202 | * The turnstile inheritor is of type turnstile. |
203 | * |
204 | * TURNSTILE_INHERITOR_WORKQ |
205 | * The turnstile inheritor is of type workqueue |
206 | * |
207 | * TURNSTILE_INHERITOR_NEEDS_PRI_UPDATE |
208 | * The inheritor needs a chain priority update. |
209 | * |
210 | * TURNSTILE_NEEDS_PRI_UPDATE |
211 | * Current turnstile needs a chain priority update. |
212 | * |
213 | * Locking order for passing thread and turnstile as inheritor |
214 | * |
215 | * Thread as an inheritor: |
216 | * When thread is passed as an inheritor of a turnstile |
217 | * turnstile lock is taken and then thread lock. |
218 | * |
219 | * Turnstile as in inheritor: |
220 | * When turnstile (T1) is passed as an inheritor of |
221 | * a turnstile (T2), turnstile lock of T2 is taken |
222 | * and then turnstile lock of T1 is taken. |
223 | * |
224 | * Caution: While passing turnstile as an inheritor, its |
225 | * job of the adopter to make sure that there is no |
226 | * lock inversion. |
227 | */ |
228 | typedef enum __attribute__((flag_enum)) __attribute__((packed)) turnstile_update_flags { |
229 | TURNSTILE_UPDATE_FLAGS_NONE = 0, |
230 | TURNSTILE_IMMEDIATE_UPDATE = 0x1, |
231 | TURNSTILE_DELAYED_UPDATE = 0x2, |
232 | TURNSTILE_INHERITOR_THREAD = 0x4, |
233 | TURNSTILE_INHERITOR_TURNSTILE = 0x8, |
234 | TURNSTILE_INHERITOR_NEEDS_PRI_UPDATE = 0x10, |
235 | TURNSTILE_NEEDS_PRI_UPDATE = 0x20, |
236 | TURNSTILE_INHERITOR_WORKQ = 0x40, |
237 | TURNSTILE_UPDATE_BOOST = 0x80, |
238 | } turnstile_update_flags_t; |
239 | |
240 | #define TURNSTILE_NULL ((struct turnstile *)0) |
241 | |
242 | typedef void * turnstile_inheritor_t; |
243 | |
244 | #define TURNSTILE_INHERITOR_NULL NULL |
245 | |
246 | #ifdef XNU_KERNEL_PRIVATE |
247 | |
248 | /* Turnstile stats update flags |
249 | * |
250 | * TSU_TURNSTILE_BLOCK_COUNT |
251 | * thread blocking on turnstile waitq, increment global |
252 | * thread block on turnstile count. |
253 | * |
254 | * TSU_REGULAR_WAITQ_BLOCK_COUNT |
255 | * thread blocking on regular waitq, increment global |
256 | * thread block on regular waitq count. |
257 | * |
258 | * TSU_PRI_PROPAGATION |
259 | * turnstile propagation update stopped at nth hop, update |
260 | * priority change count for nth element in stats array. |
261 | * |
262 | * TSU_NO_INHERITOR |
263 | * turnstile propagation update stopped due to turnstile |
264 | * not having an inheritor after nth hop, update the no |
265 | * inheritor count for nth element in the stats array. |
266 | * |
267 | * TSU_NO_TURNSTILE |
268 | * turnstile propagation update stopped due to thread |
269 | * not blocked on a turnstile waitq after nth hop, update |
270 | * the no turnstile count for the nth element in the stats |
271 | * array. |
272 | * |
273 | * TSU_NO_PRI_CHANGE_NEEDED |
274 | * turnstile propagation update stopped due to thread or |
275 | * turnstile having the correct priority or not blocked. |
276 | * update the no priority change count for the nth element |
277 | * in the stats array. |
278 | * |
279 | * TSU_THREAD_RUNNABLE |
280 | * turnstile propagation update stopped due to thread |
281 | * being runnable, update the thread runnable count for |
282 | * the nth element in the stats array. |
283 | * |
284 | * TSU_ABOVE_UI_PRI_CHANGE |
285 | * turnstile propagation caused an above UI priority change. |
286 | */ |
287 | typedef enum __attribute__((flag_enum)) turnstile_stats_update_flags { |
288 | TSU_FLAGS_NONE = 0, |
289 | TSU_TURNSTILE_BLOCK_COUNT = 0x1, |
290 | TSU_REGULAR_WAITQ_BLOCK_COUNT = 0x2, |
291 | TSU_PRI_PROPAGATION = 0x4, |
292 | TSU_NO_INHERITOR = 0x8, |
293 | TSU_NO_TURNSTILE = 0x10, |
294 | TSU_NO_PRI_CHANGE_NEEDED = 0x20, |
295 | TSU_THREAD_RUNNABLE = 0x40, |
296 | TSU_ABOVE_UI_PRI_CHANGE = 0x80, |
297 | TSU_THREAD_ARG = 0x100, |
298 | TSU_TURNSTILE_ARG = 0x200, |
299 | TSU_BOOST_ARG = 0x400, |
300 | } turnstile_stats_update_flags_t; |
301 | |
302 | SLIST_HEAD(turnstile_list, turnstile); |
303 | |
304 | struct turnstile { |
305 | struct waitq ts_waitq; /* waitq embedded in turnstile */ |
306 | turnstile_inheritor_t ts_inheritor; /* thread/turnstile inheriting the priority (IL, WL) */ |
307 | union { |
308 | struct turnstile_list ts_free_turnstiles; /* turnstile free list (IL) */ |
309 | SLIST_ENTRY(turnstile) ts_free_elm; /* turnstile free list element (IL) */ |
310 | }; |
311 | struct priority_queue ts_inheritor_queue; /* Queue of turnstile with us as an inheritor (WL) */ |
312 | union { |
313 | struct priority_queue_entry ts_inheritor_links; /* Inheritor queue links */ |
314 | queue_chain_t ts_deallocate_link; /* thread deallocate link */ |
315 | }; |
316 | SLIST_ENTRY(turnstile) ts_htable_link; /* linkage for turnstile in global hash table */ |
317 | uintptr_t ts_proprietor; /* hash key lookup turnstile (IL) */ |
318 | os_refcnt_t ts_refcount; /* reference count for turnstiles */ |
319 | _Atomic uint32_t ts_type_gencount; /* gen count used for priority chaining (IL), type of turnstile (IL) */ |
320 | uint32_t ts_port_ref; /* number of explicit refs from ports on send turnstile */ |
321 | turnstile_update_flags_t ts_inheritor_flags; /* flags for turnstile inheritor (IL, WL) */ |
322 | uint8_t ts_priority; /* priority of turnstile (WL) */ |
323 | |
324 | #if DEVELOPMENT || DEBUG |
325 | uint8_t ts_state; /* current state of turnstile (IL) */ |
326 | queue_chain_t ts_global_elm; /* global turnstile chain */ |
327 | thread_t ts_thread; /* thread the turnstile is attached to */ |
328 | thread_t ts_prev_thread; /* thread the turnstile was attached before donation */ |
329 | #endif |
330 | }; |
331 | |
332 | #define waitq_to_turnstile(waitq) __container_of(waitq, struct turnstile, ts_waitq) |
333 | |
334 | /* IL - interlock, WL - turnstile lock i.e. waitq lock */ |
335 | |
336 | #define TURNSTILE_PROPRIETOR_NULL 0 |
337 | |
338 | /* |
339 | * Name: turnstiles_init |
340 | * |
341 | * Description: Initialize turnstile sub system. |
342 | * |
343 | * Args: None. |
344 | * |
345 | * Returns: None. |
346 | */ |
347 | void |
348 | turnstiles_init(void); |
349 | |
350 | /* |
351 | * Name: turnstile_alloc |
352 | * |
353 | * Description: Allocate a turnstile. |
354 | * |
355 | * Args: None. |
356 | * |
357 | * Returns: |
358 | * turnstile on Success. |
359 | */ |
360 | struct turnstile * |
361 | turnstile_alloc(void); |
362 | |
363 | /* |
364 | * Name: turnstile_destroy |
365 | * |
366 | * Description: Deallocates the turnstile. |
367 | * |
368 | * Args: |
369 | * Arg1: turnstile |
370 | * |
371 | * Returns: None. |
372 | */ |
373 | void |
374 | turnstile_destroy(struct turnstile *turnstile); |
375 | |
376 | /* |
377 | * Name: turnstile_reference |
378 | * |
379 | * Description: Take a reference on the turnstile. |
380 | * |
381 | * Arg1: turnstile |
382 | * |
383 | * Returns: None. |
384 | */ |
385 | void |
386 | turnstile_reference(struct turnstile *turnstile); |
387 | |
388 | /* |
389 | * Name: turnstile_deallocate |
390 | * |
391 | * Description: Drop a reference on the turnstile. |
392 | * Destroy the turnstile if the last ref. |
393 | * |
394 | * Arg1: turnstile |
395 | * |
396 | * Returns: None. |
397 | */ |
398 | void |
399 | turnstile_deallocate(struct turnstile *turnstile); |
400 | |
401 | /* |
402 | * Name: turnstile_deallocate_safe |
403 | * |
404 | * Description: Drop a reference on the turnstile safely without triggering zfree. |
405 | * |
406 | * Arg1: turnstile |
407 | * |
408 | * Returns: None. |
409 | */ |
410 | void |
411 | turnstile_deallocate_safe(struct turnstile *turnstile); |
412 | |
413 | /* |
414 | * Name: turnstile_recompute_priority_locked |
415 | * |
416 | * Description: Update turnstile priority based |
417 | * on highest waiter thread and highest blocking |
418 | * turnstile. |
419 | * |
420 | * Args: turnstile |
421 | * |
422 | * Returns: TRUE: if the turnstile priority changed and needs propagation. |
423 | * FALSE: if the turnstile priority did not change or it does not need propagation. |
424 | * |
425 | * Condition: turnstile locked |
426 | */ |
427 | boolean_t |
428 | turnstile_recompute_priority_locked( |
429 | struct turnstile *turnstile); |
430 | |
431 | /* |
432 | * Name: turnstile_recompute_priority |
433 | * |
434 | * Description: Update turnstile priority based |
435 | * on highest waiter thread and highest blocking |
436 | * turnstile. |
437 | * |
438 | * Args: turnstile |
439 | * |
440 | * Returns: TRUE: if the turnstile priority changed and needs propagation. |
441 | * FALSE: if the turnstile priority did not change or it does not need propagation. |
442 | */ |
443 | boolean_t |
444 | turnstile_recompute_priority( |
445 | struct turnstile *turnstile); |
446 | |
447 | /* |
448 | * Name: turnstile_workq_proprietor_of_max_turnstile |
449 | * |
450 | * Description: Returns the highest priority and proprietor of a turnstile |
451 | * pushing on a workqueue turnstile. |
452 | * |
453 | * This will not return waiters that are at priority |
454 | * MAXPRI_THROTTLE or lower. |
455 | * |
456 | * Args: turnstile |
457 | * |
458 | * Returns: |
459 | * Priority of the max entry, or 0 |
460 | * Pointer to the max entry proprietor |
461 | */ |
462 | int |
463 | turnstile_workq_proprietor_of_max_turnstile( |
464 | struct turnstile *turnstile, |
465 | uintptr_t *proprietor); |
466 | |
467 | /* |
468 | * Name: turnstile_cleanup |
469 | * |
470 | * Description: Update priority of a turnstile inheritor |
471 | * if needed. |
472 | * |
473 | * Args: inheritor and flags passed on thread struct. |
474 | * |
475 | * Returns: None. |
476 | */ |
477 | void |
478 | turnstile_cleanup(void); |
479 | |
480 | /* |
481 | * Name: turnstile_update_inheritor_locked |
482 | * |
483 | * Description: Update the inheritor of the turnstile and boost the |
484 | * inheritor, called with turnstile locked. |
485 | * |
486 | * Args: |
487 | * Arg1: turnstile |
488 | * Implicit arg: new inheritor value is stashed in current thread's struct |
489 | * |
490 | * Returns: |
491 | * old inheritor reference is returned on current thread's struct. |
492 | */ |
493 | void |
494 | turnstile_update_inheritor_locked(struct turnstile *turnstile); |
495 | |
496 | /* |
497 | * Name: thread_get_inheritor_turnstile_priority |
498 | * |
499 | * Description: Get the max priority of all the inheritor turnstiles |
500 | * |
501 | * Arg1: thread |
502 | * |
503 | * Returns: Max priority of all the inheritor turnstiles. |
504 | * |
505 | * Condition: thread locked |
506 | */ |
507 | int |
508 | thread_get_inheritor_turnstile_priority(thread_t thread); |
509 | |
510 | /* |
511 | * Name: thread_get_waiting_turnstile |
512 | * |
513 | * Description: Get the turnstile if the thread is waiting on a turnstile. |
514 | * |
515 | * Arg1: thread |
516 | * |
517 | * Returns: turnstile: if the thread is blocked on a turnstile. |
518 | * TURNSTILE_NULL: otherwise. |
519 | * |
520 | * Condition: thread locked. |
521 | */ |
522 | struct turnstile * |
523 | thread_get_waiting_turnstile(thread_t thread); |
524 | |
525 | /* |
526 | * Name: turnstile_lookup_by_proprietor |
527 | * |
528 | * Description: Get turnstile for a proprietor from global |
529 | * turnstile hash. |
530 | * |
531 | * Arg1: port |
532 | * |
533 | * Returns: turnstile: if the proprietor has a turnstile. |
534 | * TURNSTILE_NULL: otherwise. |
535 | * |
536 | * Condition: proprietor interlock held. |
537 | */ |
538 | struct turnstile * |
539 | turnstile_lookup_by_proprietor(uintptr_t proprietor); |
540 | |
541 | /* |
542 | * Name: turnstile_stats_update |
543 | * |
544 | * Description: Function to update turnstile stats for dev kernel. |
545 | * |
546 | * Arg1: hops : number of thread hops in priority propagation |
547 | * Arg2: flags : turnstile stats update flags |
548 | * Arg3: inheritor: inheritor |
549 | * |
550 | * Returns: Nothing |
551 | */ |
552 | void |
553 | turnstile_stats_update( |
554 | int hop __assert_only, |
555 | turnstile_stats_update_flags_t flags __assert_only, |
556 | turnstile_inheritor_t inheritor __assert_only); |
557 | |
558 | #if DEVELOPMENT || DEBUG |
559 | |
560 | /* Functions used by debug test primitive exported by sysctls */ |
561 | int |
562 | tstile_test_prim_lock(boolean_t use_hashtable); |
563 | |
564 | int |
565 | tstile_test_prim_unlock(boolean_t use_hashtable); |
566 | |
567 | int |
568 | turnstile_get_boost_stats_sysctl(void *req); |
569 | int |
570 | turnstile_get_unboost_stats_sysctl(void *req); |
571 | #endif /* DEVELOPMENT || DEBUG */ |
572 | #endif /* XNU_KERNEL_PRIVATE */ |
573 | |
574 | /* Interface */ |
575 | |
576 | /* |
577 | * Name: turnstile_prepare |
578 | * |
579 | * Description: Transfer current thread's turnstile to primitive or it's free turnstile list. |
580 | * Function is called holding the interlock (spinlock) of the primitive. |
581 | * The turnstile returned by this function is safe to use untill the thread calls turnstile_complete. |
582 | * When no turnstile is provided explicitly, the calling thread will not have a turnstile attached to |
583 | * it untill it calls turnstile_complete. |
584 | * |
585 | * Args: |
586 | * Arg1: proprietor |
587 | * Arg2: pointer in primitive struct to store turnstile |
588 | * Arg3: turnstile to use instead of taking it from thread. |
589 | * Arg4: type of primitive |
590 | * |
591 | * Returns: |
592 | * turnstile. |
593 | */ |
594 | struct turnstile * |
595 | turnstile_prepare( |
596 | uintptr_t proprietor, |
597 | struct turnstile **tstore, |
598 | struct turnstile *turnstile, |
599 | turnstile_type_t type); |
600 | |
601 | /* |
602 | * Name: turnstile_complete |
603 | * |
604 | * Description: Transfer the primitive's turnstile or from it's freelist to current thread. |
605 | * Function is called holding the interlock (spinlock) of the primitive. |
606 | * Current thread will have a turnstile attached to it after this call. |
607 | * |
608 | * Args: |
609 | * Arg1: proprietor |
610 | * Arg2: pointer in primitive struct to update turnstile |
611 | * Arg3: pointer to store the returned turnstile instead of attaching it to thread |
612 | * |
613 | * Returns: |
614 | * None. |
615 | */ |
616 | void |
617 | turnstile_complete( |
618 | uintptr_t proprietor, |
619 | struct turnstile **tstore, |
620 | struct turnstile **turnstile); |
621 | |
622 | /* |
623 | * Name: turnstile_update_inheritor |
624 | * |
625 | * Description: Update the inheritor of the turnstile and boost the |
626 | * inheritor. It will take a thread reference on the inheritor. |
627 | * Called with the interlock of the primitive held. |
628 | * |
629 | * Args: |
630 | * Arg1: turnstile |
631 | * Arg2: inheritor |
632 | * Arg3: flags - TURNSTILE_DELAYED_UPDATE - update will happen later in assert_wait |
633 | * |
634 | * Returns: |
635 | * old inheritor reference is stashed on current thread's struct. |
636 | */ |
637 | void |
638 | turnstile_update_inheritor( |
639 | struct turnstile *turnstile, |
640 | turnstile_inheritor_t new_inheritor, |
641 | turnstile_update_flags_t flags); |
642 | |
643 | typedef enum turnstile_update_complete_flags { |
644 | TURNSTILE_INTERLOCK_NOT_HELD = 0x1, |
645 | TURNSTILE_INTERLOCK_HELD = 0x2, |
646 | } turnstile_update_complete_flags_t; |
647 | |
648 | /* |
649 | * Name: turnstile_update_inheritor_complete |
650 | * |
651 | * Description: Update turnstile inheritor's priority and propagate the |
652 | * priority if the inheritor is blocked on a turnstile. |
653 | * Consumes thread ref of old inheritor returned by |
654 | * turnstile_update_inheritor. Recursive priority update |
655 | * will only happen when called with interlock dropped. |
656 | * |
657 | * Args: |
658 | * Arg1: turnstile |
659 | * Arg2: interlock held |
660 | * |
661 | * Returns: None. |
662 | */ |
663 | void |
664 | turnstile_update_inheritor_complete( |
665 | struct turnstile *turnstile, |
666 | turnstile_update_complete_flags_t flags); |
667 | |
668 | #endif /* KERNEL_PRIVATE */ |
669 | #if XNU_KERNEL_PRIVATE |
670 | |
671 | struct workqueue; |
672 | |
673 | /* pthread_workqueue.c */ |
674 | extern void workq_reference(struct workqueue *wq); |
675 | extern void workq_deallocate_safe(struct workqueue *wq); |
676 | extern void workq_destroy(struct workqueue *wq); |
677 | extern bool workq_is_current_thread_updating_turnstile(struct workqueue *wq); |
678 | extern void workq_schedule_creator_turnstile_redrive(struct workqueue *wq, |
679 | bool locked); |
680 | |
681 | /* thread.c */ |
682 | extern void workq_deallocate_enqueue(struct workqueue *wq); |
683 | |
684 | #endif /* XNU_KERNEL_PRIVATE */ |
685 | |
686 | #endif /* _TURNSTILE_H_ */ |
687 | |