1 | /* |
2 | * Copyright (c) 2017 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #ifndef _TURNSTILE_H_ |
30 | #define _TURNSTILE_H_ |
31 | |
32 | #include <mach/mach_types.h> |
33 | #include <mach/kern_return.h> |
34 | #include <sys/cdefs.h> |
35 | |
36 | __BEGIN_DECLS |
37 | |
38 | #if PRIVATE |
39 | #define TURNSTILE_MAX_HOP_DEFAULT (10) |
40 | struct turnstile_stats { |
41 | uint64_t ts_priority_propagation; |
42 | uint64_t ts_no_inheritor; |
43 | uint64_t ts_thread_runnable; |
44 | uint64_t ts_no_priority_change_required; |
45 | uint64_t ts_above_ui_pri_change; |
46 | uint64_t ts_no_turnstile; |
47 | }; |
48 | #endif |
49 | |
50 | #ifdef KERNEL_PRIVATE |
51 | #include <kern/queue.h> |
52 | #include <sys/queue.h> |
53 | #include <kern/waitq.h> |
54 | #include <kern/priority_queue.h> |
55 | #include <os/refcnt.h> |
56 | #include <kern/assert.h> |
57 | #include <kern/kern_types.h> |
58 | #include <kern/mpsc_queue.h> |
59 | #include <kern/locks.h> |
60 | |
61 | /* |
62 | * turnstile_type_t : Indicates the type of primitive the turnstile is associated with |
63 | * Please populate turnstile_promote_policy array if a new type is added here. |
64 | */ |
65 | typedef enum __attribute__((packed)) turnstile_type { |
66 | TURNSTILE_NONE = 0, |
67 | TURNSTILE_KERNEL_MUTEX = 1, |
68 | TURNSTILE_ULOCK = 2, |
69 | TURNSTILE_PTHREAD_MUTEX = 3, |
70 | TURNSTILE_SYNC_IPC = 4, |
71 | TURNSTILE_WORKLOOPS = 5, |
72 | TURNSTILE_WORKQS = 6, |
73 | TURNSTILE_KNOTE = 7, |
74 | TURNSTILE_SLEEP_INHERITOR = 8, |
75 | TURNSTILE_EPOCH_KERNEL = 9, |
76 | TURNSTILE_EPOCH_USER = 10, |
77 | TURNSTILE_TOTAL_TYPES = 11, |
78 | } turnstile_type_t; |
79 | |
80 | /* |
81 | * For each type of turnstile, following are the type of |
82 | * inheritors passed: |
83 | * |
84 | * TURNSTILE_KERNEL_MUTEX |
85 | * Interlock: kernel mutex interlock. |
86 | * Inheritor: threads. |
87 | * Lock order: turnstile lock, thread lock. |
88 | * |
89 | * TURNSTILE_ULOCK |
90 | * Interlock: ulocks interlock. |
91 | * Inheritor: threads. |
92 | * Lock order: turnstile lock, thread lock. |
93 | * |
94 | * TURNSTILE_PTHREAD_MUTEX |
95 | * Interlock: pthread mtx interlock. |
96 | * Inheritor: threads. |
97 | * Lock order: turnstile lock, thread lock. |
98 | * |
99 | * TURNSTILE_SYNC_IPC |
100 | * Interlock: port's mqueue lock |
101 | * Inheritor: turnstile (of port in which we are enqueued or WL turnstile. |
102 | * Lock order: Our turnstile, then turnstile of the port we are enqueued in. |
103 | * Port circularity will make sure there is never a cycle formation |
104 | * and lock order is maintained. |
105 | * |
106 | * TURNSTILE_WORKLOOPS |
107 | * Interlock: |
108 | * - kq req lock |
109 | * - wq lock when "filt_wlworkq_interlock_needed() is true" |
110 | * Inheritor: thread, turnstile (of workq) |
111 | * Lock order: turnstile lock, thread lock |
112 | * WL turnstile lock, Workq turnstile lock |
113 | * |
114 | * TURNSTILE_WORKQS |
115 | * Interlock: workqueue lock |
116 | * Inheritor: thread |
117 | * Lock order: turnstile lock, thread lock. |
118 | * |
119 | * TURNSTILE_KNOTE |
120 | * Interlock: the knote lock |
121 | * Inheritor: WL turnstile |
122 | * |
123 | * TURNSTILE_SLEEP_INHERITOR |
124 | * Interlock: turnstile_htable bucket spinlock. |
125 | * Inheritor: threads. |
126 | * Lock order: turnstile lock, thread lock. |
127 | * |
128 | * TURNSTILE_EPOCH_KERNEL |
129 | * Interlock: the epoch sync interlock. |
130 | * Inheritor: threads. |
131 | * Lock order: turnstile lock, thread lock. |
132 | * |
133 | * TURNSTILE_EPOCH_USER |
134 | * Interlock: the epoch sync interlock. |
135 | * Inheritor: threads. |
136 | * Lock order: turnstile lock, thread lock. |
137 | */ |
138 | |
139 | typedef enum __attribute__((flag_enum)) turnstile_promote_policy { |
140 | TURNSTILE_PROMOTE_NONE = 0, |
141 | TURNSTILE_KERNEL_PROMOTE = 0x1, |
142 | TURNSTILE_USER_PROMOTE = 0x2, |
143 | TURNSTILE_USER_IPC_PROMOTE = 0x4, |
144 | } turnstile_promote_policy_t; |
145 | |
146 | typedef enum __attribute__((flag_enum)) turnstile_hash_lock_policy { |
147 | TURNSTILE_HASH_LOCK_POLICY_NONE = 0, |
148 | TURNSTILE_IRQ_UNSAFE_HASH = 0x1, |
149 | TURNSTILE_LOCKED_HASH = 0x2, |
150 | } turnstile_hash_lock_policy_t; |
151 | |
152 | /* |
153 | * Turnstile state flags |
154 | * |
155 | * The turnstile state flags represent the current ownership of a turnstile. |
156 | * The supported flags are: |
157 | * - TURNSTILE_STATE_THREAD : Turnstile is attached to a thread |
158 | * - TURNSTILE_STATE_FREELIST : Turnstile is hanging off the freelist of another turnstile |
159 | * - TURNSTILE_STATE_HASHTABLE : Turnstile is in the global hash table as the turnstile for a primitive |
160 | * - TURNSTILE_STATE_PROPRIETOR : Turnstile is attached to a proprietor |
161 | * |
162 | * The flag updates are done while holding the primitive interlock. |
163 | * */ |
164 | |
165 | #define TURNSTILE_STATE_THREAD 0x1 |
166 | #define TURNSTILE_STATE_FREELIST 0x2 |
167 | #define TURNSTILE_STATE_HASHTABLE 0x4 |
168 | #define TURNSTILE_STATE_PROPRIETOR 0x8 |
169 | |
170 | /* Helper macros to set/unset turnstile state flags */ |
171 | #if DEVELOPMENT || DEBUG |
172 | |
173 | #define turnstile_state_init(ts, state) \ |
174 | MACRO_BEGIN \ |
175 | ts->ts_state = state; \ |
176 | MACRO_END |
177 | |
178 | #define turnstile_state_add(ts, state) \ |
179 | MACRO_BEGIN \ |
180 | assert((ts->ts_state & (state)) == 0); \ |
181 | ts->ts_state |= state; \ |
182 | MACRO_END |
183 | |
184 | #define turnstile_state_remove(ts, state) \ |
185 | MACRO_BEGIN \ |
186 | assert(ts->ts_state & (state)); \ |
187 | ts->ts_state &= ~(state); \ |
188 | MACRO_END |
189 | |
190 | #else /* DEVELOPMENT || DEBUG */ |
191 | |
192 | #define turnstile_state_init(ts, state) \ |
193 | MACRO_BEGIN \ |
194 | (void)ts; \ |
195 | MACRO_END |
196 | |
197 | #define turnstile_state_add(ts, state) \ |
198 | MACRO_BEGIN \ |
199 | (void)ts; \ |
200 | MACRO_END |
201 | |
202 | #define turnstile_state_remove(ts, state) \ |
203 | MACRO_BEGIN \ |
204 | (void)ts; \ |
205 | MACRO_END |
206 | |
207 | #endif /* DEVELOPMENT || DEBUG */ |
208 | |
209 | struct knote; |
210 | struct turnstile; |
211 | |
212 | /* |
213 | * Turnstile update flags |
214 | * |
215 | * TURNSTILE_IMMEDIATE_UPDATE |
216 | * When passed to turnstile_update_inheritor |
217 | * update the inheritor of the turnstile in |
218 | * the same call. |
219 | * |
220 | * TURNSTILE_DELAYED_UPDATE |
221 | * When passed to turnstile_update_inheritor |
222 | * it stashed the inheritor on the thread and |
223 | * turnstile's inheritor is updated in |
224 | * assert wait. |
225 | * |
226 | * TURNSTILE_INHERITOR_THREAD |
227 | * The turnstile inheritor is of type thread. |
228 | * |
229 | * TURNSTILE_INHERITOR_TURNSTILE |
230 | * The turnstile inheritor is of type turnstile. |
231 | * |
232 | * TURNSTILE_INHERITOR_WORKQ |
233 | * The turnstile inheritor is of type workqueue |
234 | * |
235 | * TURNSTILE_INHERITOR_NEEDS_PRI_UPDATE |
236 | * The inheritor needs a chain priority update. |
237 | * |
238 | * TURNSTILE_NEEDS_PRI_UPDATE |
239 | * Current turnstile needs a chain priority update. |
240 | * |
241 | * Locking order for passing thread and turnstile as inheritor |
242 | * |
243 | * Thread as an inheritor: |
244 | * When thread is passed as an inheritor of a turnstile |
245 | * turnstile lock is taken and then thread lock. |
246 | * |
247 | * Turnstile as in inheritor: |
248 | * When turnstile (T1) is passed as an inheritor of |
249 | * a turnstile (T2), turnstile lock of T2 is taken |
250 | * and then turnstile lock of T1 is taken. |
251 | * |
252 | * Caution: While passing turnstile as an inheritor, its |
253 | * job of the adopter to make sure that there is no |
254 | * lock inversion. |
255 | */ |
256 | typedef enum __attribute__((flag_enum)) __attribute__((packed)) turnstile_update_flags { |
257 | TURNSTILE_UPDATE_FLAGS_NONE = 0, |
258 | TURNSTILE_IMMEDIATE_UPDATE = 0x1, |
259 | TURNSTILE_DELAYED_UPDATE = 0x2, |
260 | TURNSTILE_INHERITOR_THREAD = 0x4, |
261 | TURNSTILE_INHERITOR_TURNSTILE = 0x8, |
262 | TURNSTILE_INHERITOR_NEEDS_PRI_UPDATE = 0x10, |
263 | TURNSTILE_NEEDS_PRI_UPDATE = 0x20, |
264 | TURNSTILE_INHERITOR_WORKQ = 0x40, |
265 | TURNSTILE_UPDATE_BOOST = 0x80, |
266 | } turnstile_update_flags_t; |
267 | |
268 | #define TURNSTILE_NULL ((struct turnstile *)0) |
269 | |
270 | typedef void * turnstile_inheritor_t; |
271 | |
272 | #define TURNSTILE_INHERITOR_NULL NULL |
273 | |
274 | #ifdef XNU_KERNEL_PRIVATE |
275 | #pragma GCC visibility push(hidden) |
276 | |
277 | /* Turnstile stats update flags |
278 | * |
279 | * TSU_TURNSTILE_BLOCK_COUNT |
280 | * thread blocking on turnstile waitq, increment global |
281 | * thread block on turnstile count. |
282 | * |
283 | * TSU_REGULAR_WAITQ_BLOCK_COUNT |
284 | * thread blocking on regular waitq, increment global |
285 | * thread block on regular waitq count. |
286 | * |
287 | * TSU_PRI_PROPAGATION |
288 | * turnstile propagation update stopped at nth hop, update |
289 | * priority change count for nth element in stats array. |
290 | * |
291 | * TSU_NO_INHERITOR |
292 | * turnstile propagation update stopped due to turnstile |
293 | * not having an inheritor after nth hop, update the no |
294 | * inheritor count for nth element in the stats array. |
295 | * |
296 | * TSU_NO_TURNSTILE |
297 | * turnstile propagation update stopped due to thread |
298 | * not blocked on a turnstile waitq after nth hop, update |
299 | * the no turnstile count for the nth element in the stats |
300 | * array. |
301 | * |
302 | * TSU_NO_PRI_CHANGE_NEEDED |
303 | * turnstile propagation update stopped due to thread or |
304 | * turnstile having the correct priority or not blocked. |
305 | * update the no priority change count for the nth element |
306 | * in the stats array. |
307 | * |
308 | * TSU_THREAD_RUNNABLE |
309 | * turnstile propagation update stopped due to thread |
310 | * being runnable, update the thread runnable count for |
311 | * the nth element in the stats array. |
312 | * |
313 | * TSU_ABOVE_UI_PRI_CHANGE |
314 | * turnstile propagation caused an above UI priority change. |
315 | */ |
316 | typedef enum __attribute__((flag_enum)) turnstile_stats_update_flags { |
317 | TSU_FLAGS_NONE = 0, |
318 | TSU_TURNSTILE_BLOCK_COUNT = 0x1, |
319 | TSU_REGULAR_WAITQ_BLOCK_COUNT = 0x2, |
320 | TSU_PRI_PROPAGATION = 0x4, |
321 | TSU_NO_INHERITOR = 0x8, |
322 | TSU_NO_TURNSTILE = 0x10, |
323 | TSU_NO_PRI_CHANGE_NEEDED = 0x20, |
324 | TSU_THREAD_RUNNABLE = 0x40, |
325 | TSU_ABOVE_UI_PRI_CHANGE = 0x80, |
326 | TSU_THREAD_ARG = 0x100, |
327 | TSU_TURNSTILE_ARG = 0x200, |
328 | TSU_BOOST_ARG = 0x400, |
329 | } turnstile_stats_update_flags_t; |
330 | |
331 | SLIST_HEAD(turnstile_list, turnstile); |
332 | |
333 | #define CTSID_BITS 20 |
334 | #define CTSID_MASK ((1u << CTSID_BITS) - 1) |
335 | #define CTSID_MAX (CTSID_MASK - 1) |
336 | |
337 | struct turnstile { |
338 | union { |
339 | /* |
340 | * The waitq_eventmask field is only used on the global queues. |
341 | * We hence repurpose all those bits for our own use. |
342 | */ |
343 | #if MACH_KERNEL_PRIVATE |
344 | WAITQ_FLAGS(ts_waitq |
345 | , __ts_unused_bits: 7 |
346 | , ts_compact_id: CTSID_BITS); |
347 | #endif |
348 | struct waitq ts_waitq; /* waitq embedded in turnstile */ |
349 | }; |
350 | #define ts_inheritor ts_waitq.waitq_inheritor /* thread/turnstile inheriting the priority (IL, WL) */ |
351 | union { |
352 | struct turnstile_list ts_free_turnstiles; /* turnstile free list (IL) */ |
353 | SLIST_ENTRY(turnstile) ts_free_elm; /* turnstile free list element (IL) */ |
354 | struct mpsc_queue_chain ts_deallocate_link; /* thread deallocate link */ |
355 | }; |
356 | struct priority_queue_sched_max ts_inheritor_queue; /* Queue of turnstile with us as an inheritor (WL) */ |
357 | struct priority_queue_entry_sched ts_inheritor_links; /* Inheritor queue links */ |
358 | SLIST_ENTRY(turnstile) ts_htable_link; /* linkage for turnstile in global hash table */ |
359 | uintptr_t ts_proprietor; /* hash key lookup turnstile (IL) */ |
360 | os_ref_atomic_t ts_refcount; /* reference count for turnstiles */ |
361 | _Atomic uint32_t ts_type_gencount; /* gen count used for priority chaining (IL), type of turnstile (IL) */ |
362 | uint32_t ts_prim_count; /* counter used by the primitive */ |
363 | turnstile_update_flags_t ts_inheritor_flags; /* flags for turnstile inheritor (IL, WL) */ |
364 | uint8_t ts_priority; /* priority of turnstile (WL) */ |
365 | |
366 | #if DEVELOPMENT || DEBUG |
367 | uint8_t ts_state; /* current state of turnstile (IL) */ |
368 | thread_t ts_thread; /* thread the turnstile is attached to */ |
369 | thread_t ts_prev_thread; /* thread the turnstile was attached before donation */ |
370 | #endif |
371 | }; |
372 | |
373 | #define waitq_to_turnstile(waitq) __container_of(waitq, struct turnstile, ts_waitq) |
374 | |
375 | /* IL - interlock, WL - turnstile lock i.e. waitq lock */ |
376 | |
377 | #define TURNSTILE_PROPRIETOR_NULL 0ul |
378 | |
379 | /* |
380 | * Name: turnstiles_init |
381 | * |
382 | * Description: Initialize turnstile sub system. |
383 | * |
384 | * Args: None. |
385 | * |
386 | * Returns: None. |
387 | */ |
388 | void |
389 | turnstiles_init(void); |
390 | |
391 | /* |
392 | * Name: turnstile_alloc |
393 | * |
394 | * Description: Allocate a turnstile. |
395 | * |
396 | * Args: None. |
397 | * |
398 | * Returns: |
399 | * turnstile on Success. |
400 | */ |
401 | struct turnstile * |
402 | turnstile_alloc(void); |
403 | |
404 | /* |
405 | * Name: turnstile_compact_id_get() |
406 | * |
407 | * Description: Allocate a compact turnstile ID slot. |
408 | * |
409 | * Args: None. |
410 | * |
411 | * Returns: |
412 | * A non 0 compact compact turnstile ID. |
413 | */ |
414 | uint32_t |
415 | turnstile_compact_id_get(void); |
416 | |
417 | /* |
418 | * Name: turnstile_compact_id_put() |
419 | * |
420 | * Description: Frees a compact turnstile ID slot. |
421 | * |
422 | * Args: |
423 | * Args1: the compact ID to free. |
424 | */ |
425 | void |
426 | turnstile_compact_id_put(uint32_t cid); |
427 | |
428 | /* |
429 | * Name: turnstile_get_by_id |
430 | * |
431 | * Description: Resolve a turnstile by compact ID |
432 | * |
433 | * Args: |
434 | * Arg1: turnstile compact ID |
435 | * |
436 | * Returns: a turnstile |
437 | */ |
438 | struct turnstile * |
439 | turnstile_get_by_id(uint32_t tsid); |
440 | |
441 | /* |
442 | * Name: turnstile_reference |
443 | * |
444 | * Description: Take a reference on the turnstile. |
445 | * |
446 | * Arg1: turnstile |
447 | * |
448 | * Returns: None. |
449 | */ |
450 | void |
451 | turnstile_reference(struct turnstile *turnstile); |
452 | |
453 | /* |
454 | * Name: turnstile_deallocate |
455 | * |
456 | * Description: Drop a reference on the turnstile. |
457 | * Destroy the turnstile if the last ref. |
458 | * |
459 | * Arg1: turnstile |
460 | * |
461 | * Returns: None. |
462 | */ |
463 | void |
464 | turnstile_deallocate(struct turnstile *turnstile); |
465 | |
466 | /* |
467 | * Name: turnstile_waitq_add_thread_priority_queue |
468 | * |
469 | * Description: add thread to the turnstile waitq |
470 | * |
471 | * Arg1: waitq |
472 | * Arg2: thread |
473 | * |
474 | * Conditions: waitq locked |
475 | */ |
476 | void |
477 | turnstile_waitq_add_thread_priority_queue( |
478 | struct waitq* wq, |
479 | thread_t thread); |
480 | |
481 | /* |
482 | * Name: turnstile_deallocate_safe |
483 | * |
484 | * Description: Drop a reference on the turnstile safely without triggering zfree. |
485 | * |
486 | * Arg1: turnstile |
487 | * |
488 | * Returns: None. |
489 | */ |
490 | void |
491 | turnstile_deallocate_safe(struct turnstile *turnstile); |
492 | |
493 | /* |
494 | * Name: turnstile_recompute_priority_locked |
495 | * |
496 | * Description: Update turnstile priority based |
497 | * on highest waiter thread and highest blocking |
498 | * turnstile. |
499 | * |
500 | * Args: turnstile |
501 | * |
502 | * Returns: TRUE: if the turnstile priority changed and needs propagation. |
503 | * FALSE: if the turnstile priority did not change or it does not need propagation. |
504 | * |
505 | * Condition: turnstile locked |
506 | */ |
507 | boolean_t |
508 | turnstile_recompute_priority_locked( |
509 | struct turnstile *turnstile); |
510 | |
511 | /* |
512 | * Name: turnstile_recompute_priority |
513 | * |
514 | * Description: Update turnstile priority based |
515 | * on highest waiter thread and highest blocking |
516 | * turnstile. |
517 | * |
518 | * Args: turnstile |
519 | * |
520 | * Returns: TRUE: if the turnstile priority changed and needs propagation. |
521 | * FALSE: if the turnstile priority did not change or it does not need propagation. |
522 | */ |
523 | boolean_t |
524 | turnstile_recompute_priority( |
525 | struct turnstile *turnstile); |
526 | |
527 | /* |
528 | * Name: turnstile_workq_proprietor_of_max_turnstile |
529 | * |
530 | * Description: Returns the highest priority and proprietor of a turnstile |
531 | * pushing on a workqueue turnstile. |
532 | * |
533 | * This will not return waiters that are at priority |
534 | * MAXPRI_THROTTLE or lower. |
535 | * |
536 | * Args: turnstile |
537 | * |
538 | * Returns: |
539 | * Priority of the max entry, or 0 |
540 | * Pointer to the max entry proprietor |
541 | */ |
542 | int |
543 | turnstile_workq_proprietor_of_max_turnstile( |
544 | struct turnstile *turnstile, |
545 | uintptr_t *proprietor); |
546 | |
547 | /* |
548 | * Name: turnstile_workloop_pusher_info |
549 | * |
550 | * Description: Returns the priority of the turnstile push for a workloop, |
551 | * and the thread or knote responsible for this push. |
552 | * |
553 | * Args: workloop turnstile |
554 | * |
555 | * Returns: |
556 | * Priority of the push or 0 |
557 | * Thread (with a +1 reference) with that push or THREAD_NULL. |
558 | * Port (with a +1 reference) with that push, or IP_NULL. |
559 | * Sync IPC knote with the highest push (or NULL) |
560 | */ |
561 | int |
562 | turnstile_workloop_pusher_info( |
563 | struct turnstile *turnstile, |
564 | thread_t *thread, |
565 | ipc_port_t *port, |
566 | struct knote **knote_out); |
567 | |
568 | /* |
569 | * Name: turnstile_cleanup |
570 | * |
571 | * Description: Update priority of a turnstile inheritor |
572 | * if needed. |
573 | * |
574 | * Args: inheritor and flags passed on thread struct. |
575 | * |
576 | * Returns: None. |
577 | */ |
578 | void |
579 | turnstile_cleanup(void); |
580 | |
581 | /* |
582 | * Name: turnstile_update_thread_priority_chain |
583 | * |
584 | * Description: Priority of a thread blocked on a turnstile |
585 | * has changed, update the turnstile priority. |
586 | * |
587 | * Arg1: thread: thread whose priority has changed. |
588 | * |
589 | * Returns: None. |
590 | */ |
591 | void |
592 | turnstile_update_thread_priority_chain(thread_t thread); |
593 | |
594 | /* |
595 | * Name: turnstile_update_inheritor_locked |
596 | * |
597 | * Description: Update the inheritor of the turnstile and boost the |
598 | * inheritor, called with turnstile locked. |
599 | * |
600 | * Args: |
601 | * Arg1: turnstile |
602 | * Implicit arg: new inheritor value is stashed in current thread's struct |
603 | * |
604 | * Returns: |
605 | * old inheritor reference is returned on current thread's struct. |
606 | */ |
607 | void |
608 | turnstile_update_inheritor_locked(struct turnstile *turnstile); |
609 | |
610 | /* |
611 | * Name: thread_get_inheritor_turnstile_base_priority |
612 | * |
613 | * Description: Get the max base priority of all the inheritor turnstiles |
614 | * |
615 | * Arg1: thread |
616 | * |
617 | * Returns: Max base priority of all the inheritor turnstiles. |
618 | * |
619 | * Condition: thread locked |
620 | */ |
621 | int |
622 | thread_get_inheritor_turnstile_base_priority(thread_t thread); |
623 | |
624 | /* |
625 | * Name: thread_get_inheritor_turnstile_sched_priority |
626 | * |
627 | * Description: Get the max sched priority of all the inheritor turnstiles |
628 | * |
629 | * Arg1: thread |
630 | * |
631 | * Returns: Max sched priority of all the inheritor turnstiles. |
632 | * |
633 | * Condition: thread locked |
634 | */ |
635 | int |
636 | thread_get_inheritor_turnstile_sched_priority(thread_t thread); |
637 | |
638 | /* |
639 | * Name: thread_get_waiting_turnstile |
640 | * |
641 | * Description: Get the turnstile if the thread is waiting on a turnstile. |
642 | * |
643 | * Arg1: thread |
644 | * |
645 | * Returns: turnstile: if the thread is blocked on a turnstile. |
646 | * TURNSTILE_NULL: otherwise. |
647 | * |
648 | * Condition: thread locked. |
649 | */ |
650 | struct turnstile * |
651 | thread_get_waiting_turnstile(thread_t thread); |
652 | |
653 | /* |
654 | * Name: turnstile_lookup_by_proprietor |
655 | * |
656 | * Description: Get turnstile for a proprietor from global |
657 | * turnstile hash. |
658 | * |
659 | * Arg1: port |
660 | * Arg2: turnstile_type_t type |
661 | * |
662 | * Returns: turnstile: if the proprietor has a turnstile. |
663 | * TURNSTILE_NULL: otherwise. |
664 | * |
665 | * Condition: proprietor interlock held. |
666 | */ |
667 | struct turnstile * |
668 | turnstile_lookup_by_proprietor(uintptr_t proprietor, turnstile_type_t type); |
669 | |
670 | /* |
671 | * Name: turnstile_has_waiters |
672 | * |
673 | * Description: returns if there are waiters on the turnstile |
674 | * |
675 | * Arg1: turnstile: turnstile |
676 | * |
677 | * Returns: TRUE if there are waiters, FALSE otherwise. |
678 | */ |
679 | |
680 | boolean_t |
681 | turnstile_has_waiters(struct turnstile *turnstile); |
682 | |
683 | /* |
684 | * Name: turnstile_stats_update |
685 | * |
686 | * Description: Function to update turnstile stats for dev kernel. |
687 | * |
688 | * Arg1: hops : number of thread hops in priority propagation |
689 | * Arg2: flags : turnstile stats update flags |
690 | * Arg3: inheritor: inheritor |
691 | * |
692 | * Returns: Nothing |
693 | */ |
694 | void |
695 | turnstile_stats_update( |
696 | int hop __assert_only, |
697 | turnstile_stats_update_flags_t flags __assert_only, |
698 | turnstile_inheritor_t inheritor __assert_only); |
699 | |
700 | #if DEVELOPMENT || DEBUG |
701 | |
702 | #define SYSCTL_TURNSTILE_TEST_USER_DEFAULT 1 |
703 | #define SYSCTL_TURNSTILE_TEST_USER_HASHTABLE 2 |
704 | #define SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT 3 |
705 | #define SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE 4 |
706 | |
707 | /* Functions used by debug test primitive exported by sysctls */ |
708 | int |
709 | tstile_test_prim_lock(int val); |
710 | |
711 | int |
712 | tstile_test_prim_unlock(int val); |
713 | |
714 | int |
715 | turnstile_get_boost_stats_sysctl(void *req); |
716 | int |
717 | turnstile_get_unboost_stats_sysctl(void *req); |
718 | #endif /* DEVELOPMENT || DEBUG */ |
719 | |
720 | #pragma GCC visibility pop |
721 | #endif /* XNU_KERNEL_PRIVATE */ |
722 | |
723 | /* Interface */ |
724 | |
725 | /* |
726 | * Name: turnstile_hash_bucket_lock |
727 | * |
728 | * Description: locks the spinlock associated with proprietor's bucket. |
729 | * if proprietor is specified the index for the hash will be |
730 | * recomputed and returned in index_proprietor, |
731 | * otherwise the value save in index_proprietor is used as index. |
732 | * |
733 | * Args: |
734 | * Arg1: proprietor (key) for hashing |
735 | * Arg2: index for proprietor in the hash |
736 | * Arg3: turnstile type |
737 | * |
738 | * Returns: old value of irq if irq were disabled before acquiring the lock. |
739 | */ |
740 | unsigned |
741 | turnstile_hash_bucket_lock(uintptr_t proprietor, uint32_t *index_proprietor, turnstile_type_t type); |
742 | |
743 | /* |
744 | * Name: turnstile_hash_bucket_unlock |
745 | * |
746 | * Description: unlocks the spinlock associated with proprietor's bucket. |
747 | * if proprietor is specified the index for the hash will be |
748 | * recomputed and returned in index_proprietor, |
749 | * otherwise the value save in index_proprietor is used as index. |
750 | * |
751 | * Args: |
752 | * Arg1: proprietor (key) for hashing |
753 | * Arg2: index for proprietor in the hash |
754 | * Arg3: turnstile type |
755 | * Arg4: irq value returned by turnstile_hash_bucket_lock |
756 | * |
757 | */ |
758 | void |
759 | turnstile_hash_bucket_unlock(uintptr_t proprietor, uint32_t *index_proprietor, turnstile_type_t type, unsigned s); |
760 | |
761 | /* |
762 | * Name: turnstile_prepare |
763 | * |
764 | * Description: Transfer current thread's turnstile to primitive or it's free turnstile list. |
765 | * Function is called holding the interlock (spinlock) of the primitive. |
766 | * The turnstile returned by this function is safe to use until |
767 | * the thread calls turnstile_complete. |
768 | * When no turnstile is provided explicitly, the calling thread will not have a turnstile attached to |
769 | * it until it calls turnstile_complete. |
770 | * |
771 | * Args: |
772 | * Arg1: proprietor |
773 | * Arg2: pointer in primitive struct to store turnstile |
774 | * Arg3: turnstile to use instead of taking it from thread. |
775 | * Arg4: type of primitive |
776 | * |
777 | * Returns: |
778 | * turnstile. |
779 | */ |
780 | struct turnstile * |
781 | turnstile_prepare( |
782 | uintptr_t proprietor, |
783 | struct turnstile **tstore, |
784 | struct turnstile *turnstile, |
785 | turnstile_type_t type); |
786 | |
787 | /* |
788 | * Name: turnstile_complete |
789 | * |
790 | * Description: Transfer the primitive's turnstile or from it's freelist to current thread. |
791 | * Function is called holding the interlock (spinlock) of the primitive. |
792 | * Current thread will have a turnstile attached to it after this call. |
793 | * |
794 | * Args: |
795 | * Arg1: proprietor |
796 | * Arg2: pointer in primitive struct to update turnstile |
797 | * Arg3: pointer to store the returned turnstile instead of attaching it to thread |
798 | * Arg4: type of primitive |
799 | * |
800 | * Returns: |
801 | * None. |
802 | */ |
803 | void |
804 | turnstile_complete( |
805 | uintptr_t proprietor, |
806 | struct turnstile **tstore, |
807 | struct turnstile **turnstile, |
808 | turnstile_type_t type); |
809 | |
810 | /* |
811 | * Name: turnstile_prepare_compact_id |
812 | * |
813 | * Description: Transfer current thread's turnstile to primitive or it's free turnstile list. |
814 | * Function is called holding the interlock (spinlock) of the primitive. |
815 | * The turnstile returned by this function is safe to use until |
816 | * the thread calls turnstile_complete_compact_id. |
817 | * The calling thread will not have a turnstile attached to |
818 | * it until it calls turnstile_complete_compact_id. |
819 | * |
820 | * Args: |
821 | * Arg1: proprietor |
822 | * Arg2: the current compact ID for the turnstile head |
823 | * Arg3: type of primitive |
824 | * |
825 | * Returns: |
826 | * turnstile. |
827 | */ |
828 | struct turnstile * |
829 | turnstile_prepare_compact_id( |
830 | uintptr_t proprietor, |
831 | uint32_t compact_id, |
832 | turnstile_type_t type); |
833 | |
834 | /* |
835 | * Name: turnstile_complete_compact_id |
836 | * |
837 | * Description: Transfer the primitive's turnstile or from it's freelist to current thread. |
838 | * Function is called holding the interlock (spinlock) of the primitive. |
839 | * Current thread will have a turnstile attached to it after this call. |
840 | * |
841 | * Args: |
842 | * Arg1: proprietor |
843 | * Arg2: the turnstile pointer that was returned by turnstile_prepare_compact_id() |
844 | * Arg3: type of primitive |
845 | * |
846 | * Returns: |
847 | * Whether the primitive no longer has a turnstile. |
848 | */ |
849 | bool |
850 | turnstile_complete_compact_id( |
851 | uintptr_t proprietor, |
852 | struct turnstile *turnstile, |
853 | turnstile_type_t type); |
854 | |
855 | /* |
856 | * Name: turnstile_prepare_hash |
857 | * |
858 | * Description: Transfer current thread's turnstile to primitive or it's free turnstile list. |
859 | * Function is called holding the interlock (spinlock) of the primitive. |
860 | * The turnstile returned by this function is safe to use until |
861 | * the thread calls turnstile_complete_hash. |
862 | * The calling thread will not have a turnstile attached to |
863 | * it until it calls turnstile_complete_hash. |
864 | * |
865 | * The turnstile used for this proprietor will be stored in |
866 | * a global hash table. |
867 | * |
868 | * Args: |
869 | * Arg1: proprietor |
870 | * Arg3: type of primitive |
871 | * |
872 | * Returns: |
873 | * turnstile. |
874 | */ |
875 | struct turnstile * |
876 | turnstile_prepare_hash( |
877 | uintptr_t proprietor, |
878 | turnstile_type_t type); |
879 | |
880 | /* |
881 | * Name: turnstile_complete_hash |
882 | * |
883 | * Description: Transfer the primitive's turnstile or from it's freelist to current thread. |
884 | * Function is called holding the interlock (spinlock) of the primitive. |
885 | * Current thread will have a turnstile attached to it after this call. |
886 | * |
887 | * Args: |
888 | * Arg1: proprietor |
889 | * Arg3: type of primitive |
890 | * |
891 | * Returns: |
892 | * None. |
893 | */ |
894 | void |
895 | turnstile_complete_hash( |
896 | uintptr_t proprietor, |
897 | turnstile_type_t type); |
898 | |
899 | /* |
900 | * Name: turnstile_update_inheritor |
901 | * |
902 | * Description: Update the inheritor of the turnstile and boost the |
903 | * inheritor. It will take a thread reference on the inheritor. |
904 | * Called with the interlock of the primitive held. |
905 | * |
906 | * Args: |
907 | * Arg1: turnstile |
908 | * Arg2: inheritor |
909 | * Arg3: flags - TURNSTILE_DELAYED_UPDATE - update will happen later in assert_wait |
910 | * |
911 | * Returns: |
912 | * old inheritor reference is stashed on current thread's struct. |
913 | */ |
914 | void |
915 | turnstile_update_inheritor( |
916 | struct turnstile *turnstile, |
917 | turnstile_inheritor_t new_inheritor, |
918 | turnstile_update_flags_t flags); |
919 | |
920 | typedef enum turnstile_update_complete_flags { |
921 | TURNSTILE_INTERLOCK_NOT_HELD = 0x1, |
922 | TURNSTILE_INTERLOCK_HELD = 0x2, |
923 | } turnstile_update_complete_flags_t; |
924 | |
925 | /* |
926 | * Name: turnstile_update_inheritor_complete |
927 | * |
928 | * Description: Update turnstile inheritor's priority and propagate the |
929 | * priority if the inheritor is blocked on a turnstile. |
930 | * Consumes thread ref of old inheritor returned by |
931 | * turnstile_update_inheritor. Recursive priority update |
932 | * will only happen when called with interlock dropped. |
933 | * |
934 | * Args: |
935 | * Arg1: turnstile |
936 | * Arg2: interlock held |
937 | * |
938 | * Returns: None. |
939 | */ |
940 | void |
941 | turnstile_update_inheritor_complete( |
942 | struct turnstile *turnstile, |
943 | turnstile_update_complete_flags_t flags); |
944 | |
945 | |
946 | /* |
947 | * Name: turnstile_kernel_update_inheritor_on_wake_locked |
948 | * |
949 | * Description: Set thread as the inheritor of the turnstile and |
950 | * boost the inheritor. |
951 | * Args: |
952 | * Arg1: turnstile |
953 | * Arg2: new_inheritor |
954 | * Arg3: flags |
955 | * |
956 | * Called with turnstile locked |
957 | */ |
958 | void |
959 | turnstile_kernel_update_inheritor_on_wake_locked( |
960 | struct turnstile *turnstile, |
961 | turnstile_inheritor_t new_inheritor, |
962 | turnstile_update_flags_t flags); |
963 | |
964 | #endif /* KERNEL_PRIVATE */ |
965 | #if XNU_KERNEL_PRIVATE |
966 | |
967 | struct workqueue; |
968 | |
969 | /* pthread_workqueue.c */ |
970 | extern void workq_reference(struct workqueue *wq); |
971 | extern void workq_deallocate_safe(struct workqueue *wq); |
972 | extern bool workq_is_current_thread_updating_turnstile(struct workqueue *wq); |
973 | extern void workq_schedule_creator_turnstile_redrive(struct workqueue *wq, |
974 | bool locked); |
975 | |
976 | #endif /* XNU_KERNEL_PRIVATE */ |
977 | |
978 | __END_DECLS |
979 | |
980 | #endif /* _TURNSTILE_H_ */ |
981 | |