1 | /* |
2 | * Copyright (c) 2000-2012 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * @OSF_COPYRIGHT@ |
30 | */ |
31 | /* |
32 | * Mach Operating System |
33 | * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University |
34 | * All Rights Reserved. |
35 | * |
36 | * Permission to use, copy, modify and distribute this software and its |
37 | * documentation is hereby granted, provided that both the copyright |
38 | * notice and this permission notice appear in all copies of the |
39 | * software, derivative works or modified versions, and any portions |
40 | * thereof, and that both notices appear in supporting documentation. |
41 | * |
42 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" |
43 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR |
44 | * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. |
45 | * |
46 | * Carnegie Mellon requests users of this software to return to |
47 | * |
48 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU |
49 | * School of Computer Science |
50 | * Carnegie Mellon University |
51 | * Pittsburgh PA 15213-3890 |
52 | * |
53 | * any improvements or extensions that they make and grant Carnegie Mellon |
54 | * the rights to redistribute these changes. |
55 | */ |
56 | /* |
57 | */ |
58 | /* |
59 | * File: sched_prim.h |
60 | * Author: David Golub |
61 | * |
62 | * Scheduling primitive definitions file |
63 | * |
64 | */ |
65 | |
66 | #ifndef _KERN_SCHED_PRIM_H_ |
67 | #define _KERN_SCHED_PRIM_H_ |
68 | |
69 | #include <sys/cdefs.h> |
70 | #include <mach/boolean.h> |
71 | #include <mach/machine/vm_types.h> |
72 | #include <mach/kern_return.h> |
73 | #include <kern/clock.h> |
74 | #include <kern/kern_types.h> |
75 | #include <kern/percpu.h> |
76 | #include <kern/thread.h> |
77 | #include <kern/block_hint.h> |
78 | |
79 | extern int thread_get_current_cpuid(void); |
80 | |
81 | #ifdef MACH_KERNEL_PRIVATE |
82 | |
83 | #include <kern/sched_urgency.h> |
84 | #include <kern/thread_group.h> |
85 | #include <kern/waitq.h> |
86 | |
87 | /* Initialization */ |
88 | extern void sched_init(void); |
89 | |
90 | extern void sched_startup(void); |
91 | |
92 | extern void sched_timebase_init(void); |
93 | |
94 | extern void pset_rt_init(processor_set_t pset); |
95 | |
96 | extern void sched_rtlocal_init(processor_set_t pset); |
97 | |
98 | extern rt_queue_t sched_rtlocal_runq(processor_set_t pset); |
99 | |
100 | extern void sched_rtlocal_queue_shutdown(processor_t processor); |
101 | |
102 | extern int64_t sched_rtlocal_runq_count_sum(void); |
103 | |
104 | extern thread_t sched_rtlocal_steal_thread(processor_set_t stealing_pset, uint64_t earliest_deadline); |
105 | |
106 | extern thread_t sched_rt_choose_thread(processor_set_t pset); |
107 | |
108 | extern void sched_check_spill(processor_set_t pset, thread_t thread); |
109 | |
110 | extern bool sched_thread_should_yield(processor_t processor, thread_t thread); |
111 | |
112 | extern bool sched_steal_thread_DISABLED(processor_set_t pset); |
113 | extern bool sched_steal_thread_enabled(processor_set_t pset); |
114 | |
115 | /* Force a preemption point for a thread and wait for it to stop running */ |
116 | extern boolean_t thread_stop( |
117 | thread_t thread, |
118 | boolean_t until_not_runnable); |
119 | |
120 | /* Release a previous stop request */ |
121 | extern void thread_unstop( |
122 | thread_t thread); |
123 | |
124 | /* Wait for a thread to stop running */ |
125 | extern void thread_wait( |
126 | thread_t thread, |
127 | boolean_t until_not_runnable); |
128 | |
129 | /* Unblock thread on wake up */ |
130 | extern boolean_t thread_unblock( |
131 | thread_t thread, |
132 | wait_result_t wresult); |
133 | |
134 | /* Unblock and dispatch thread */ |
135 | extern void thread_go( |
136 | thread_t thread, |
137 | wait_result_t wresult, |
138 | bool try_handoff); |
139 | |
140 | /* Check if direct handoff is allowed */ |
141 | extern boolean_t |
142 | thread_allowed_for_handoff( |
143 | thread_t thread); |
144 | |
145 | /* Handle threads at context switch */ |
146 | extern void thread_dispatch( |
147 | thread_t old_thread, |
148 | thread_t new_thread); |
149 | |
150 | /* Switch directly to a particular thread */ |
151 | extern int thread_run( |
152 | thread_t self, |
153 | thread_continue_t continuation, |
154 | void *parameter, |
155 | thread_t new_thread); |
156 | |
157 | /* Resume thread with new stack */ |
158 | extern __dead2 void thread_continue(thread_t old_thread); |
159 | |
160 | /* Invoke continuation */ |
161 | extern __dead2 void call_continuation( |
162 | thread_continue_t continuation, |
163 | void *parameter, |
164 | wait_result_t wresult, |
165 | boolean_t enable_interrupts); |
166 | |
167 | /* |
168 | * Flags that can be passed to set_sched_pri |
169 | * to skip side effects |
170 | */ |
171 | __options_decl(set_sched_pri_options_t, uint32_t, { |
172 | SETPRI_DEFAULT = 0x0, |
173 | SETPRI_LAZY = 0x1, /* Avoid setting AST flags or sending IPIs */ |
174 | }); |
175 | |
176 | /* Set the current scheduled priority */ |
177 | extern void set_sched_pri( |
178 | thread_t thread, |
179 | int16_t priority, |
180 | set_sched_pri_options_t options); |
181 | |
182 | /* Set base priority of the specified thread */ |
183 | extern void sched_set_thread_base_priority( |
184 | thread_t thread, |
185 | int priority); |
186 | |
187 | /* Set absolute base priority of the specified thread */ |
188 | extern void sched_set_kernel_thread_priority( |
189 | thread_t thread, |
190 | int priority); |
191 | |
192 | |
193 | /* Set the thread's true scheduling mode */ |
194 | extern void sched_set_thread_mode(thread_t thread, |
195 | sched_mode_t mode); |
196 | |
197 | /* |
198 | * Set the thread's scheduling mode taking into account that the thread may have |
199 | * been demoted. |
200 | * */ |
201 | extern void sched_set_thread_mode_user(thread_t thread, |
202 | sched_mode_t mode); |
203 | |
204 | /* |
205 | * Get the thread's scheduling mode taking into account that the thread may have |
206 | * been demoted. |
207 | * */ |
208 | extern sched_mode_t sched_get_thread_mode_user(thread_t thread); |
209 | |
210 | |
211 | /* Demote the true scheduler mode */ |
212 | extern void sched_thread_mode_demote(thread_t thread, |
213 | uint32_t reason); |
214 | /* Un-demote the true scheduler mode */ |
215 | extern void sched_thread_mode_undemote(thread_t thread, |
216 | uint32_t reason); |
217 | /* Check for a specific demotion */ |
218 | extern bool sched_thread_mode_has_demotion(thread_t thread, |
219 | uint32_t reason); |
220 | |
221 | extern void sched_thread_promote_reason(thread_t thread, uint32_t reason, uintptr_t trace_obj); |
222 | extern void sched_thread_unpromote_reason(thread_t thread, uint32_t reason, uintptr_t trace_obj); |
223 | |
224 | /* Re-evaluate base priority of thread (thread locked) */ |
225 | void thread_recompute_priority(thread_t thread); |
226 | |
227 | /* Re-evaluate scheduled priority of thread (thread locked) */ |
228 | extern void thread_recompute_sched_pri( |
229 | thread_t thread, |
230 | set_sched_pri_options_t options); |
231 | |
232 | /* Periodic scheduler activity */ |
233 | extern void sched_init_thread(void); |
234 | |
235 | /* Perform sched_tick housekeeping activities */ |
236 | extern boolean_t can_update_priority( |
237 | thread_t thread); |
238 | |
239 | extern void update_priority( |
240 | thread_t thread); |
241 | |
242 | extern void lightweight_update_priority( |
243 | thread_t thread); |
244 | |
245 | extern void sched_default_quantum_expire(thread_t thread); |
246 | |
247 | /* Idle processor thread continuation */ |
248 | extern void idle_thread( |
249 | void* parameter, |
250 | wait_result_t result); |
251 | |
252 | extern kern_return_t idle_thread_create( |
253 | processor_t processor); |
254 | |
255 | /* Continuation return from syscall */ |
256 | extern void thread_syscall_return( |
257 | kern_return_t ret); |
258 | |
259 | /* Context switch */ |
260 | extern wait_result_t thread_block_reason( |
261 | thread_continue_t continuation, |
262 | void *parameter, |
263 | ast_t reason); |
264 | |
265 | __options_decl(sched_options_t, uint32_t, { |
266 | SCHED_NONE = 0x0, |
267 | SCHED_TAILQ = 0x1, |
268 | SCHED_HEADQ = 0x2, |
269 | SCHED_PREEMPT = 0x4, |
270 | SCHED_REBALANCE = 0x8, |
271 | }); |
272 | |
273 | /* Reschedule thread for execution */ |
274 | extern void thread_setrun( |
275 | thread_t thread, |
276 | sched_options_t options); |
277 | |
278 | extern processor_set_t task_choose_pset( |
279 | task_t task); |
280 | |
281 | /* Bind the current thread to a particular processor */ |
282 | extern processor_t thread_bind( |
283 | processor_t processor); |
284 | |
285 | extern void thread_bind_during_wakeup( |
286 | thread_t thread, |
287 | processor_t processor); |
288 | |
289 | extern void thread_unbind_after_queue_shutdown( |
290 | thread_t thread, |
291 | processor_t processor); |
292 | |
293 | extern bool pset_has_stealable_threads( |
294 | processor_set_t pset); |
295 | |
296 | extern bool pset_has_stealable_rt_threads( |
297 | processor_set_t pset); |
298 | |
299 | extern processor_set_t choose_starting_pset( |
300 | pset_node_t node, |
301 | thread_t thread, |
302 | processor_t *processor_hint); |
303 | |
304 | extern int pset_available_cpu_count( |
305 | processor_set_t pset); |
306 | |
307 | extern bool pset_is_recommended( |
308 | processor_set_t pset); |
309 | |
310 | extern pset_node_t sched_choose_node( |
311 | thread_t thread); |
312 | |
313 | /* Choose the best processor to run a thread */ |
314 | extern processor_t choose_processor( |
315 | processor_set_t pset, |
316 | processor_t processor, |
317 | thread_t thread); |
318 | |
319 | extern bool sched_SMT_balance( |
320 | processor_t processor, |
321 | processor_set_t pset); |
322 | |
323 | extern void thread_quantum_init( |
324 | thread_t thread, |
325 | uint64_t now); |
326 | |
327 | |
328 | extern void run_queue_init( |
329 | run_queue_t runq); |
330 | |
331 | extern thread_t run_queue_dequeue( |
332 | run_queue_t runq, |
333 | sched_options_t options); |
334 | |
335 | extern boolean_t run_queue_enqueue( |
336 | run_queue_t runq, |
337 | thread_t thread, |
338 | sched_options_t options); |
339 | |
340 | extern void run_queue_remove( |
341 | run_queue_t runq, |
342 | thread_t thread); |
343 | |
344 | extern thread_t run_queue_peek( |
345 | run_queue_t runq); |
346 | |
347 | struct sched_update_scan_context { |
348 | uint64_t earliest_bg_make_runnable_time; |
349 | uint64_t earliest_normal_make_runnable_time; |
350 | uint64_t earliest_rt_make_runnable_time; |
351 | uint64_t sched_tick_last_abstime; |
352 | }; |
353 | typedef struct sched_update_scan_context *sched_update_scan_context_t; |
354 | |
355 | extern void sched_rtlocal_runq_scan(sched_update_scan_context_t scan_context); |
356 | |
357 | extern void sched_pset_made_schedulable( |
358 | processor_t processor, |
359 | processor_set_t pset, |
360 | boolean_t drop_lock); |
361 | |
362 | extern void sched_cpu_init_completed(void); |
363 | |
364 | /* |
365 | * Enum to define various events which need IPIs. The IPI policy |
366 | * engine decides what kind of IPI to use based on destination |
367 | * processor state, thread and one of the following scheduling events. |
368 | */ |
369 | typedef enum { |
370 | SCHED_IPI_EVENT_BOUND_THR = 0x1, |
371 | SCHED_IPI_EVENT_PREEMPT = 0x2, |
372 | SCHED_IPI_EVENT_SMT_REBAL = 0x3, |
373 | SCHED_IPI_EVENT_SPILL = 0x4, |
374 | SCHED_IPI_EVENT_REBALANCE = 0x5, |
375 | SCHED_IPI_EVENT_RT_PREEMPT = 0x6, |
376 | } sched_ipi_event_t; |
377 | |
378 | |
379 | /* Enum to define various IPI types used by the scheduler */ |
380 | typedef enum { |
381 | SCHED_IPI_NONE = 0x0, |
382 | SCHED_IPI_IMMEDIATE = 0x1, |
383 | SCHED_IPI_IDLE = 0x2, |
384 | SCHED_IPI_DEFERRED = 0x3, |
385 | } sched_ipi_type_t; |
386 | |
387 | /* The IPI policy engine behaves in the following manner: |
388 | * - All scheduler events which need an IPI invoke sched_ipi_action() with |
389 | * the appropriate destination processor, thread and event. |
390 | * - sched_ipi_action() performs basic checks, invokes the scheduler specific |
391 | * ipi_policy routine and sets pending_AST bits based on the result. |
392 | * - Once the pset lock is dropped, the scheduler invokes sched_ipi_perform() |
393 | * routine which actually sends the appropriate IPI to the destination core. |
394 | */ |
395 | extern sched_ipi_type_t sched_ipi_action(processor_t dst, thread_t thread, sched_ipi_event_t event); |
396 | extern void sched_ipi_perform(processor_t dst, sched_ipi_type_t ipi); |
397 | |
398 | /* sched_ipi_policy() is the global default IPI policy for all schedulers */ |
399 | extern sched_ipi_type_t sched_ipi_policy(processor_t dst, thread_t thread, |
400 | boolean_t dst_idle, sched_ipi_event_t event); |
401 | |
402 | /* sched_ipi_deferred_policy() is the global default deferred IPI policy for all schedulers */ |
403 | extern sched_ipi_type_t sched_ipi_deferred_policy(processor_set_t pset, |
404 | processor_t dst, thread_t thread, sched_ipi_event_t event); |
405 | |
406 | #if defined(CONFIG_SCHED_TIMESHARE_CORE) |
407 | |
408 | extern boolean_t thread_update_add_thread(thread_t thread); |
409 | extern void thread_update_process_threads(void); |
410 | extern boolean_t runq_scan(run_queue_t runq, sched_update_scan_context_t scan_context); |
411 | |
412 | #if CONFIG_SCHED_CLUTCH |
413 | extern boolean_t sched_clutch_timeshare_scan(queue_t thread_queue, uint16_t count, sched_update_scan_context_t scan_context); |
414 | #endif /* CONFIG_SCHED_CLUTCH */ |
415 | |
416 | extern void sched_timeshare_init(void); |
417 | extern void sched_timeshare_timebase_init(void); |
418 | extern void sched_timeshare_maintenance_continue(void); |
419 | |
420 | extern boolean_t priority_is_urgent(int priority); |
421 | extern uint32_t sched_timeshare_initial_quantum_size(thread_t thread); |
422 | |
423 | extern int sched_compute_timeshare_priority(thread_t thread); |
424 | |
425 | #endif /* CONFIG_SCHED_TIMESHARE_CORE */ |
426 | |
427 | /* Remove thread from its run queue */ |
428 | extern boolean_t thread_run_queue_remove(thread_t thread); |
429 | thread_t thread_run_queue_remove_for_handoff(thread_t thread); |
430 | |
431 | /* Put a thread back in the run queue after being yanked */ |
432 | extern void thread_run_queue_reinsert(thread_t thread, sched_options_t options); |
433 | |
434 | extern void thread_timer_expire( |
435 | void *thread, |
436 | void *p1); |
437 | |
438 | extern bool thread_is_eager_preempt(thread_t thread); |
439 | |
440 | extern boolean_t sched_generic_direct_dispatch_to_idle_processors; |
441 | |
442 | /* Set the maximum interrupt level for the thread */ |
443 | __private_extern__ wait_interrupt_t thread_interrupt_level( |
444 | wait_interrupt_t interruptible); |
445 | |
446 | __private_extern__ wait_result_t thread_mark_wait_locked( |
447 | thread_t thread, |
448 | wait_interrupt_t interruptible); |
449 | |
450 | /* Wake up locked thread directly, passing result */ |
451 | __private_extern__ kern_return_t clear_wait_internal( |
452 | thread_t thread, |
453 | wait_result_t result); |
454 | |
455 | struct sched_statistics { |
456 | uint32_t csw_count; |
457 | uint32_t preempt_count; |
458 | uint32_t preempted_rt_count; |
459 | uint32_t preempted_by_rt_count; |
460 | uint32_t rt_sched_count; |
461 | uint32_t interrupt_count; |
462 | uint32_t ipi_count; |
463 | uint32_t timer_pop_count; |
464 | uint32_t idle_transitions; |
465 | uint32_t quantum_timer_expirations; |
466 | }; |
467 | PERCPU_DECL(struct sched_statistics, sched_stats); |
468 | extern bool sched_stats_active; |
469 | |
470 | extern void sched_stats_handle_csw( |
471 | processor_t processor, |
472 | int reasons, |
473 | int selfpri, |
474 | int otherpri); |
475 | |
476 | extern void sched_stats_handle_runq_change( |
477 | struct runq_stats *stats, |
478 | int old_count); |
479 | |
480 | #define SCHED_STATS_INC(field) \ |
481 | MACRO_BEGIN \ |
482 | if (__improbable(sched_stats_active)) { \ |
483 | PERCPU_GET(sched_stats)->field++; \ |
484 | } \ |
485 | MACRO_END |
486 | |
487 | #if DEBUG |
488 | |
489 | #define SCHED_STATS_CSW(processor, reasons, selfpri, otherpri) \ |
490 | MACRO_BEGIN \ |
491 | if (__improbable(sched_stats_active)) { \ |
492 | sched_stats_handle_csw((processor), \ |
493 | (reasons), (selfpri), (otherpri)); \ |
494 | } \ |
495 | MACRO_END |
496 | |
497 | |
498 | #define SCHED_STATS_RUNQ_CHANGE(stats, old_count) \ |
499 | MACRO_BEGIN \ |
500 | if (__improbable(sched_stats_active)) { \ |
501 | sched_stats_handle_runq_change((stats), (old_count)); \ |
502 | } \ |
503 | MACRO_END |
504 | |
505 | #else /* DEBUG */ |
506 | |
507 | #define SCHED_STATS_CSW(processor, reasons, selfpri, otherpri) do { }while(0) |
508 | #define SCHED_STATS_RUNQ_CHANGE(stats, old_count) do { }while(0) |
509 | |
510 | #endif /* DEBUG */ |
511 | |
512 | extern uint32_t sched_debug_flags; |
513 | #define SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS 0x00000001 |
514 | #define SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS 0x00000002 |
515 | #define SCHED_DEBUG_FLAG_AST_CHECK_TRACEPOINTS 0x00000004 |
516 | |
517 | #define SCHED_DEBUG_PLATFORM_KERNEL_DEBUG_CONSTANT(...) \ |
518 | MACRO_BEGIN \ |
519 | if (__improbable(sched_debug_flags & \ |
520 | SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS)) { \ |
521 | KERNEL_DEBUG_CONSTANT(__VA_ARGS__); \ |
522 | } \ |
523 | MACRO_END |
524 | |
525 | #define SCHED_DEBUG_CHOOSE_PROCESSOR_KERNEL_DEBUG_CONSTANT_IST(...) \ |
526 | MACRO_BEGIN \ |
527 | if (__improbable(sched_debug_flags & \ |
528 | SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS)) { \ |
529 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, __VA_ARGS__); \ |
530 | } \ |
531 | MACRO_END |
532 | |
533 | #define SCHED_DEBUG_AST_CHECK_KDBG_RELEASE(...) \ |
534 | MACRO_BEGIN \ |
535 | if (__improbable(sched_debug_flags & \ |
536 | SCHED_DEBUG_FLAG_AST_CHECK_TRACEPOINTS)) { \ |
537 | KDBG_RELEASE(__VA_ARGS__); \ |
538 | } \ |
539 | MACRO_END |
540 | |
541 | |
542 | /* Tells if there are "active" RT threads in the system (provided by CPU PM) */ |
543 | extern void active_rt_threads( |
544 | boolean_t active); |
545 | |
546 | /* Returns the perfcontrol attribute for the thread */ |
547 | extern perfcontrol_class_t thread_get_perfcontrol_class( |
548 | thread_t thread); |
549 | |
550 | /* Generic routine for Non-AMP schedulers to calculate parallelism */ |
551 | extern uint32_t sched_qos_max_parallelism(int qos, uint64_t options); |
552 | |
553 | extern void check_monotonic_time(uint64_t ctime); |
554 | |
555 | #endif /* MACH_KERNEL_PRIVATE */ |
556 | |
557 | __BEGIN_DECLS |
558 | |
559 | #ifdef XNU_KERNEL_PRIVATE |
560 | |
561 | extern void thread_bind_cluster_type(thread_t, char cluster_type, bool soft_bind); |
562 | |
563 | __options_decl(thread_bind_option_t, uint64_t, { |
564 | /* Unbind a previously cluster bound thread */ |
565 | THREAD_UNBIND = 0x1, |
566 | /* |
567 | * Soft bind the thread to the cluster; soft binding means the thread will be |
568 | * moved to an available cluster if the bound cluster is de-recommended/offline. |
569 | */ |
570 | THREAD_BIND_SOFT = 0x2, |
571 | /* |
572 | * Bind thread to the cluster only if it is eligible to run on that cluster. If |
573 | * the thread is not eligible to run on the cluster, thread_bind_cluster_id() |
574 | * returns KERN_INVALID_POLICY. |
575 | */ |
576 | THREAD_BIND_ELIGIBLE_ONLY = 0x4, |
577 | }); |
578 | extern kern_return_t thread_bind_cluster_id(thread_t thread, uint32_t cluster_id, thread_bind_option_t options); |
579 | |
580 | extern int sched_get_rt_n_backup_processors(void); |
581 | extern void sched_set_rt_n_backup_processors(int n); |
582 | |
583 | extern int sched_get_rt_deadline_epsilon(void); |
584 | extern void sched_set_rt_deadline_epsilon(int new_epsilon_us); |
585 | |
586 | /* Toggles a global override to turn off CPU Throttling */ |
587 | extern void sys_override_cpu_throttle(boolean_t enable_override); |
588 | |
589 | extern int sched_get_powered_cores(void); |
590 | extern void sched_set_powered_cores(int n); |
591 | |
592 | /* |
593 | ****************** Only exported until BSD stops using ******************** |
594 | */ |
595 | |
596 | extern void thread_vm_bind_group_add(void); |
597 | |
598 | /* Wake up thread directly, passing result */ |
599 | extern kern_return_t clear_wait( |
600 | thread_t thread, |
601 | wait_result_t result); |
602 | |
603 | /* Start thread running */ |
604 | extern void thread_bootstrap_return(void) __attribute__((noreturn)); |
605 | |
606 | /* Return from exception (BSD-visible interface) */ |
607 | extern void thread_exception_return(void) __dead2; |
608 | |
609 | #define SCHED_STRING_MAX_LENGTH (48) |
610 | /* String declaring the name of the current scheduler */ |
611 | extern char sched_string[SCHED_STRING_MAX_LENGTH]; |
612 | |
613 | __options_decl(thread_handoff_option_t, uint32_t, { |
614 | THREAD_HANDOFF_NONE = 0, |
615 | THREAD_HANDOFF_SETRUN_NEEDED = 0x1, |
616 | }); |
617 | |
618 | /* Remove thread from its run queue */ |
619 | thread_t thread_prepare_for_handoff(thread_t thread, thread_handoff_option_t option); |
620 | |
621 | /* Attempt to context switch to a specific runnable thread */ |
622 | extern wait_result_t thread_handoff_deallocate(thread_t thread, thread_handoff_option_t option); |
623 | |
624 | __attribute__((nonnull(2))) |
625 | extern void thread_handoff_parameter(thread_t thread, |
626 | thread_continue_t continuation, void *parameter, thread_handoff_option_t) __dead2; |
627 | |
628 | extern struct waitq *assert_wait_queue(event_t event); |
629 | |
630 | extern kern_return_t thread_wakeup_one_with_pri(event_t event, int priority); |
631 | |
632 | extern thread_t thread_wakeup_identify(event_t event, int priority); |
633 | |
634 | /* |
635 | * sched_cond_t: |
636 | * |
637 | * A atomic condition variable used to synchronize wake/block operations on threads. |
638 | * Bits defined below are reserved for use by sched_prim. Remaining |
639 | * bits may be used by caller for additional synchronization semantics. |
640 | */ |
641 | __options_decl(sched_cond_t, uint32_t, { |
642 | SCHED_COND_INIT = 0x0000, /* initialize all bits to zero (inactive and not awoken) */ |
643 | SCHED_COND_ACTIVE = 0x0001, /* target thread is active */ |
644 | SCHED_COND_WAKEUP = 0x0002 /* wakeup has been issued for target thread */ |
645 | }); |
646 | typedef _Atomic sched_cond_t sched_cond_atomic_t; |
647 | |
648 | /* |
649 | * sched_cond_init: |
650 | * |
651 | * Initialize an atomic condition variable. Note that this does not occur atomically and should be |
652 | * performed during thread initialization, before the condition is observable by other threads. |
653 | */ |
654 | extern void sched_cond_init( |
655 | sched_cond_atomic_t *cond); |
656 | |
657 | /* |
658 | * sched_cond_signal: |
659 | * |
660 | * Wakeup the specified thread if it is waiting on this event and it has not already been issued a wakeup. |
661 | * |
662 | * parameters: |
663 | * thread thread to awaken |
664 | * cond atomic condition variable |
665 | */ |
666 | extern kern_return_t sched_cond_signal( |
667 | sched_cond_atomic_t *cond, |
668 | thread_t thread); |
669 | |
670 | /* |
671 | * sched_cond_wait_parameter: |
672 | * |
673 | * Assert wait and block on cond if no wakeup has been issued. |
674 | * If a wakeup has been issued on cond since the last `sched_cond_ack`, clear_wait and |
675 | * return `THREAD_AWAKENED`. |
676 | * |
677 | * `sched_cond_wait_parameter` must be paired with `sched_cond_ack`. |
678 | * |
679 | * NOTE: `continuation` will only be jumped to if a wakeup has not been issued |
680 | * |
681 | * parameters: |
682 | * cond atomic condition variable to synchronize on |
683 | * interruptible interruptible value to pass to assert_wait |
684 | * continuation continuation if block succeeds |
685 | * parameter |
686 | */ |
687 | extern wait_result_t sched_cond_wait_parameter( |
688 | sched_cond_atomic_t *cond, |
689 | wait_interrupt_t interruptible, |
690 | thread_continue_t continuation, |
691 | void *parameter); |
692 | |
693 | /* |
694 | * sched_cond_wait: |
695 | * |
696 | * Assert wait and block on cond if no wakeup has been issued. |
697 | * If a wakeup has been issued on cond since the last `sched_cond_ack`, clear_wait and |
698 | * return `THREAD_AWAKENED`. |
699 | * |
700 | * `sched_cond_wait` must be paired with `sched_cond_ack`. |
701 | * |
702 | * NOTE: `continuation` will only be jumped to if a wakeup has not been issued |
703 | * |
704 | * parameters: |
705 | * cond atomic condition variable to synchronize on |
706 | * interruptible interruptible value to pass to assert_wait |
707 | * continuation continuation if block succeeds |
708 | */ |
709 | extern wait_result_t sched_cond_wait( |
710 | sched_cond_atomic_t *cond, |
711 | wait_interrupt_t interruptible, |
712 | thread_continue_t continuation); |
713 | |
714 | /* |
715 | * sched_cond_ack: |
716 | * |
717 | * Acknowledge an issued wakeup by clearing WAKEUP and setting ACTIVE (via XOR). |
718 | * It is the callers responsibility to ensure that the ACTIVE bit is always low prior to calling |
719 | * (i.e. by calling `sched_cond_wait` prior to any rerun or block). |
720 | * Synchronization schemes that allow for WAKEUP bit to be reset prior to wakeup |
721 | * (e.g. a cancellation mechanism) should check that WAKEUP was indeed cleared. |
722 | * |
723 | * e.g. |
724 | * ``` |
725 | * if (sched_cond_ack(&my_state) & SCHED_THREAD_WAKEUP) { |
726 | * // WAKEUP bit was no longer set by the time this thread woke up |
727 | * do_cancellation_policy(); |
728 | * } |
729 | * ``` |
730 | * |
731 | * parameters: |
732 | * cond: atomic condition variable |
733 | */ |
734 | extern sched_cond_t sched_cond_ack( |
735 | sched_cond_atomic_t *cond); |
736 | |
737 | #endif /* XNU_KERNEL_PRIVATE */ |
738 | |
739 | #ifdef KERNEL_PRIVATE |
740 | /* Set pending block hint for a particular object before we go into a wait state */ |
741 | extern void thread_set_pending_block_hint( |
742 | thread_t thread, |
743 | block_hint_t block_hint); |
744 | |
745 | #define QOS_PARALLELISM_COUNT_LOGICAL 0x1 |
746 | #define QOS_PARALLELISM_REALTIME 0x2 |
747 | #define QOS_PARALLELISM_CLUSTER_SHARED_RESOURCE 0x4 |
748 | |
749 | extern uint32_t qos_max_parallelism(int qos, uint64_t options); |
750 | #endif /* KERNEL_PRIVATE */ |
751 | |
752 | #if XNU_KERNEL_PRIVATE |
753 | extern void thread_yield_with_continuation( |
754 | thread_continue_t continuation, |
755 | void *parameter) __dead2; |
756 | #endif |
757 | |
758 | /* Context switch */ |
759 | extern wait_result_t thread_block( |
760 | thread_continue_t continuation); |
761 | |
762 | extern wait_result_t thread_block_parameter( |
763 | thread_continue_t continuation, |
764 | void *parameter); |
765 | |
766 | /* Declare thread will wait on a particular event */ |
767 | extern wait_result_t assert_wait( |
768 | event_t event, |
769 | wait_interrupt_t interruptible); |
770 | |
771 | /* Assert that the thread intends to wait with a timeout */ |
772 | extern wait_result_t assert_wait_timeout( |
773 | event_t event, |
774 | wait_interrupt_t interruptible, |
775 | uint32_t interval, |
776 | uint32_t scale_factor); |
777 | |
778 | /* Assert that the thread intends to wait with an urgency, timeout and leeway */ |
779 | extern wait_result_t assert_wait_timeout_with_leeway( |
780 | event_t event, |
781 | wait_interrupt_t interruptible, |
782 | wait_timeout_urgency_t urgency, |
783 | uint32_t interval, |
784 | uint32_t leeway, |
785 | uint32_t scale_factor); |
786 | |
787 | extern wait_result_t assert_wait_deadline( |
788 | event_t event, |
789 | wait_interrupt_t interruptible, |
790 | uint64_t deadline); |
791 | |
792 | /* Assert that the thread intends to wait with an urgency, deadline, and leeway */ |
793 | extern wait_result_t assert_wait_deadline_with_leeway( |
794 | event_t event, |
795 | wait_interrupt_t interruptible, |
796 | wait_timeout_urgency_t urgency, |
797 | uint64_t deadline, |
798 | uint64_t leeway); |
799 | |
800 | |
801 | /* Wake up thread (or threads) waiting on a particular event */ |
802 | extern kern_return_t thread_wakeup_prim( |
803 | event_t event, |
804 | boolean_t one_thread, |
805 | wait_result_t result); |
806 | |
807 | #define thread_wakeup(x) \ |
808 | thread_wakeup_prim((x), FALSE, THREAD_AWAKENED) |
809 | #define thread_wakeup_with_result(x, z) \ |
810 | thread_wakeup_prim((x), FALSE, (z)) |
811 | #define thread_wakeup_one(x) \ |
812 | thread_wakeup_prim((x), TRUE, THREAD_AWAKENED) |
813 | |
814 | /* Wakeup the specified thread if it is waiting on this event */ |
815 | extern kern_return_t thread_wakeup_thread(event_t event, thread_t thread); |
816 | |
817 | extern boolean_t preemption_enabled(void); |
818 | |
819 | #ifdef MACH_KERNEL_PRIVATE |
820 | |
821 | #if !defined(CONFIG_SCHED_TRADITIONAL) && !defined(CONFIG_SCHED_PROTO) && !defined(CONFIG_SCHED_GRRR) && !defined(CONFIG_SCHED_MULTIQ) && !defined(CONFIG_SCHED_CLUTCH) && !defined(CONFIG_SCHED_EDGE) |
822 | #error Enable at least one scheduler algorithm in osfmk/conf/MASTER.XXX |
823 | #endif |
824 | |
825 | /* |
826 | * The scheduling policy is fixed at compile-time, in order to save the performance |
827 | * cost of function pointer indirection that we would otherwise pay each time when |
828 | * making a policy-specific callout. |
829 | */ |
830 | #if __AMP__ |
831 | |
832 | #if CONFIG_SCHED_EDGE |
833 | extern const struct sched_dispatch_table sched_edge_dispatch; |
834 | #define SCHED(f) (sched_edge_dispatch.f) |
835 | #else /* CONFIG_SCHED_EDGE */ |
836 | extern const struct sched_dispatch_table sched_amp_dispatch; |
837 | #define SCHED(f) (sched_amp_dispatch.f) |
838 | #endif /* CONFIG_SCHED_EDGE */ |
839 | |
840 | #else /* __AMP__ */ |
841 | |
842 | #if CONFIG_SCHED_CLUTCH |
843 | extern const struct sched_dispatch_table sched_clutch_dispatch; |
844 | #define SCHED(f) (sched_clutch_dispatch.f) |
845 | #else /* CONFIG_SCHED_CLUTCH */ |
846 | extern const struct sched_dispatch_table sched_dualq_dispatch; |
847 | #define SCHED(f) (sched_dualq_dispatch.f) |
848 | #endif /* CONFIG_SCHED_CLUTCH */ |
849 | |
850 | #endif /* __AMP__ */ |
851 | |
852 | struct sched_dispatch_table { |
853 | const char *sched_name; |
854 | void (*init)(void); /* Init global state */ |
855 | void (*timebase_init)(void); /* Timebase-dependent initialization */ |
856 | void (*processor_init)(processor_t processor); /* Per-processor scheduler init */ |
857 | void (*pset_init)(processor_set_t pset); /* Per-processor set scheduler init */ |
858 | |
859 | void (*maintenance_continuation)(void); /* Function called regularly */ |
860 | |
861 | /* |
862 | * Choose a thread of greater or equal priority from the per-processor |
863 | * runqueue for timeshare/fixed threads |
864 | */ |
865 | thread_t (*choose_thread)( |
866 | processor_t processor, |
867 | int priority, |
868 | ast_t reason); |
869 | |
870 | /* True if scheduler supports stealing threads for this pset */ |
871 | bool (*steal_thread_enabled)(processor_set_t pset); |
872 | |
873 | /* |
874 | * Steal a thread from another processor in the pset so that it can run |
875 | * immediately |
876 | */ |
877 | thread_t (*steal_thread)( |
878 | processor_set_t pset); |
879 | |
880 | /* |
881 | * Compute priority for a timeshare thread based on base priority. |
882 | */ |
883 | int (*compute_timeshare_priority)(thread_t thread); |
884 | |
885 | /* |
886 | * Pick the best node for a thread to run on. |
887 | */ |
888 | pset_node_t (*choose_node)( |
889 | thread_t thread); |
890 | |
891 | /* |
892 | * Pick the best processor for a thread (any kind of thread) to run on. |
893 | */ |
894 | processor_t (*choose_processor)( |
895 | processor_set_t pset, |
896 | processor_t processor, |
897 | thread_t thread); |
898 | /* |
899 | * Enqueue a timeshare or fixed priority thread onto the per-processor |
900 | * runqueue |
901 | */ |
902 | boolean_t (*processor_enqueue)( |
903 | processor_t processor, |
904 | thread_t thread, |
905 | sched_options_t options); |
906 | |
907 | /* Migrate threads away in preparation for processor shutdown */ |
908 | void (*processor_queue_shutdown)( |
909 | processor_t processor); |
910 | |
911 | /* Remove the specific thread from the per-processor runqueue */ |
912 | boolean_t (*processor_queue_remove)( |
913 | processor_t processor, |
914 | thread_t thread); |
915 | |
916 | /* |
917 | * Does the per-processor runqueue have any timeshare or fixed priority |
918 | * threads on it? Called without pset lock held, so should |
919 | * not assume immutability while executing. |
920 | */ |
921 | boolean_t (*processor_queue_empty)(processor_t processor); |
922 | |
923 | /* |
924 | * Would this priority trigger an urgent preemption if it's sitting |
925 | * on the per-processor runqueue? |
926 | */ |
927 | boolean_t (*priority_is_urgent)(int priority); |
928 | |
929 | /* |
930 | * Does the per-processor runqueue contain runnable threads that |
931 | * should cause the currently-running thread to be preempted? |
932 | */ |
933 | ast_t (*processor_csw_check)(processor_t processor); |
934 | |
935 | /* |
936 | * Does the per-processor runqueue contain a runnable thread |
937 | * of > or >= priority, as a preflight for choose_thread() or other |
938 | * thread selection |
939 | */ |
940 | boolean_t (*processor_queue_has_priority)(processor_t processor, |
941 | int priority, |
942 | boolean_t gte); |
943 | |
944 | /* Quantum size for the specified non-realtime thread. */ |
945 | uint32_t (*initial_quantum_size)(thread_t thread); |
946 | |
947 | /* Scheduler mode for a new thread */ |
948 | sched_mode_t (*initial_thread_sched_mode)(task_t parent_task); |
949 | |
950 | /* |
951 | * Is it safe to call update_priority, which may change a thread's |
952 | * runqueue or other state. This can be used to throttle changes |
953 | * to dynamic priority. |
954 | */ |
955 | boolean_t (*can_update_priority)(thread_t thread); |
956 | |
957 | /* |
958 | * Update both scheduled priority and other persistent state. |
959 | * Side effects may including migration to another processor's runqueue. |
960 | */ |
961 | void (*update_priority)(thread_t thread); |
962 | |
963 | /* Lower overhead update to scheduled priority and state. */ |
964 | void (*lightweight_update_priority)(thread_t thread); |
965 | |
966 | /* Callback for non-realtime threads when the quantum timer fires */ |
967 | void (*quantum_expire)(thread_t thread); |
968 | |
969 | /* |
970 | * Runnable threads on per-processor runqueue. Should only |
971 | * be used for relative comparisons of load between processors. |
972 | */ |
973 | int (*processor_runq_count)(processor_t processor); |
974 | |
975 | /* Aggregate runcount statistics for per-processor runqueue */ |
976 | uint64_t (*processor_runq_stats_count_sum)(processor_t processor); |
977 | |
978 | boolean_t (*processor_bound_count)(processor_t processor); |
979 | |
980 | void (*thread_update_scan)(sched_update_scan_context_t scan_context); |
981 | |
982 | /* Supports more than one pset */ |
983 | boolean_t multiple_psets_enabled; |
984 | /* Supports scheduler groups */ |
985 | boolean_t sched_groups_enabled; |
986 | |
987 | /* Supports avoid-processor */ |
988 | boolean_t avoid_processor_enabled; |
989 | |
990 | /* Returns true if this processor should avoid running this thread. */ |
991 | bool (*thread_avoid_processor)(processor_t processor, thread_t thread, ast_t reason); |
992 | |
993 | /* |
994 | * Invoked when a processor is about to choose the idle thread |
995 | * Used to send IPIs to a processor which would be preferred to be idle instead. |
996 | * Returns true if the current processor should anticipate a quick IPI reply back |
997 | * from another core. |
998 | * Called with pset lock held, returns with pset lock unlocked. |
999 | */ |
1000 | bool (*processor_balance)(processor_t processor, processor_set_t pset); |
1001 | rt_queue_t (*rt_runq)(processor_set_t pset); |
1002 | void (*rt_init)(processor_set_t pset); |
1003 | void (*rt_queue_shutdown)(processor_t processor); |
1004 | void (*rt_runq_scan)(sched_update_scan_context_t scan_context); |
1005 | int64_t (*rt_runq_count_sum)(void); |
1006 | thread_t (*rt_steal_thread)(processor_set_t pset, uint64_t earliest_deadline); |
1007 | |
1008 | uint32_t (*qos_max_parallelism)(int qos, uint64_t options); |
1009 | void (*check_spill)(processor_set_t pset, thread_t thread); |
1010 | sched_ipi_type_t (*ipi_policy)(processor_t dst, thread_t thread, boolean_t dst_idle, sched_ipi_event_t event); |
1011 | bool (*thread_should_yield)(processor_t processor, thread_t thread); |
1012 | |
1013 | /* Routine to update run counts */ |
1014 | uint32_t (*run_count_incr)(thread_t thread); |
1015 | uint32_t (*run_count_decr)(thread_t thread); |
1016 | |
1017 | /* Routine to update scheduling bucket for a thread */ |
1018 | void (*update_thread_bucket)(thread_t thread); |
1019 | |
1020 | /* Routine to inform the scheduler when a new pset becomes schedulable */ |
1021 | void (*pset_made_schedulable)(processor_t processor, processor_set_t pset, boolean_t drop_lock); |
1022 | #if CONFIG_THREAD_GROUPS |
1023 | /* Routine to inform the scheduler when CLPC changes a thread group recommendation */ |
1024 | void (*thread_group_recommendation_change)(struct thread_group *tg, cluster_type_t new_recommendation); |
1025 | #endif |
1026 | /* Routine to inform the scheduler when all CPUs have finished initializing */ |
1027 | void (*cpu_init_completed)(void); |
1028 | /* Routine to check if a thread is eligible to execute on a specific pset */ |
1029 | bool (*thread_eligible_for_pset)(thread_t thread, processor_set_t pset); |
1030 | }; |
1031 | |
1032 | #if defined(CONFIG_SCHED_TRADITIONAL) |
1033 | extern const struct sched_dispatch_table sched_traditional_dispatch; |
1034 | extern const struct sched_dispatch_table sched_traditional_with_pset_runqueue_dispatch; |
1035 | #endif |
1036 | |
1037 | #if defined(CONFIG_SCHED_MULTIQ) |
1038 | extern const struct sched_dispatch_table sched_multiq_dispatch; |
1039 | extern const struct sched_dispatch_table sched_dualq_dispatch; |
1040 | #if __AMP__ |
1041 | extern const struct sched_dispatch_table sched_amp_dispatch; |
1042 | #endif |
1043 | #endif |
1044 | |
1045 | #if defined(CONFIG_SCHED_PROTO) |
1046 | extern const struct sched_dispatch_table sched_proto_dispatch; |
1047 | #endif |
1048 | |
1049 | #if defined(CONFIG_SCHED_GRRR) |
1050 | extern const struct sched_dispatch_table sched_grrr_dispatch; |
1051 | #endif |
1052 | |
1053 | #if defined(CONFIG_SCHED_CLUTCH) |
1054 | extern const struct sched_dispatch_table sched_clutch_dispatch; |
1055 | #endif |
1056 | |
1057 | #if defined(CONFIG_SCHED_EDGE) |
1058 | extern const struct sched_dispatch_table sched_edge_dispatch; |
1059 | #endif |
1060 | |
1061 | extern void sched_set_max_unsafe_rt_quanta(int max); |
1062 | extern void sched_set_max_unsafe_fixed_quanta(int max); |
1063 | |
1064 | #endif /* MACH_KERNEL_PRIVATE */ |
1065 | |
1066 | __END_DECLS |
1067 | |
1068 | #endif /* _KERN_SCHED_PRIM_H_ */ |
1069 | |