1 | /* |
2 | * Copyright (c) 2000-2019 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * @OSF_COPYRIGHT@ |
30 | */ |
31 | /* |
32 | * Mach Operating System |
33 | * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University |
34 | * All Rights Reserved. |
35 | * |
36 | * Permission to use, copy, modify and distribute this software and its |
37 | * documentation is hereby granted, provided that both the copyright |
38 | * notice and this permission notice appear in all copies of the |
39 | * software, derivative works or modified versions, and any portions |
40 | * thereof, and that both notices appear in supporting documentation. |
41 | * |
42 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" |
43 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR |
44 | * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. |
45 | * |
46 | * Carnegie Mellon requests users of this software to return to |
47 | * |
48 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU |
49 | * School of Computer Science |
50 | * Carnegie Mellon University |
51 | * Pittsburgh PA 15213-3890 |
52 | * |
53 | * any improvements or extensions that they make and grant Carnegie Mellon |
54 | * the rights to redistribute these changes. |
55 | */ |
56 | /* |
57 | */ |
58 | /* |
59 | * File: priority.c |
60 | * Author: Avadis Tevanian, Jr. |
61 | * Date: 1986 |
62 | * |
63 | * Priority related scheduler bits. |
64 | */ |
65 | |
66 | #include <mach/boolean.h> |
67 | #include <mach/kern_return.h> |
68 | #include <mach/machine.h> |
69 | #include <kern/host.h> |
70 | #include <kern/mach_param.h> |
71 | #include <kern/sched.h> |
72 | #include <sys/kdebug.h> |
73 | #include <kern/spl.h> |
74 | #include <kern/thread.h> |
75 | #include <kern/processor.h> |
76 | #include <kern/ledger.h> |
77 | #include <kern/monotonic.h> |
78 | #include <machine/machparam.h> |
79 | #include <kern/machine.h> |
80 | #include <kern/policy_internal.h> |
81 | #include <kern/sched_clutch.h> |
82 | |
83 | #ifdef CONFIG_MACH_APPROXIMATE_TIME |
84 | #include <machine/commpage.h> /* for commpage_update_mach_approximate_time */ |
85 | #endif |
86 | |
87 | /* |
88 | * thread_quantum_expire: |
89 | * |
90 | * Recalculate the quantum and priority for a thread. |
91 | * |
92 | * Called at splsched. |
93 | */ |
94 | |
95 | void |
96 | thread_quantum_expire( |
97 | timer_call_param_t p0, |
98 | timer_call_param_t p1) |
99 | { |
100 | processor_t processor = p0; |
101 | thread_t thread = p1; |
102 | ast_t preempt; |
103 | uint64_t ctime; |
104 | |
105 | assert(processor == current_processor()); |
106 | assert(thread == current_thread()); |
107 | |
108 | KDBG_RELEASE(MACHDBG_CODE( |
109 | DBG_MACH_SCHED, MACH_SCHED_QUANTUM_EXPIRED) | DBG_FUNC_START); |
110 | |
111 | SCHED_STATS_INC(quantum_timer_expirations); |
112 | |
113 | /* |
114 | * We bill CPU time to both the individual thread and its task. |
115 | * |
116 | * Because this balance adjustment could potentially attempt to wake this |
117 | * very thread, we must credit the ledger before taking the thread lock. |
118 | * The ledger pointers are only manipulated by the thread itself at the ast |
119 | * boundary. |
120 | * |
121 | * TODO: This fails to account for the time between when the timer was |
122 | * armed and when it fired. It should be based on the system_timer and |
123 | * running a timer_update operation here. |
124 | */ |
125 | ledger_credit(ledger: thread->t_ledger, entry: task_ledgers.cpu_time, amount: thread->quantum_remaining); |
126 | ledger_credit(ledger: thread->t_threadledger, entry: thread_ledgers.cpu_time, amount: thread->quantum_remaining); |
127 | if (thread->t_bankledger) { |
128 | ledger_credit(ledger: thread->t_bankledger, entry: bank_ledgers.cpu_time, |
129 | amount: (thread->quantum_remaining - thread->t_deduct_bank_ledger_time)); |
130 | } |
131 | thread->t_deduct_bank_ledger_time = 0; |
132 | |
133 | struct recount_snap snap = { 0 }; |
134 | recount_snapshot(snap: &snap); |
135 | ctime = snap.rsn_time_mach; |
136 | check_monotonic_time(ctime); |
137 | #ifdef CONFIG_MACH_APPROXIMATE_TIME |
138 | commpage_update_mach_approximate_time(ctime); |
139 | #endif /* CONFIG_MACH_APPROXIMATE_TIME */ |
140 | |
141 | sched_update_pset_avg_execution_time(pset: processor->processor_set, delta: thread->quantum_remaining, curtime: ctime, sched_bucket: thread->th_sched_bucket); |
142 | |
143 | recount_switch_thread(snap: &snap, off_thread: thread, off_task: get_threadtask(thread)); |
144 | recount_log_switch_thread(snap: &snap); |
145 | |
146 | thread_lock(thread); |
147 | |
148 | /* |
149 | * We've run up until our quantum expiration, and will (potentially) |
150 | * continue without re-entering the scheduler, so update this now. |
151 | */ |
152 | processor->last_dispatch = ctime; |
153 | thread->last_run_time = ctime; |
154 | |
155 | /* |
156 | * Check for fail-safe trip. |
157 | */ |
158 | if ((thread->sched_mode == TH_MODE_REALTIME || thread->sched_mode == TH_MODE_FIXED) && |
159 | !(thread->sched_flags & TH_SFLAG_PROMOTED) && |
160 | !(thread->kern_promotion_schedpri != 0) && |
161 | !(thread->sched_flags & TH_SFLAG_PROMOTE_REASON_MASK) && |
162 | !(thread->options & TH_OPT_SYSTEM_CRITICAL)) { |
163 | uint64_t new_computation; |
164 | |
165 | new_computation = ctime - thread->computation_epoch; |
166 | new_computation += thread->computation_metered; |
167 | /* |
168 | * Remove any time spent handling interrupts outside of the thread's |
169 | * control. |
170 | */ |
171 | new_computation -= recount_current_thread_interrupt_time_mach() - thread->computation_interrupt_epoch; |
172 | |
173 | bool demote = false; |
174 | switch (thread->sched_mode) { |
175 | case TH_MODE_REALTIME: |
176 | if (new_computation > max_unsafe_rt_computation) { |
177 | thread->safe_release = ctime + sched_safe_rt_duration; |
178 | demote = true; |
179 | } |
180 | break; |
181 | case TH_MODE_FIXED: |
182 | if (new_computation > max_unsafe_fixed_computation) { |
183 | thread->safe_release = ctime + sched_safe_fixed_duration; |
184 | demote = true; |
185 | } |
186 | break; |
187 | default: |
188 | panic("unexpected mode: %d" , thread->sched_mode); |
189 | } |
190 | |
191 | if (demote) { |
192 | KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_FAILSAFE) | DBG_FUNC_NONE, |
193 | (uintptr_t)thread->sched_pri, (uintptr_t)thread->sched_mode, 0, 0, 0); |
194 | sched_thread_mode_demote(thread, TH_SFLAG_FAILSAFE); |
195 | } |
196 | } |
197 | |
198 | /* |
199 | * Recompute scheduled priority if appropriate. |
200 | */ |
201 | if (SCHED(can_update_priority)(thread)) { |
202 | SCHED(update_priority)(thread); |
203 | } else { |
204 | SCHED(lightweight_update_priority)(thread); |
205 | } |
206 | |
207 | if (thread->sched_mode != TH_MODE_REALTIME) { |
208 | SCHED(quantum_expire)(thread); |
209 | } |
210 | |
211 | /* |
212 | * This quantum is up, give this thread another. |
213 | */ |
214 | processor->first_timeslice = FALSE; |
215 | |
216 | thread_quantum_init(thread, now: ctime); |
217 | |
218 | timer_update(timer: &thread->runnable_timer, tstamp: ctime); |
219 | |
220 | processor->quantum_end = ctime + thread->quantum_remaining; |
221 | |
222 | /* |
223 | * Context switch check |
224 | * |
225 | * non-urgent flags don't affect kernel threads, so upgrade to urgent |
226 | * to ensure that rebalancing and non-recommendation kick in quickly. |
227 | */ |
228 | |
229 | ast_t check_reason = AST_QUANTUM; |
230 | if (get_threadtask(thread) == kernel_task) { |
231 | check_reason |= AST_URGENT; |
232 | } |
233 | |
234 | if ((preempt = csw_check(thread, processor, check_reason)) != AST_NONE) { |
235 | ast_on(reasons: preempt); |
236 | } |
237 | |
238 | /* |
239 | * AST_KEVENT does not send an IPI when setting the AST, |
240 | * to avoid waiting for the next context switch to propagate the AST, |
241 | * the AST is propagated here at quantum expiration. |
242 | */ |
243 | ast_propagate(thread); |
244 | |
245 | thread_unlock(thread); |
246 | |
247 | /* Now that the processor->thread_timer has been updated, evaluate to see if |
248 | * the workqueue quantum expired and set AST_KEVENT if it has */ |
249 | if (thread_get_tag(thread) & THREAD_TAG_WORKQUEUE) { |
250 | thread_evaluate_workqueue_quantum_expiry(thread); |
251 | } |
252 | |
253 | running_timer_enter(processor, timer: RUNNING_TIMER_QUANTUM, param: thread, |
254 | deadline: processor->quantum_end, now: ctime); |
255 | |
256 | /* Tell platform layer that we are still running this thread */ |
257 | thread_urgency_t urgency = thread_get_urgency(thread, NULL, NULL); |
258 | machine_thread_going_on_core(new_thread: thread, urgency, sched_latency: 0, same_pri_latency: 0, dispatch_time: ctime); |
259 | machine_switch_perfcontrol_state_update(event: QUANTUM_EXPIRY, timestamp: ctime, |
260 | flags: 0, thread); |
261 | |
262 | #if defined(CONFIG_SCHED_TIMESHARE_CORE) |
263 | sched_timeshare_consider_maintenance(ctime, false); |
264 | #endif /* CONFIG_SCHED_TIMESHARE_CORE */ |
265 | |
266 | #if __arm64__ |
267 | if (thread->sched_mode == TH_MODE_REALTIME) { |
268 | sched_consider_recommended_cores(ctime, thread); |
269 | } |
270 | #endif /* __arm64__ */ |
271 | |
272 | KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_QUANTUM_EXPIRED) | DBG_FUNC_END, preempt, 0, 0, 0, 0); |
273 | } |
274 | |
275 | /* |
276 | * sched_set_thread_base_priority: |
277 | * |
278 | * Set the base priority of the thread |
279 | * and reset its scheduled priority. |
280 | * |
281 | * This is the only path to change base_pri. |
282 | * |
283 | * Called with the thread locked. |
284 | */ |
285 | void |
286 | sched_set_thread_base_priority(thread_t thread, int priority) |
287 | { |
288 | assert(priority >= MINPRI); |
289 | uint64_t ctime = 0; |
290 | |
291 | if (thread->sched_mode == TH_MODE_REALTIME) { |
292 | assert((priority >= BASEPRI_RTQUEUES) && (priority <= MAXPRI)); |
293 | } else { |
294 | assert(priority < BASEPRI_RTQUEUES); |
295 | } |
296 | |
297 | int old_base_pri = thread->base_pri; |
298 | thread->req_base_pri = (int16_t)priority; |
299 | if (thread->sched_flags & TH_SFLAG_BASE_PRI_FROZEN) { |
300 | priority = MAX(priority, old_base_pri); |
301 | } |
302 | thread->base_pri = (int16_t)priority; |
303 | |
304 | if ((thread->state & TH_RUN) == TH_RUN) { |
305 | assert(thread->last_made_runnable_time != THREAD_NOT_RUNNABLE); |
306 | ctime = mach_approximate_time(); |
307 | thread->last_basepri_change_time = ctime; |
308 | } else { |
309 | assert(thread->last_basepri_change_time == THREAD_NOT_RUNNABLE); |
310 | assert(thread->last_made_runnable_time == THREAD_NOT_RUNNABLE); |
311 | } |
312 | |
313 | /* |
314 | * Currently the perfcontrol_attr depends on the base pri of the |
315 | * thread. Therefore, we use this function as the hook for the |
316 | * perfcontrol callout. |
317 | */ |
318 | if (thread == current_thread() && old_base_pri != priority) { |
319 | if (!ctime) { |
320 | ctime = mach_approximate_time(); |
321 | } |
322 | machine_switch_perfcontrol_state_update(event: PERFCONTROL_ATTR_UPDATE, |
323 | timestamp: ctime, PERFCONTROL_CALLOUT_WAKE_UNSAFE, thread); |
324 | } |
325 | #if !CONFIG_SCHED_CLUTCH |
326 | /* For the clutch scheduler, this operation is done in set_sched_pri() */ |
327 | SCHED(update_thread_bucket)(thread); |
328 | #endif /* !CONFIG_SCHED_CLUTCH */ |
329 | |
330 | thread_recompute_sched_pri(thread, options: SETPRI_DEFAULT); |
331 | } |
332 | |
333 | /* |
334 | * sched_set_kernel_thread_priority: |
335 | * |
336 | * Set the absolute base priority of the thread |
337 | * and reset its scheduled priority. |
338 | * |
339 | * Called with the thread unlocked. |
340 | */ |
341 | void |
342 | sched_set_kernel_thread_priority(thread_t thread, int new_priority) |
343 | { |
344 | spl_t s = splsched(); |
345 | |
346 | thread_lock(thread); |
347 | |
348 | assert(thread->sched_mode != TH_MODE_REALTIME); |
349 | assert(thread->effective_policy.thep_qos == THREAD_QOS_UNSPECIFIED); |
350 | |
351 | if (new_priority > thread->max_priority) { |
352 | new_priority = thread->max_priority; |
353 | } |
354 | #if !defined(XNU_TARGET_OS_OSX) |
355 | if (new_priority < MAXPRI_THROTTLE) { |
356 | new_priority = MAXPRI_THROTTLE; |
357 | } |
358 | #endif /* !defined(XNU_TARGET_OS_OSX) */ |
359 | |
360 | thread->importance = new_priority - thread->task_priority; |
361 | |
362 | sched_set_thread_base_priority(thread, priority: new_priority); |
363 | |
364 | thread_unlock(thread); |
365 | splx(s); |
366 | } |
367 | |
368 | /* |
369 | * thread_recompute_sched_pri: |
370 | * |
371 | * Reset the scheduled priority of the thread |
372 | * according to its base priority if the |
373 | * thread has not been promoted or depressed. |
374 | * |
375 | * This is the only way to push base_pri changes into sched_pri, |
376 | * or to recalculate the appropriate sched_pri after changing |
377 | * a promotion or depression. |
378 | * |
379 | * Called at splsched with the thread locked. |
380 | * |
381 | * TODO: Add an 'update urgency' flag to avoid urgency callouts on every rwlock operation |
382 | */ |
383 | void |
384 | thread_recompute_sched_pri(thread_t thread, set_sched_pri_options_t options) |
385 | { |
386 | uint32_t sched_flags = thread->sched_flags; |
387 | sched_mode_t sched_mode = thread->sched_mode; |
388 | |
389 | int16_t priority = thread->base_pri; |
390 | |
391 | if (sched_mode == TH_MODE_TIMESHARE) { |
392 | priority = (int16_t)SCHED(compute_timeshare_priority)(thread); |
393 | } |
394 | |
395 | if (sched_flags & TH_SFLAG_DEPRESS) { |
396 | /* thread_yield_internal overrides kernel mutex promotion */ |
397 | priority = DEPRESSPRI; |
398 | } else { |
399 | /* poll-depress is overridden by mutex promotion and promote-reasons */ |
400 | if ((sched_flags & TH_SFLAG_POLLDEPRESS)) { |
401 | priority = DEPRESSPRI; |
402 | } |
403 | |
404 | if (thread->kern_promotion_schedpri > 0) { |
405 | priority = MAX(priority, thread->kern_promotion_schedpri); |
406 | |
407 | if (sched_mode != TH_MODE_REALTIME) { |
408 | priority = MIN(priority, MAXPRI_PROMOTE); |
409 | } |
410 | } |
411 | |
412 | if (sched_flags & TH_SFLAG_PROMOTED) { |
413 | priority = MAX(priority, thread->promotion_priority); |
414 | |
415 | if (sched_mode != TH_MODE_REALTIME) { |
416 | priority = MIN(priority, MAXPRI_PROMOTE); |
417 | } |
418 | } |
419 | |
420 | if (sched_flags & TH_SFLAG_PROMOTE_REASON_MASK) { |
421 | if (sched_flags & TH_SFLAG_RW_PROMOTED) { |
422 | priority = MAX(priority, MINPRI_RWLOCK); |
423 | } |
424 | |
425 | if (sched_flags & TH_SFLAG_WAITQ_PROMOTED) { |
426 | priority = MAX(priority, MINPRI_WAITQ); |
427 | } |
428 | |
429 | if (sched_flags & TH_SFLAG_EXEC_PROMOTED) { |
430 | priority = MAX(priority, MINPRI_EXEC); |
431 | } |
432 | |
433 | if (sched_flags & TH_SFLAG_FLOOR_PROMOTED) { |
434 | priority = MAX(priority, MINPRI_FLOOR); |
435 | } |
436 | } |
437 | } |
438 | |
439 | set_sched_pri(thread, priority, options); |
440 | } |
441 | |
442 | void |
443 | sched_default_quantum_expire(thread_t thread __unused) |
444 | { |
445 | /* |
446 | * No special behavior when a timeshare, fixed, or realtime thread |
447 | * uses up its entire quantum |
448 | */ |
449 | } |
450 | |
451 | int smt_timeshare_enabled = 1; |
452 | int smt_sched_bonus_16ths = 8; |
453 | |
454 | #if defined(CONFIG_SCHED_TIMESHARE_CORE) |
455 | |
456 | /* |
457 | * lightweight_update_priority: |
458 | * |
459 | * Update the scheduled priority for |
460 | * a timesharing thread. |
461 | * |
462 | * Only for use on the current thread. |
463 | * |
464 | * Called with the thread locked. |
465 | */ |
466 | void |
467 | lightweight_update_priority(thread_t thread) |
468 | { |
469 | thread_assert_runq_null(thread); |
470 | assert(thread == current_thread()); |
471 | |
472 | if (thread->sched_mode == TH_MODE_TIMESHARE) { |
473 | int priority; |
474 | uint32_t delta; |
475 | |
476 | sched_tick_delta(thread, delta); |
477 | |
478 | /* |
479 | * Accumulate timesharing usage only |
480 | * during contention for processor |
481 | * resources. |
482 | */ |
483 | if (thread->pri_shift < INT8_MAX) { |
484 | if (thread_no_smt(thread) && smt_timeshare_enabled) { |
485 | thread->sched_usage += (delta + ((delta * smt_sched_bonus_16ths) >> 4)); |
486 | } else { |
487 | thread->sched_usage += delta; |
488 | } |
489 | } |
490 | |
491 | thread->cpu_delta += delta; |
492 | |
493 | #if CONFIG_SCHED_CLUTCH |
494 | /* |
495 | * Update the CPU usage for the thread group to which the thread belongs. |
496 | * The implementation assumes that the thread ran for the entire delta |
497 | * as part of the same thread group. |
498 | */ |
499 | sched_clutch_cpu_usage_update(thread, delta); |
500 | #endif /* CONFIG_SCHED_CLUTCH */ |
501 | |
502 | priority = sched_compute_timeshare_priority(thread); |
503 | |
504 | if (priority != thread->sched_pri) { |
505 | thread_recompute_sched_pri(thread, options: SETPRI_LAZY); |
506 | } |
507 | } |
508 | } |
509 | |
510 | /* |
511 | * Define shifts for simulating (5/8) ** n |
512 | * |
513 | * Shift structures for holding update shifts. Actual computation |
514 | * is usage = (usage >> shift1) +/- (usage >> abs(shift2)) where the |
515 | * +/- is determined by the sign of shift 2. |
516 | */ |
517 | |
518 | const struct shift_data sched_decay_shifts[SCHED_DECAY_TICKS] = { |
519 | { .shift1 = 1, .shift2 = 1 }, |
520 | { .shift1 = 1, .shift2 = 3 }, |
521 | { .shift1 = 1, .shift2 = -3 }, |
522 | { .shift1 = 2, .shift2 = -7 }, |
523 | { .shift1 = 3, .shift2 = 5 }, |
524 | { .shift1 = 3, .shift2 = -5 }, |
525 | { .shift1 = 4, .shift2 = -8 }, |
526 | { .shift1 = 5, .shift2 = 7 }, |
527 | { .shift1 = 5, .shift2 = -7 }, |
528 | { .shift1 = 6, .shift2 = -10 }, |
529 | { .shift1 = 7, .shift2 = 10 }, |
530 | { .shift1 = 7, .shift2 = -9 }, |
531 | { .shift1 = 8, .shift2 = -11 }, |
532 | { .shift1 = 9, .shift2 = 12 }, |
533 | { .shift1 = 9, .shift2 = -11 }, |
534 | { .shift1 = 10, .shift2 = -13 }, |
535 | { .shift1 = 11, .shift2 = 14 }, |
536 | { .shift1 = 11, .shift2 = -13 }, |
537 | { .shift1 = 12, .shift2 = -15 }, |
538 | { .shift1 = 13, .shift2 = 17 }, |
539 | { .shift1 = 13, .shift2 = -15 }, |
540 | { .shift1 = 14, .shift2 = -17 }, |
541 | { .shift1 = 15, .shift2 = 19 }, |
542 | { .shift1 = 16, .shift2 = 18 }, |
543 | { .shift1 = 16, .shift2 = -19 }, |
544 | { .shift1 = 17, .shift2 = 22 }, |
545 | { .shift1 = 18, .shift2 = 20 }, |
546 | { .shift1 = 18, .shift2 = -20 }, |
547 | { .shift1 = 19, .shift2 = 26 }, |
548 | { .shift1 = 20, .shift2 = 22 }, |
549 | { .shift1 = 20, .shift2 = -22 }, |
550 | { .shift1 = 21, .shift2 = -27 } |
551 | }; |
552 | |
553 | /* |
554 | * sched_compute_timeshare_priority: |
555 | * |
556 | * Calculate the timesharing priority based upon usage and load. |
557 | */ |
558 | extern int sched_pri_decay_band_limit; |
559 | |
560 | |
561 | /* Only use the decay floor logic on non-macOS and non-clutch schedulers */ |
562 | #if !defined(XNU_TARGET_OS_OSX) && !CONFIG_SCHED_CLUTCH |
563 | |
564 | int |
565 | sched_compute_timeshare_priority(thread_t thread) |
566 | { |
567 | int decay_amount; |
568 | int decay_limit = sched_pri_decay_band_limit; |
569 | |
570 | if (thread->base_pri > BASEPRI_FOREGROUND) { |
571 | decay_limit += (thread->base_pri - BASEPRI_FOREGROUND); |
572 | } |
573 | |
574 | if (thread->pri_shift == INT8_MAX) { |
575 | decay_amount = 0; |
576 | } else { |
577 | decay_amount = (thread->sched_usage >> thread->pri_shift); |
578 | } |
579 | |
580 | if (decay_amount > decay_limit) { |
581 | decay_amount = decay_limit; |
582 | } |
583 | |
584 | /* start with base priority */ |
585 | int priority = thread->base_pri - decay_amount; |
586 | |
587 | if (priority < MAXPRI_THROTTLE) { |
588 | if (get_threadtask(thread)->max_priority > MAXPRI_THROTTLE) { |
589 | priority = MAXPRI_THROTTLE; |
590 | } else if (priority < MINPRI_USER) { |
591 | priority = MINPRI_USER; |
592 | } |
593 | } else if (priority > MAXPRI_KERNEL) { |
594 | priority = MAXPRI_KERNEL; |
595 | } |
596 | |
597 | return priority; |
598 | } |
599 | |
600 | #else /* !defined(XNU_TARGET_OS_OSX) && !CONFIG_SCHED_CLUTCH */ |
601 | |
602 | int |
603 | sched_compute_timeshare_priority(thread_t thread) |
604 | { |
605 | /* start with base priority */ |
606 | int priority = thread->base_pri; |
607 | |
608 | if (thread->pri_shift != INT8_MAX) { |
609 | priority -= (thread->sched_usage >> thread->pri_shift); |
610 | } |
611 | |
612 | if (priority < MINPRI_USER) { |
613 | priority = MINPRI_USER; |
614 | } else if (priority > MAXPRI_KERNEL) { |
615 | priority = MAXPRI_KERNEL; |
616 | } |
617 | |
618 | return priority; |
619 | } |
620 | |
621 | #endif /* !defined(XNU_TARGET_OS_OSX) && !CONFIG_SCHED_CLUTCH */ |
622 | |
623 | /* |
624 | * can_update_priority |
625 | * |
626 | * Make sure we don't do re-dispatches more frequently than a scheduler tick. |
627 | * |
628 | * Called with the thread locked. |
629 | */ |
630 | boolean_t |
631 | can_update_priority( |
632 | thread_t thread) |
633 | { |
634 | if (sched_tick == thread->sched_stamp) { |
635 | return FALSE; |
636 | } else { |
637 | return TRUE; |
638 | } |
639 | } |
640 | |
641 | /* |
642 | * update_priority |
643 | * |
644 | * Perform housekeeping operations driven by scheduler tick. |
645 | * |
646 | * Called with the thread locked. |
647 | */ |
648 | void |
649 | update_priority( |
650 | thread_t thread) |
651 | { |
652 | uint32_t ticks, delta; |
653 | |
654 | ticks = sched_tick - thread->sched_stamp; |
655 | assert(ticks != 0); |
656 | |
657 | thread->sched_stamp += ticks; |
658 | |
659 | /* If requested, accelerate aging of sched_usage */ |
660 | if (sched_decay_usage_age_factor > 1) { |
661 | ticks *= sched_decay_usage_age_factor; |
662 | } |
663 | |
664 | /* |
665 | * Gather cpu usage data. |
666 | */ |
667 | sched_tick_delta(thread, delta); |
668 | if (ticks < SCHED_DECAY_TICKS) { |
669 | /* |
670 | * Accumulate timesharing usage only during contention for processor |
671 | * resources. Use the pri_shift from the previous tick window to |
672 | * determine if the system was in a contended state. |
673 | */ |
674 | if (thread->pri_shift < INT8_MAX) { |
675 | if (thread_no_smt(thread) && smt_timeshare_enabled) { |
676 | thread->sched_usage += (delta + ((delta * smt_sched_bonus_16ths) >> 4)); |
677 | } else { |
678 | thread->sched_usage += delta; |
679 | } |
680 | } |
681 | |
682 | thread->cpu_usage += delta + thread->cpu_delta; |
683 | thread->cpu_delta = 0; |
684 | |
685 | #if CONFIG_SCHED_CLUTCH |
686 | /* |
687 | * Update the CPU usage for the thread group to which the thread belongs. |
688 | * The implementation assumes that the thread ran for the entire delta |
689 | * as part of the same thread group. |
690 | */ |
691 | sched_clutch_cpu_usage_update(thread, delta); |
692 | #endif /* CONFIG_SCHED_CLUTCH */ |
693 | |
694 | const struct shift_data *shiftp = &sched_decay_shifts[ticks]; |
695 | |
696 | if (shiftp->shift2 > 0) { |
697 | thread->cpu_usage = (thread->cpu_usage >> shiftp->shift1) + |
698 | (thread->cpu_usage >> shiftp->shift2); |
699 | thread->sched_usage = (thread->sched_usage >> shiftp->shift1) + |
700 | (thread->sched_usage >> shiftp->shift2); |
701 | } else { |
702 | thread->cpu_usage = (thread->cpu_usage >> shiftp->shift1) - |
703 | (thread->cpu_usage >> -(shiftp->shift2)); |
704 | thread->sched_usage = (thread->sched_usage >> shiftp->shift1) - |
705 | (thread->sched_usage >> -(shiftp->shift2)); |
706 | } |
707 | } else { |
708 | thread->cpu_usage = thread->cpu_delta = 0; |
709 | thread->sched_usage = 0; |
710 | } |
711 | |
712 | /* |
713 | * Check for fail-safe release. |
714 | */ |
715 | if ((thread->sched_flags & TH_SFLAG_FAILSAFE) && |
716 | mach_absolute_time() >= thread->safe_release) { |
717 | sched_thread_mode_undemote(thread, TH_SFLAG_FAILSAFE); |
718 | } |
719 | |
720 | /* |
721 | * Now that the thread's CPU usage has been accumulated and aged |
722 | * based on contention of the previous tick window, update the |
723 | * pri_shift of the thread to match the current global load/shift |
724 | * values. The updated pri_shift would be used to calculate the |
725 | * new priority of the thread. |
726 | */ |
727 | #if CONFIG_SCHED_CLUTCH |
728 | thread->pri_shift = sched_clutch_thread_pri_shift(thread, thread->th_sched_bucket); |
729 | #else /* CONFIG_SCHED_CLUTCH */ |
730 | thread->pri_shift = sched_pri_shifts[thread->th_sched_bucket]; |
731 | #endif /* CONFIG_SCHED_CLUTCH */ |
732 | |
733 | /* Recompute scheduled priority if appropriate. */ |
734 | if (thread->sched_mode == TH_MODE_TIMESHARE) { |
735 | thread_recompute_sched_pri(thread, options: SETPRI_LAZY); |
736 | } |
737 | } |
738 | |
739 | #endif /* CONFIG_SCHED_TIMESHARE_CORE */ |
740 | |
741 | |
742 | /* |
743 | * TH_BUCKET_RUN is a count of *all* runnable non-idle threads. |
744 | * Each other bucket is a count of the runnable non-idle threads |
745 | * with that property. All updates to these counts should be |
746 | * performed with os_atomic_* operations. |
747 | * |
748 | * For the clutch scheduler, this global bucket is used only for |
749 | * keeping the total global run count. |
750 | */ |
751 | uint32_t sched_run_buckets[TH_BUCKET_MAX]; |
752 | |
753 | static void |
754 | sched_incr_bucket(sched_bucket_t bucket) |
755 | { |
756 | assert(bucket >= TH_BUCKET_FIXPRI && |
757 | bucket <= TH_BUCKET_SHARE_BG); |
758 | |
759 | os_atomic_inc(&sched_run_buckets[bucket], relaxed); |
760 | } |
761 | |
762 | static void |
763 | sched_decr_bucket(sched_bucket_t bucket) |
764 | { |
765 | assert(bucket >= TH_BUCKET_FIXPRI && |
766 | bucket <= TH_BUCKET_SHARE_BG); |
767 | |
768 | assert(os_atomic_load(&sched_run_buckets[bucket], relaxed) > 0); |
769 | |
770 | os_atomic_dec(&sched_run_buckets[bucket], relaxed); |
771 | } |
772 | |
773 | static void |
774 | sched_add_bucket(sched_bucket_t bucket, uint8_t run_weight) |
775 | { |
776 | assert(bucket >= TH_BUCKET_FIXPRI && |
777 | bucket <= TH_BUCKET_SHARE_BG); |
778 | |
779 | os_atomic_add(&sched_run_buckets[bucket], run_weight, relaxed); |
780 | } |
781 | |
782 | static void |
783 | sched_sub_bucket(sched_bucket_t bucket, uint8_t run_weight) |
784 | { |
785 | assert(bucket >= TH_BUCKET_FIXPRI && |
786 | bucket <= TH_BUCKET_SHARE_BG); |
787 | |
788 | assert(os_atomic_load(&sched_run_buckets[bucket], relaxed) > 0); |
789 | |
790 | os_atomic_sub(&sched_run_buckets[bucket], run_weight, relaxed); |
791 | } |
792 | |
793 | uint32_t |
794 | sched_run_incr(thread_t thread) |
795 | { |
796 | assert((thread->state & (TH_RUN | TH_IDLE)) == TH_RUN); |
797 | |
798 | uint32_t new_count = os_atomic_inc(&sched_run_buckets[TH_BUCKET_RUN], relaxed); |
799 | |
800 | sched_incr_bucket(bucket: thread->th_sched_bucket); |
801 | |
802 | return new_count; |
803 | } |
804 | |
805 | uint32_t |
806 | sched_run_decr(thread_t thread) |
807 | { |
808 | assert((thread->state & (TH_RUN | TH_IDLE)) != TH_RUN); |
809 | |
810 | sched_decr_bucket(bucket: thread->th_sched_bucket); |
811 | |
812 | uint32_t new_count = os_atomic_dec(&sched_run_buckets[TH_BUCKET_RUN], relaxed); |
813 | |
814 | return new_count; |
815 | } |
816 | |
817 | uint32_t |
818 | sched_smt_run_incr(thread_t thread) |
819 | { |
820 | assert((thread->state & (TH_RUN | TH_IDLE)) == TH_RUN); |
821 | |
822 | uint8_t run_weight = (thread_no_smt(thread) && smt_timeshare_enabled) ? 2 : 1; |
823 | thread->sched_saved_run_weight = run_weight; |
824 | |
825 | uint32_t new_count = os_atomic_add(&sched_run_buckets[TH_BUCKET_RUN], run_weight, relaxed); |
826 | |
827 | sched_add_bucket(bucket: thread->th_sched_bucket, run_weight); |
828 | |
829 | return new_count; |
830 | } |
831 | |
832 | uint32_t |
833 | sched_smt_run_decr(thread_t thread) |
834 | { |
835 | assert((thread->state & (TH_RUN | TH_IDLE)) != TH_RUN); |
836 | |
837 | uint8_t run_weight = thread->sched_saved_run_weight; |
838 | |
839 | sched_sub_bucket(bucket: thread->th_sched_bucket, run_weight); |
840 | |
841 | uint32_t new_count = os_atomic_sub(&sched_run_buckets[TH_BUCKET_RUN], run_weight, relaxed); |
842 | |
843 | return new_count; |
844 | } |
845 | |
846 | void |
847 | sched_update_thread_bucket(thread_t thread) |
848 | { |
849 | sched_bucket_t old_bucket = thread->th_sched_bucket; |
850 | sched_bucket_t new_bucket = TH_BUCKET_RUN; |
851 | |
852 | switch (thread->sched_mode) { |
853 | case TH_MODE_FIXED: |
854 | case TH_MODE_REALTIME: |
855 | new_bucket = TH_BUCKET_FIXPRI; |
856 | break; |
857 | |
858 | case TH_MODE_TIMESHARE: |
859 | if (thread->base_pri > BASEPRI_DEFAULT) { |
860 | new_bucket = TH_BUCKET_SHARE_FG; |
861 | } else if (thread->base_pri > BASEPRI_UTILITY) { |
862 | new_bucket = TH_BUCKET_SHARE_DF; |
863 | } else if (thread->base_pri > MAXPRI_THROTTLE) { |
864 | new_bucket = TH_BUCKET_SHARE_UT; |
865 | } else { |
866 | new_bucket = TH_BUCKET_SHARE_BG; |
867 | } |
868 | break; |
869 | |
870 | default: |
871 | panic("unexpected mode: %d" , thread->sched_mode); |
872 | break; |
873 | } |
874 | |
875 | if (old_bucket != new_bucket) { |
876 | thread->th_sched_bucket = new_bucket; |
877 | thread->pri_shift = sched_pri_shifts[new_bucket]; |
878 | |
879 | if ((thread->state & (TH_RUN | TH_IDLE)) == TH_RUN) { |
880 | sched_decr_bucket(bucket: old_bucket); |
881 | sched_incr_bucket(bucket: new_bucket); |
882 | } |
883 | } |
884 | } |
885 | |
886 | void |
887 | sched_smt_update_thread_bucket(thread_t thread) |
888 | { |
889 | sched_bucket_t old_bucket = thread->th_sched_bucket; |
890 | sched_bucket_t new_bucket = TH_BUCKET_RUN; |
891 | |
892 | switch (thread->sched_mode) { |
893 | case TH_MODE_FIXED: |
894 | case TH_MODE_REALTIME: |
895 | new_bucket = TH_BUCKET_FIXPRI; |
896 | break; |
897 | |
898 | case TH_MODE_TIMESHARE: |
899 | if (thread->base_pri > BASEPRI_DEFAULT) { |
900 | new_bucket = TH_BUCKET_SHARE_FG; |
901 | } else if (thread->base_pri > BASEPRI_UTILITY) { |
902 | new_bucket = TH_BUCKET_SHARE_DF; |
903 | } else if (thread->base_pri > MAXPRI_THROTTLE) { |
904 | new_bucket = TH_BUCKET_SHARE_UT; |
905 | } else { |
906 | new_bucket = TH_BUCKET_SHARE_BG; |
907 | } |
908 | break; |
909 | |
910 | default: |
911 | panic("unexpected mode: %d" , thread->sched_mode); |
912 | break; |
913 | } |
914 | |
915 | if (old_bucket != new_bucket) { |
916 | thread->th_sched_bucket = new_bucket; |
917 | thread->pri_shift = sched_pri_shifts[new_bucket]; |
918 | |
919 | if ((thread->state & (TH_RUN | TH_IDLE)) == TH_RUN) { |
920 | sched_sub_bucket(bucket: old_bucket, run_weight: thread->sched_saved_run_weight); |
921 | sched_add_bucket(bucket: new_bucket, run_weight: thread->sched_saved_run_weight); |
922 | } |
923 | } |
924 | } |
925 | |
926 | static inline void |
927 | sched_validate_mode(sched_mode_t mode) |
928 | { |
929 | switch (mode) { |
930 | case TH_MODE_FIXED: |
931 | case TH_MODE_REALTIME: |
932 | case TH_MODE_TIMESHARE: |
933 | break; |
934 | |
935 | default: |
936 | panic("unexpected mode: %d" , mode); |
937 | break; |
938 | } |
939 | } |
940 | |
941 | /* |
942 | * Set the thread's true scheduling mode |
943 | * Called with thread mutex and thread locked |
944 | * The thread has already been removed from the runqueue. |
945 | * |
946 | * (saved_mode is handled before this point) |
947 | */ |
948 | void |
949 | sched_set_thread_mode(thread_t thread, sched_mode_t new_mode) |
950 | { |
951 | thread_assert_runq_null(thread); |
952 | |
953 | sched_validate_mode(mode: new_mode); |
954 | |
955 | #if CONFIG_SCHED_AUTO_JOIN |
956 | /* |
957 | * Realtime threads might have auto-joined a work interval based on |
958 | * make runnable relationships. If such an RT thread is now being demoted |
959 | * to non-RT, unjoin the thread from the work interval. |
960 | */ |
961 | if ((thread->sched_flags & TH_SFLAG_THREAD_GROUP_AUTO_JOIN) && (new_mode != TH_MODE_REALTIME)) { |
962 | assert((thread->sched_mode == TH_MODE_REALTIME) || (thread->th_work_interval_flags & TH_WORK_INTERVAL_FLAGS_AUTO_JOIN_LEAK)); |
963 | work_interval_auto_join_demote(thread); |
964 | } |
965 | #endif /* CONFIG_SCHED_AUTO_JOIN */ |
966 | |
967 | thread->sched_mode = new_mode; |
968 | |
969 | SCHED(update_thread_bucket)(thread); |
970 | } |
971 | |
972 | /* |
973 | * TODO: Instead of having saved mode, have 'user mode' and 'true mode'. |
974 | * That way there's zero confusion over which the user wants |
975 | * and which the kernel wants. |
976 | */ |
977 | void |
978 | sched_set_thread_mode_user(thread_t thread, sched_mode_t new_mode) |
979 | { |
980 | thread_assert_runq_null(thread); |
981 | |
982 | sched_validate_mode(mode: new_mode); |
983 | |
984 | /* If demoted, only modify the saved mode. */ |
985 | if (thread->sched_flags & TH_SFLAG_DEMOTED_MASK) { |
986 | thread->saved_mode = new_mode; |
987 | } else { |
988 | sched_set_thread_mode(thread, new_mode); |
989 | } |
990 | } |
991 | |
992 | sched_mode_t |
993 | sched_get_thread_mode_user(thread_t thread) |
994 | { |
995 | if (thread->sched_flags & TH_SFLAG_DEMOTED_MASK) { |
996 | return thread->saved_mode; |
997 | } else { |
998 | return thread->sched_mode; |
999 | } |
1000 | } |
1001 | |
1002 | /* |
1003 | * Demote the true scheduler mode to timeshare (called with the thread locked) |
1004 | */ |
1005 | void |
1006 | sched_thread_mode_demote(thread_t thread, uint32_t reason) |
1007 | { |
1008 | assert(reason & TH_SFLAG_DEMOTED_MASK); |
1009 | assert((thread->sched_flags & reason) != reason); |
1010 | |
1011 | if (thread->policy_reset) { |
1012 | return; |
1013 | } |
1014 | |
1015 | switch (reason) { |
1016 | case TH_SFLAG_THROTTLED: |
1017 | KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_MODE_DEMOTE_THROTTLED), |
1018 | thread_tid(thread), thread->sched_flags); |
1019 | break; |
1020 | case TH_SFLAG_FAILSAFE: |
1021 | KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_MODE_DEMOTE_FAILSAFE), |
1022 | thread_tid(thread), thread->sched_flags); |
1023 | break; |
1024 | case TH_SFLAG_RT_DISALLOWED: |
1025 | KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_MODE_DEMOTE_RT_DISALLOWED), |
1026 | thread_tid(thread), thread->sched_flags); |
1027 | break; |
1028 | } |
1029 | |
1030 | if (thread->sched_flags & TH_SFLAG_DEMOTED_MASK) { |
1031 | /* Another demotion reason is already active */ |
1032 | thread->sched_flags |= reason; |
1033 | return; |
1034 | } |
1035 | |
1036 | assert(thread->saved_mode == TH_MODE_NONE); |
1037 | |
1038 | boolean_t removed = thread_run_queue_remove(thread); |
1039 | |
1040 | thread->sched_flags |= reason; |
1041 | |
1042 | thread->saved_mode = thread->sched_mode; |
1043 | |
1044 | sched_set_thread_mode(thread, new_mode: TH_MODE_TIMESHARE); |
1045 | |
1046 | thread_recompute_priority(thread); |
1047 | |
1048 | if (removed) { |
1049 | thread_run_queue_reinsert(thread, options: SCHED_TAILQ); |
1050 | } |
1051 | } |
1052 | |
1053 | /* |
1054 | * Return true if the thread is demoted for the specified reason |
1055 | */ |
1056 | bool |
1057 | sched_thread_mode_has_demotion(thread_t thread, uint32_t reason) |
1058 | { |
1059 | assert(reason & TH_SFLAG_DEMOTED_MASK); |
1060 | return (thread->sched_flags & reason) != 0; |
1061 | } |
1062 | |
1063 | /* |
1064 | * Un-demote the true scheduler mode back to the saved mode (called with the thread locked) |
1065 | */ |
1066 | void |
1067 | sched_thread_mode_undemote(thread_t thread, uint32_t reason) |
1068 | { |
1069 | assert(reason & TH_SFLAG_DEMOTED_MASK); |
1070 | assert((thread->sched_flags & reason) == reason); |
1071 | assert(thread->saved_mode != TH_MODE_NONE); |
1072 | assert(thread->sched_mode == TH_MODE_TIMESHARE); |
1073 | assert(thread->policy_reset == 0); |
1074 | |
1075 | switch (reason) { |
1076 | case TH_SFLAG_THROTTLED: |
1077 | KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_MODE_UNDEMOTE_THROTTLED), |
1078 | thread_tid(thread), thread->sched_flags); |
1079 | break; |
1080 | case TH_SFLAG_FAILSAFE: |
1081 | KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_MODE_UNDEMOTE_FAILSAFE), |
1082 | thread_tid(thread), thread->sched_flags); |
1083 | break; |
1084 | case TH_SFLAG_RT_DISALLOWED: |
1085 | KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_MODE_UNDEMOTE_RT_DISALLOWED), |
1086 | thread_tid(thread), thread->sched_flags); |
1087 | break; |
1088 | } |
1089 | |
1090 | thread->sched_flags &= ~reason; |
1091 | |
1092 | if (thread->sched_flags & TH_SFLAG_DEMOTED_MASK) { |
1093 | /* Another demotion reason is still active */ |
1094 | return; |
1095 | } |
1096 | |
1097 | boolean_t removed = thread_run_queue_remove(thread); |
1098 | |
1099 | sched_set_thread_mode(thread, new_mode: thread->saved_mode); |
1100 | |
1101 | thread->saved_mode = TH_MODE_NONE; |
1102 | |
1103 | thread_recompute_priority(thread); |
1104 | |
1105 | if (removed) { |
1106 | thread_run_queue_reinsert(thread, options: SCHED_TAILQ); |
1107 | } |
1108 | } |
1109 | |
1110 | /* |
1111 | * Promote thread to have a sched pri floor for a specific reason |
1112 | * |
1113 | * Promotion must not last past syscall boundary |
1114 | * Clients must always pair promote and demote 1:1, |
1115 | * Handling nesting of the same promote reason is the client's responsibility |
1116 | * |
1117 | * Called at splsched with thread locked |
1118 | */ |
1119 | void |
1120 | sched_thread_promote_reason(thread_t thread, |
1121 | uint32_t reason, |
1122 | __kdebug_only uintptr_t trace_obj /* already unslid */) |
1123 | { |
1124 | assert(reason & TH_SFLAG_PROMOTE_REASON_MASK); |
1125 | assert((thread->sched_flags & reason) != reason); |
1126 | |
1127 | switch (reason) { |
1128 | case TH_SFLAG_RW_PROMOTED: |
1129 | KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_RW_PROMOTE), |
1130 | thread_tid(thread), thread->sched_pri, |
1131 | thread->base_pri, trace_obj); |
1132 | break; |
1133 | case TH_SFLAG_WAITQ_PROMOTED: |
1134 | KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_WAITQ_PROMOTE), |
1135 | thread_tid(thread), thread->sched_pri, |
1136 | thread->base_pri, trace_obj); |
1137 | break; |
1138 | case TH_SFLAG_EXEC_PROMOTED: |
1139 | KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_EXEC_PROMOTE), |
1140 | thread_tid(thread), thread->sched_pri, |
1141 | thread->base_pri, trace_obj); |
1142 | break; |
1143 | case TH_SFLAG_FLOOR_PROMOTED: |
1144 | KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_FLOOR_PROMOTE), |
1145 | thread_tid(thread), thread->sched_pri, |
1146 | thread->base_pri, trace_obj); |
1147 | break; |
1148 | } |
1149 | |
1150 | thread->sched_flags |= reason; |
1151 | thread_recompute_sched_pri(thread, options: SETPRI_DEFAULT); |
1152 | } |
1153 | |
1154 | /* |
1155 | * End a specific promotion reason |
1156 | * Demotes a thread back to its expected priority without the promotion in place |
1157 | * |
1158 | * Called at splsched with thread locked |
1159 | */ |
1160 | void |
1161 | sched_thread_unpromote_reason(thread_t thread, |
1162 | uint32_t reason, |
1163 | __kdebug_only uintptr_t trace_obj /* already unslid */) |
1164 | { |
1165 | assert(reason & TH_SFLAG_PROMOTE_REASON_MASK); |
1166 | assert((thread->sched_flags & reason) == reason); |
1167 | |
1168 | switch (reason) { |
1169 | case TH_SFLAG_RW_PROMOTED: |
1170 | KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_RW_DEMOTE), |
1171 | thread_tid(thread), thread->sched_pri, |
1172 | thread->base_pri, trace_obj); |
1173 | break; |
1174 | case TH_SFLAG_WAITQ_PROMOTED: |
1175 | KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_WAITQ_DEMOTE), |
1176 | thread_tid(thread), thread->sched_pri, |
1177 | thread->base_pri, trace_obj); |
1178 | break; |
1179 | case TH_SFLAG_EXEC_PROMOTED: |
1180 | KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_EXEC_DEMOTE), |
1181 | thread_tid(thread), thread->sched_pri, |
1182 | thread->base_pri, trace_obj); |
1183 | break; |
1184 | case TH_SFLAG_FLOOR_PROMOTED: |
1185 | KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_FLOOR_DEMOTE), |
1186 | thread_tid(thread), thread->sched_pri, |
1187 | thread->base_pri, trace_obj); |
1188 | break; |
1189 | } |
1190 | |
1191 | thread->sched_flags &= ~reason; |
1192 | |
1193 | thread_recompute_sched_pri(thread, options: SETPRI_DEFAULT); |
1194 | } |
1195 | |