1 | /* |
2 | * Copyright (c) 2000-2017 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * @OSF_COPYRIGHT@ |
30 | */ |
31 | /* |
32 | * Mach Operating System |
33 | * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University |
34 | * All Rights Reserved. |
35 | * |
36 | * Permission to use, copy, modify and distribute this software and its |
37 | * documentation is hereby granted, provided that both the copyright |
38 | * notice and this permission notice appear in all copies of the |
39 | * software, derivative works or modified versions, and any portions |
40 | * thereof, and that both notices appear in supporting documentation. |
41 | * |
42 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" |
43 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR |
44 | * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. |
45 | * |
46 | * Carnegie Mellon requests users of this software to return to |
47 | * |
48 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU |
49 | * School of Computer Science |
50 | * Carnegie Mellon University |
51 | * Pittsburgh PA 15213-3890 |
52 | * |
53 | * any improvements or extensions that they make and grant Carnegie Mellon |
54 | * the rights to redistribute these changes. |
55 | */ |
56 | |
57 | #include <kern/ast.h> |
58 | #include <kern/counter.h> |
59 | #include <kern/misc_protos.h> |
60 | #include <kern/queue.h> |
61 | #include <kern/sched_prim.h> |
62 | #include <kern/thread.h> |
63 | #include <kern/processor.h> |
64 | #include <kern/restartable.h> |
65 | #include <kern/spl.h> |
66 | #include <kern/sfi.h> |
67 | #if CONFIG_TELEMETRY |
68 | #include <kern/telemetry.h> |
69 | #endif |
70 | #include <kern/waitq.h> |
71 | #include <kern/ledger.h> |
72 | #include <kern/machine.h> |
73 | #include <kern/kpc.h> |
74 | #include <kperf/kperf.h> |
75 | #include <mach/policy.h> |
76 | #include <security/mac_mach_internal.h> // for MACF AST hook |
77 | #include <stdatomic.h> |
78 | |
79 | #if CONFIG_ARCADE |
80 | #include <kern/arcade.h> |
81 | #endif |
82 | |
83 | static void __attribute__((noinline, noreturn, disable_tail_calls)) |
84 | thread_preempted(__unused void* parameter, __unused wait_result_t result) |
85 | { |
86 | /* |
87 | * We've been scheduled again after a userspace preemption, |
88 | * try again to return to userspace. |
89 | */ |
90 | thread_exception_return(); |
91 | } |
92 | |
93 | /* |
94 | * Create a dedicated frame to clarify that this thread has been preempted |
95 | * while running in kernel space. |
96 | */ |
97 | static void __attribute__((noinline, disable_tail_calls)) |
98 | thread_preempted_in_kernel(ast_t urgent_reason) |
99 | { |
100 | thread_block_reason(THREAD_CONTINUE_NULL, NULL, reason: urgent_reason); |
101 | |
102 | assert(ml_get_interrupts_enabled() == FALSE); |
103 | } |
104 | |
105 | /* |
106 | * AST_URGENT was detected while in kernel mode |
107 | * Called with interrupts disabled, returns the same way |
108 | * Must return to caller |
109 | */ |
110 | void |
111 | ast_taken_kernel(void) |
112 | { |
113 | assert(ml_get_interrupts_enabled() == FALSE); |
114 | |
115 | thread_t thread = current_thread(); |
116 | |
117 | /* Idle threads handle preemption themselves */ |
118 | if ((thread->state & TH_IDLE)) { |
119 | ast_off(AST_PREEMPTION); |
120 | return; |
121 | } |
122 | |
123 | /* |
124 | * It's possible for this to be called after AST_URGENT |
125 | * has already been handled, due to races in enable_preemption |
126 | */ |
127 | if (ast_peek(AST_URGENT) != AST_URGENT) { |
128 | return; |
129 | } |
130 | |
131 | /* |
132 | * Don't preempt if the thread is already preparing to block. |
133 | * TODO: the thread can cheese this with clear_wait() |
134 | */ |
135 | if (waitq_wait_possible(thread) == FALSE) { |
136 | /* Consume AST_URGENT or the interrupt will call us again */ |
137 | ast_consume(AST_URGENT); |
138 | return; |
139 | } |
140 | |
141 | /* TODO: Should we csw_check again to notice if conditions have changed? */ |
142 | |
143 | ast_t urgent_reason = ast_consume(AST_PREEMPTION); |
144 | |
145 | assert(urgent_reason & AST_PREEMPT); |
146 | |
147 | /* We've decided to try context switching */ |
148 | thread_preempted_in_kernel(urgent_reason); |
149 | } |
150 | |
151 | /* |
152 | * An AST flag was set while returning to user mode |
153 | * Called with interrupts disabled, returns with interrupts enabled |
154 | * May call continuation instead of returning |
155 | */ |
156 | void |
157 | ast_taken_user(void) |
158 | { |
159 | assert(ml_get_interrupts_enabled() == FALSE); |
160 | |
161 | thread_t thread = current_thread(); |
162 | task_t task = get_threadtask(thread); |
163 | |
164 | /* We are about to return to userspace, there must not be a pending wait */ |
165 | assert(waitq_wait_possible(thread)); |
166 | assert((thread->state & TH_IDLE) == 0); |
167 | |
168 | /* TODO: Add more 'return to userspace' assertions here */ |
169 | |
170 | /* |
171 | * If this thread was urgently preempted in userspace, |
172 | * take the preemption before processing the ASTs. |
173 | * The trap handler will call us again if we have more ASTs, so it's |
174 | * safe to block in a continuation here. |
175 | */ |
176 | if (ast_peek(AST_URGENT) == AST_URGENT) { |
177 | ast_t urgent_reason = ast_consume(AST_PREEMPTION); |
178 | |
179 | assert(urgent_reason & AST_PREEMPT); |
180 | |
181 | /* TODO: Should we csw_check again to notice if conditions have changed? */ |
182 | |
183 | thread_block_reason(continuation: thread_preempted, NULL, reason: urgent_reason); |
184 | /* NOTREACHED */ |
185 | } |
186 | |
187 | /* |
188 | * AST_KEVENT does not send an IPI when setting the ast for a thread running in parallel |
189 | * on a different processor. Only the ast bit on the thread will be set. |
190 | * |
191 | * Force a propagate for concurrent updates without an IPI. |
192 | */ |
193 | ast_propagate(thread); |
194 | |
195 | /* |
196 | * Consume all non-preemption processor ASTs matching reasons |
197 | * because we're handling them here. |
198 | * |
199 | * If one of the AST handlers blocks in a continuation, |
200 | * we'll reinstate the unserviced thread-level AST flags |
201 | * from the thread to the processor on context switch. |
202 | * If one of the AST handlers sets another AST, |
203 | * the trap handler will call ast_taken_user again. |
204 | * |
205 | * We expect the AST handlers not to thread_exception_return |
206 | * without an ast_propagate or context switch to reinstate |
207 | * the per-processor ASTs. |
208 | * |
209 | * TODO: Why are AST_DTRACE and AST_KPERF not per-thread ASTs? |
210 | */ |
211 | ast_t reasons = ast_consume(AST_PER_THREAD | AST_KPERF | AST_DTRACE); |
212 | |
213 | ml_set_interrupts_enabled(TRUE); |
214 | |
215 | #if CONFIG_DTRACE |
216 | if (reasons & AST_DTRACE) { |
217 | dtrace_ast(); |
218 | } |
219 | #endif |
220 | |
221 | #ifdef MACH_BSD |
222 | if (reasons & AST_BSD) { |
223 | thread_ast_clear(thread, AST_BSD); |
224 | bsd_ast(thread); |
225 | } |
226 | #endif |
227 | |
228 | #if CONFIG_MACF |
229 | if (reasons & AST_MACF) { |
230 | thread_ast_clear(thread, AST_MACF); |
231 | mac_thread_userret(thread); |
232 | } |
233 | #endif |
234 | |
235 | #if CONFIG_ARCADE |
236 | if (reasons & AST_ARCADE) { |
237 | thread_ast_clear(thread, AST_ARCADE); |
238 | arcade_ast(thread); |
239 | } |
240 | #endif |
241 | |
242 | if (reasons & AST_APC) { |
243 | thread_ast_clear(thread, AST_APC); |
244 | thread_apc_ast(thread); |
245 | } |
246 | |
247 | if (reasons & AST_GUARD) { |
248 | thread_ast_clear(thread, AST_GUARD); |
249 | guard_ast(thread); |
250 | } |
251 | |
252 | if (reasons & AST_LEDGER) { |
253 | thread_ast_clear(thread, AST_LEDGER); |
254 | ledger_ast(thread); |
255 | } |
256 | |
257 | if (reasons & AST_KPERF) { |
258 | thread_ast_clear(thread, AST_KPERF); |
259 | #if CONFIG_CPU_COUNTERS |
260 | kpc_thread_ast_handler(thread); |
261 | #endif /* CONFIG_CPU_COUNTERS */ |
262 | kperf_thread_ast_handler(thread); |
263 | thread->kperf_ast = 0; |
264 | } |
265 | |
266 | if (reasons & AST_RESET_PCS) { |
267 | thread_ast_clear(thread, AST_RESET_PCS); |
268 | thread_reset_pcs_ast(task, thread); |
269 | } |
270 | |
271 | if (reasons & AST_KEVENT) { |
272 | thread_ast_clear(thread, AST_KEVENT); |
273 | uint16_t bits = atomic_exchange(&thread->kevent_ast_bits, 0); |
274 | if (bits) { |
275 | kevent_ast(thread, bits); |
276 | } |
277 | } |
278 | |
279 | if (reasons & AST_PROC_RESOURCE) { |
280 | thread_ast_clear(thread, AST_PROC_RESOURCE); |
281 | task_port_space_ast(task); |
282 | #if MACH_BSD |
283 | proc_filedesc_ast(task); |
284 | #endif /* MACH_BSD */ |
285 | } |
286 | |
287 | #if CONFIG_TELEMETRY |
288 | if (reasons & AST_TELEMETRY_ALL) { |
289 | ast_t telemetry_reasons = reasons & AST_TELEMETRY_ALL; |
290 | thread_ast_clear(thread, AST_TELEMETRY_ALL); |
291 | telemetry_ast(thread, reasons: telemetry_reasons); |
292 | } |
293 | #endif |
294 | |
295 | #if MACH_ASSERT |
296 | if (reasons & AST_DEBUG_ASSERT) { |
297 | thread_ast_clear(thread, AST_DEBUG_ASSERT); |
298 | thread_debug_return_to_user_ast(thread); |
299 | } |
300 | #endif |
301 | |
302 | spl_t s = splsched(); |
303 | |
304 | #if CONFIG_SCHED_SFI |
305 | /* |
306 | * SFI is currently a per-processor AST, not a per-thread AST |
307 | * TODO: SFI should be a per-thread AST |
308 | */ |
309 | if (ast_consume(AST_SFI) == AST_SFI) { |
310 | sfi_ast(thread); |
311 | } |
312 | #endif |
313 | |
314 | /* We are about to return to userspace, there must not be a pending wait */ |
315 | assert(waitq_wait_possible(thread)); |
316 | |
317 | /* |
318 | * We've handled all per-thread ASTs, time to handle non-urgent preemption. |
319 | * |
320 | * We delay reading the preemption bits until now in case the thread |
321 | * blocks while handling per-thread ASTs. |
322 | * |
323 | * If one of the AST handlers had managed to set a new AST bit, |
324 | * thread_exception_return will call ast_taken again. |
325 | */ |
326 | ast_t preemption_reasons = ast_consume(AST_PREEMPTION); |
327 | |
328 | if (preemption_reasons & AST_PREEMPT) { |
329 | /* Conditions may have changed from when the AST_PREEMPT was originally set, so re-check. */ |
330 | |
331 | thread_lock(thread); |
332 | preemption_reasons = csw_check(thread, processor: current_processor(), check_reason: (preemption_reasons & AST_QUANTUM)); |
333 | thread_unlock(thread); |
334 | |
335 | #if CONFIG_SCHED_SFI |
336 | /* csw_check might tell us that SFI is needed */ |
337 | if (preemption_reasons & AST_SFI) { |
338 | sfi_ast(thread); |
339 | } |
340 | #endif |
341 | |
342 | if (preemption_reasons & AST_PREEMPT) { |
343 | /* switching to a continuation implicitly re-enables interrupts */ |
344 | thread_block_reason(continuation: thread_preempted, NULL, reason: preemption_reasons); |
345 | /* NOTREACHED */ |
346 | } |
347 | |
348 | /* |
349 | * We previously had a pending AST_PREEMPT, but csw_check |
350 | * decided that it should no longer be set, and to keep |
351 | * executing the current thread instead. |
352 | * Clear the pending preemption timer as we no longer |
353 | * have a pending AST_PREEMPT to time out. |
354 | * |
355 | * TODO: just do the thread block if we see AST_PREEMPT |
356 | * to avoid taking the pset lock twice. |
357 | * To do that thread block needs to be smarter |
358 | * about not context switching when it's not necessary |
359 | * e.g. the first-timeslice check for queue has priority |
360 | */ |
361 | clear_pending_nonurgent_preemption(processor: current_processor()); |
362 | } |
363 | |
364 | splx(s); |
365 | |
366 | /* |
367 | * Here's a good place to put assertions of things which must be true |
368 | * upon return to userspace. |
369 | */ |
370 | assert(thread->kern_promotion_schedpri == 0); |
371 | if (thread->rwlock_count > 0) { |
372 | panic("rwlock_count is %d for thread %p, possibly it still holds a rwlock" , thread->rwlock_count, thread); |
373 | } |
374 | assert(thread->priority_floor_count == 0); |
375 | |
376 | assert3u(0, ==, thread->sched_flags & |
377 | (TH_SFLAG_WAITQ_PROMOTED | |
378 | TH_SFLAG_RW_PROMOTED | |
379 | TH_SFLAG_EXEC_PROMOTED | |
380 | TH_SFLAG_FLOOR_PROMOTED | |
381 | TH_SFLAG_PROMOTED | |
382 | TH_SFLAG_DEPRESS)); |
383 | } |
384 | |
385 | /* |
386 | * Set AST flags on current processor |
387 | * Called at splsched |
388 | */ |
389 | void |
390 | ast_on(ast_t reasons) |
391 | { |
392 | ast_t *pending_ast = ast_pending(); |
393 | |
394 | *pending_ast |= reasons; |
395 | } |
396 | |
397 | /* |
398 | * Clear AST flags on current processor |
399 | * Called at splsched |
400 | */ |
401 | void |
402 | ast_off(ast_t reasons) |
403 | { |
404 | ast_t *pending_ast = ast_pending(); |
405 | |
406 | *pending_ast &= ~reasons; |
407 | } |
408 | |
409 | /* |
410 | * Consume the requested subset of the AST flags set on the processor |
411 | * Return the bits that were set |
412 | * Called at splsched |
413 | */ |
414 | ast_t |
415 | ast_consume(ast_t reasons) |
416 | { |
417 | ast_t *pending_ast = ast_pending(); |
418 | |
419 | reasons &= *pending_ast; |
420 | *pending_ast &= ~reasons; |
421 | |
422 | return reasons; |
423 | } |
424 | |
425 | /* |
426 | * Read the requested subset of the AST flags set on the processor |
427 | * Return the bits that were set, don't modify the processor |
428 | * Called at splsched |
429 | */ |
430 | ast_t |
431 | ast_peek(ast_t reasons) |
432 | { |
433 | ast_t *pending_ast = ast_pending(); |
434 | |
435 | reasons &= *pending_ast; |
436 | |
437 | return reasons; |
438 | } |
439 | |
440 | /* |
441 | * Re-set current processor's per-thread AST flags to those set on thread |
442 | * Called at splsched |
443 | */ |
444 | void |
445 | ast_context(thread_t thread) |
446 | { |
447 | ast_t *pending_ast = ast_pending(); |
448 | |
449 | *pending_ast = (*pending_ast & ~AST_PER_THREAD) | thread_ast_get(thread); |
450 | } |
451 | |
452 | /* |
453 | * Propagate ASTs set on a thread to the current processor |
454 | * Called at splsched |
455 | */ |
456 | void |
457 | ast_propagate(thread_t thread) |
458 | { |
459 | ast_on(thread_ast_get(thread)); |
460 | } |
461 | |
462 | void |
463 | ast_dtrace_on(void) |
464 | { |
465 | ast_on(AST_DTRACE); |
466 | } |
467 | |