1 | /* |
2 | * Copyright (c) 2012-2023 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <arm/caches_internal.h> |
30 | #include <arm/cpu_data.h> |
31 | #include <arm/cpu_data_internal.h> |
32 | #include <arm/misc_protos.h> |
33 | #include <arm/thread.h> |
34 | #include <arm/rtclock.h> |
35 | #include <arm/trap_internal.h> /* for IS_ARM_GDB_TRAP() et al */ |
36 | #include <arm64/proc_reg.h> |
37 | #include <arm64/machine_machdep.h> |
38 | #include <arm64/monotonic.h> |
39 | #include <arm64/instructions.h> |
40 | |
41 | #include <kern/debug.h> |
42 | #include <kern/restartable.h> |
43 | #include <kern/socd_client.h> |
44 | #include <kern/task.h> |
45 | #include <kern/thread.h> |
46 | #include <kern/zalloc_internal.h> |
47 | #include <mach/exception.h> |
48 | #include <mach/arm/traps.h> |
49 | #include <mach/vm_types.h> |
50 | #include <mach/machine/thread_status.h> |
51 | |
52 | #include <machine/atomic.h> |
53 | #include <machine/limits.h> |
54 | |
55 | #include <pexpert/arm/protos.h> |
56 | #include <pexpert/arm64/apple_arm64_regs.h> |
57 | #include <pexpert/arm64/board_config.h> |
58 | |
59 | #include <vm/vm_page.h> |
60 | #include <vm/pmap.h> |
61 | #include <vm/vm_fault.h> |
62 | #include <vm/vm_kern.h> |
63 | |
64 | #include <sys/errno.h> |
65 | #include <sys/kdebug.h> |
66 | #include <sys/code_signing.h> |
67 | #include <kperf/kperf.h> |
68 | |
69 | #include <kern/policy_internal.h> |
70 | #if CONFIG_TELEMETRY |
71 | #include <kern/telemetry.h> |
72 | #endif |
73 | |
74 | #include <prng/entropy.h> |
75 | |
76 | |
77 | |
78 | |
79 | #include <arm64/platform_error_handler.h> |
80 | |
81 | #if KASAN_TBI |
82 | #include <san/kasan.h> |
83 | #endif /* KASAN_TBI */ |
84 | |
85 | #if CONFIG_UBSAN_MINIMAL |
86 | #include <san/ubsan_minimal.h> |
87 | #endif |
88 | |
89 | |
90 | #ifndef __arm64__ |
91 | #error Should only be compiling for arm64. |
92 | #endif |
93 | |
94 | #if DEBUG || DEVELOPMENT |
95 | #define HAS_TELEMETRY_KERNEL_BRK 1 |
96 | #endif |
97 | |
98 | |
99 | #define TEST_CONTEXT32_SANITY(context) \ |
100 | (context->ss.ash.flavor == ARM_SAVED_STATE32 && context->ss.ash.count == ARM_SAVED_STATE32_COUNT && \ |
101 | context->ns.nsh.flavor == ARM_NEON_SAVED_STATE32 && context->ns.nsh.count == ARM_NEON_SAVED_STATE32_COUNT) |
102 | |
103 | #define TEST_CONTEXT64_SANITY(context) \ |
104 | (context->ss.ash.flavor == ARM_SAVED_STATE64 && context->ss.ash.count == ARM_SAVED_STATE64_COUNT && \ |
105 | context->ns.nsh.flavor == ARM_NEON_SAVED_STATE64 && context->ns.nsh.count == ARM_NEON_SAVED_STATE64_COUNT) |
106 | |
107 | #define ASSERT_CONTEXT_SANITY(context) \ |
108 | assert(TEST_CONTEXT32_SANITY(context) || TEST_CONTEXT64_SANITY(context)) |
109 | |
110 | |
111 | #define COPYIN(src, dst, size) \ |
112 | (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \ |
113 | copyin_kern(src, dst, size) : \ |
114 | copyin(src, dst, size) |
115 | |
116 | #define COPYOUT(src, dst, size) \ |
117 | (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \ |
118 | copyout_kern(src, dst, size) : \ |
119 | copyout(src, dst, size) |
120 | |
121 | // Below is for concatenating a string param to a string literal |
122 | #define STR1(x) #x |
123 | #define STR(x) STR1(x) |
124 | |
125 | #define ARM64_KDBG_CODE_KERNEL (0 << 8) |
126 | #define ARM64_KDBG_CODE_USER (1 << 8) |
127 | #define ARM64_KDBG_CODE_GUEST (2 << 8) |
128 | |
129 | _Static_assert(ARM64_KDBG_CODE_GUEST <= KDBG_CODE_MAX, "arm64 KDBG trace codes out of range" ); |
130 | _Static_assert(ARM64_KDBG_CODE_GUEST <= UINT16_MAX, "arm64 KDBG trace codes out of range" ); |
131 | |
132 | void panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss) __abortlike; |
133 | |
134 | void sleh_synchronous_sp1(arm_context_t *, uint32_t, vm_offset_t) __abortlike; |
135 | void sleh_synchronous(arm_context_t *, uint32_t, vm_offset_t, bool); |
136 | |
137 | |
138 | |
139 | void sleh_irq(arm_saved_state_t *); |
140 | void sleh_fiq(arm_saved_state_t *); |
141 | void sleh_serror(arm_context_t *context, uint32_t esr, vm_offset_t far); |
142 | void sleh_invalid_stack(arm_context_t *context, uint32_t esr, vm_offset_t far) __dead2; |
143 | |
144 | static void sleh_interrupt_handler_prologue(arm_saved_state_t *, unsigned int type); |
145 | static void sleh_interrupt_handler_epilogue(void); |
146 | |
147 | static void handle_svc(arm_saved_state_t *); |
148 | static void handle_mach_absolute_time_trap(arm_saved_state_t *); |
149 | static void handle_mach_continuous_time_trap(arm_saved_state_t *); |
150 | |
151 | static void handle_msr_trap(arm_saved_state_t *state, uint32_t esr); |
152 | #if __has_feature(ptrauth_calls) |
153 | static void handle_pac_fail(arm_saved_state_t *state, uint32_t esr) __dead2; |
154 | static inline uint64_t fault_addr_bitmask(unsigned int bit_from, unsigned int bit_to); |
155 | #endif |
156 | |
157 | extern kern_return_t arm_fast_fault(pmap_t, vm_map_address_t, vm_prot_t, bool, bool); |
158 | |
159 | static void handle_uncategorized(arm_saved_state_t *); |
160 | |
161 | static void handle_kernel_breakpoint(arm_saved_state_t *, uint32_t); |
162 | |
163 | static void handle_breakpoint(arm_saved_state_t *, uint32_t) __dead2; |
164 | |
165 | typedef void (*abort_inspector_t)(uint32_t, fault_status_t *, vm_prot_t *); |
166 | static void inspect_instruction_abort(uint32_t, fault_status_t *, vm_prot_t *); |
167 | static void inspect_data_abort(uint32_t, fault_status_t *, vm_prot_t *); |
168 | |
169 | static int is_vm_fault(fault_status_t); |
170 | static int is_translation_fault(fault_status_t); |
171 | static int is_alignment_fault(fault_status_t); |
172 | |
173 | typedef void (*abort_handler_t)(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t); |
174 | static void handle_user_abort(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t); |
175 | static void handle_kernel_abort(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t); |
176 | |
177 | static void handle_pc_align(arm_saved_state_t *ss) __dead2; |
178 | static void handle_sp_align(arm_saved_state_t *ss) __dead2; |
179 | static void handle_sw_step_debug(arm_saved_state_t *ss) __dead2; |
180 | static void handle_wf_trap(arm_saved_state_t *ss) __dead2; |
181 | static void handle_fp_trap(arm_saved_state_t *ss, uint32_t esr) __dead2; |
182 | |
183 | static void handle_watchpoint(vm_offset_t fault_addr) __dead2; |
184 | |
185 | static void handle_abort(arm_saved_state_t *, uint32_t, vm_offset_t, abort_inspector_t, abort_handler_t, expected_fault_handler_t); |
186 | |
187 | static void handle_user_trapped_instruction32(arm_saved_state_t *, uint32_t esr) __dead2; |
188 | |
189 | static void handle_simd_trap(arm_saved_state_t *, uint32_t esr) __dead2; |
190 | |
191 | extern void current_cached_proc_cred_update(void); |
192 | void mach_syscall_trace_exit(unsigned int retval, unsigned int call_number); |
193 | |
194 | struct proc; |
195 | |
196 | typedef uint32_t arm64_instr_t; |
197 | |
198 | extern void |
199 | unix_syscall(struct arm_saved_state * regs, thread_t thread_act, struct proc * proc); |
200 | |
201 | extern void |
202 | mach_syscall(struct arm_saved_state*); |
203 | |
204 | #if CONFIG_DTRACE |
205 | extern kern_return_t dtrace_user_probe(arm_saved_state_t* regs); |
206 | extern boolean_t dtrace_tally_fault(user_addr_t); |
207 | |
208 | /* |
209 | * Traps for userland processing. Can't include bsd/sys/fasttrap_isa.h, so copy |
210 | * and paste the trap instructions |
211 | * over from that file. Need to keep these in sync! |
212 | */ |
213 | #define FASTTRAP_ARM32_INSTR 0xe7ffdefc |
214 | #define FASTTRAP_THUMB32_INSTR 0xdefc |
215 | #define FASTTRAP_ARM64_INSTR 0xe7eeee7e |
216 | |
217 | #define FASTTRAP_ARM32_RET_INSTR 0xe7ffdefb |
218 | #define FASTTRAP_THUMB32_RET_INSTR 0xdefb |
219 | #define FASTTRAP_ARM64_RET_INSTR 0xe7eeee7d |
220 | |
221 | /* See <rdar://problem/4613924> */ |
222 | perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */ |
223 | #endif |
224 | |
225 | |
226 | |
227 | extern void arm64_thread_exception_return(void) __dead2; |
228 | |
229 | #if defined(APPLETYPHOON) |
230 | #define CPU_NAME "Typhoon" |
231 | #elif defined(APPLETWISTER) |
232 | #define CPU_NAME "Twister" |
233 | #elif defined(APPLEHURRICANE) |
234 | #define CPU_NAME "Hurricane" |
235 | #elif defined(APPLELIGHTNING) |
236 | #define CPU_NAME "Lightning" |
237 | #else |
238 | #define CPU_NAME "Unknown" |
239 | #endif |
240 | |
241 | #if (CONFIG_KERNEL_INTEGRITY && defined(KERNEL_INTEGRITY_WT)) |
242 | #define ESR_WT_SERROR(esr) (((esr) & 0xffffff00) == 0xbf575400) |
243 | #define ESR_WT_REASON(esr) ((esr) & 0xff) |
244 | |
245 | #define WT_REASON_NONE 0 |
246 | #define WT_REASON_INTEGRITY_FAIL 1 |
247 | #define WT_REASON_BAD_SYSCALL 2 |
248 | #define WT_REASON_NOT_LOCKED 3 |
249 | #define WT_REASON_ALREADY_LOCKED 4 |
250 | #define WT_REASON_SW_REQ 5 |
251 | #define WT_REASON_PT_INVALID 6 |
252 | #define WT_REASON_PT_VIOLATION 7 |
253 | #define WT_REASON_REG_VIOLATION 8 |
254 | #endif |
255 | |
256 | #if defined(HAS_IPI) |
257 | void cpu_signal_handler(void); |
258 | extern unsigned int gFastIPI; |
259 | #endif /* defined(HAS_IPI) */ |
260 | |
261 | static arm_saved_state64_t *original_faulting_state = NULL; |
262 | |
263 | |
264 | TUNABLE(bool, fp_exceptions_enabled, "-fp_exceptions" , false); |
265 | |
266 | extern vm_offset_t static_memory_end; |
267 | |
268 | /* |
269 | * Fault copyio_recovery_entry in copyin/copyout routines. |
270 | * |
271 | * Offets are expressed in bytes from ©_recovery_table |
272 | */ |
273 | struct copyio_recovery_entry { |
274 | ptrdiff_t cre_start; |
275 | ptrdiff_t cre_end; |
276 | ptrdiff_t cre_recovery; |
277 | }; |
278 | |
279 | extern struct copyio_recovery_entry copyio_recover_table[]; |
280 | extern struct copyio_recovery_entry copyio_recover_table_end[]; |
281 | |
282 | static inline ptrdiff_t |
283 | copyio_recovery_offset(uintptr_t addr) |
284 | { |
285 | return (ptrdiff_t)(addr - (uintptr_t)copyio_recover_table); |
286 | } |
287 | |
288 | #if !HAS_APPLE_PAC |
289 | static inline uintptr_t |
290 | copyio_recovery_addr(ptrdiff_t offset) |
291 | { |
292 | return (uintptr_t)copyio_recover_table + (uintptr_t)offset; |
293 | } |
294 | #endif |
295 | |
296 | static inline struct copyio_recovery_entry * |
297 | find_copyio_recovery_entry(arm_saved_state_t *state) |
298 | { |
299 | ptrdiff_t offset = copyio_recovery_offset(addr: get_saved_state_pc(iss: state)); |
300 | struct copyio_recovery_entry *e; |
301 | |
302 | for (e = copyio_recover_table; e < copyio_recover_table_end; e++) { |
303 | if (offset >= e->cre_start && offset < e->cre_end) { |
304 | return e; |
305 | } |
306 | } |
307 | |
308 | return NULL; |
309 | } |
310 | |
311 | static inline int |
312 | is_vm_fault(fault_status_t status) |
313 | { |
314 | switch (status) { |
315 | case FSC_TRANSLATION_FAULT_L0: |
316 | case FSC_TRANSLATION_FAULT_L1: |
317 | case FSC_TRANSLATION_FAULT_L2: |
318 | case FSC_TRANSLATION_FAULT_L3: |
319 | case FSC_ACCESS_FLAG_FAULT_L1: |
320 | case FSC_ACCESS_FLAG_FAULT_L2: |
321 | case FSC_ACCESS_FLAG_FAULT_L3: |
322 | case FSC_PERMISSION_FAULT_L1: |
323 | case FSC_PERMISSION_FAULT_L2: |
324 | case FSC_PERMISSION_FAULT_L3: |
325 | return TRUE; |
326 | default: |
327 | return FALSE; |
328 | } |
329 | } |
330 | |
331 | static inline int |
332 | is_translation_fault(fault_status_t status) |
333 | { |
334 | switch (status) { |
335 | case FSC_TRANSLATION_FAULT_L0: |
336 | case FSC_TRANSLATION_FAULT_L1: |
337 | case FSC_TRANSLATION_FAULT_L2: |
338 | case FSC_TRANSLATION_FAULT_L3: |
339 | return TRUE; |
340 | default: |
341 | return FALSE; |
342 | } |
343 | } |
344 | |
345 | static inline int |
346 | is_permission_fault(fault_status_t status) |
347 | { |
348 | switch (status) { |
349 | case FSC_PERMISSION_FAULT_L1: |
350 | case FSC_PERMISSION_FAULT_L2: |
351 | case FSC_PERMISSION_FAULT_L3: |
352 | return TRUE; |
353 | default: |
354 | return FALSE; |
355 | } |
356 | } |
357 | |
358 | static inline int |
359 | is_alignment_fault(fault_status_t status) |
360 | { |
361 | return status == FSC_ALIGNMENT_FAULT; |
362 | } |
363 | |
364 | static inline int |
365 | is_parity_error(fault_status_t status) |
366 | { |
367 | switch (status) { |
368 | #if defined(ARM64_BOARD_CONFIG_T6020) |
369 | /* |
370 | * H14 Erratum (rdar://61553243): Despite having FEAT_RAS implemented, |
371 | * FSC_SYNC_PARITY_X can be reported for data and instruction aborts |
372 | * and should be interpreted as FSC_SYNC_EXT_ABORT_x |
373 | */ |
374 | #else |
375 | /* |
376 | * TODO: According to ARM ARM, Async Parity (0b011001) is a DFSC that is |
377 | * only applicable to AArch32 HSR register. Can this be removed? |
378 | */ |
379 | case FSC_ASYNC_PARITY: |
380 | case FSC_SYNC_PARITY: |
381 | case FSC_SYNC_PARITY_TT_L1: |
382 | case FSC_SYNC_PARITY_TT_L2: |
383 | case FSC_SYNC_PARITY_TT_L3: |
384 | return TRUE; |
385 | #endif |
386 | default: |
387 | return FALSE; |
388 | } |
389 | } |
390 | |
391 | |
392 | |
393 | __dead2 __unused |
394 | static void |
395 | arm64_implementation_specific_error(arm_saved_state_t *state, uint32_t esr, vm_offset_t far) |
396 | { |
397 | #pragma unused (state, esr, far) |
398 | panic_plain("Unhandled implementation specific error\n" ); |
399 | } |
400 | |
401 | #if CONFIG_KERNEL_INTEGRITY |
402 | #pragma clang diagnostic push |
403 | #pragma clang diagnostic ignored "-Wunused-parameter" |
404 | static void |
405 | kernel_integrity_error_handler(uint32_t esr, vm_offset_t far) |
406 | { |
407 | #if defined(KERNEL_INTEGRITY_WT) |
408 | #if (DEVELOPMENT || DEBUG) |
409 | if (ESR_WT_SERROR(esr)) { |
410 | switch (ESR_WT_REASON(esr)) { |
411 | case WT_REASON_INTEGRITY_FAIL: |
412 | panic_plain("Kernel integrity, violation in frame 0x%016lx." , far); |
413 | case WT_REASON_BAD_SYSCALL: |
414 | panic_plain("Kernel integrity, bad syscall." ); |
415 | case WT_REASON_NOT_LOCKED: |
416 | panic_plain("Kernel integrity, not locked." ); |
417 | case WT_REASON_ALREADY_LOCKED: |
418 | panic_plain("Kernel integrity, already locked." ); |
419 | case WT_REASON_SW_REQ: |
420 | panic_plain("Kernel integrity, software request." ); |
421 | case WT_REASON_PT_INVALID: |
422 | panic_plain("Kernel integrity, encountered invalid TTE/PTE while " |
423 | "walking 0x%016lx." , far); |
424 | case WT_REASON_PT_VIOLATION: |
425 | panic_plain("Kernel integrity, violation in mapping 0x%016lx." , |
426 | far); |
427 | case WT_REASON_REG_VIOLATION: |
428 | panic_plain("Kernel integrity, violation in system register %d." , |
429 | (unsigned) far); |
430 | default: |
431 | panic_plain("Kernel integrity, unknown (esr=0x%08x)." , esr); |
432 | } |
433 | } |
434 | #else |
435 | if (ESR_WT_SERROR(esr)) { |
436 | panic_plain("SError esr: 0x%08x far: 0x%016lx." , esr, far); |
437 | } |
438 | #endif |
439 | #endif |
440 | } |
441 | #pragma clang diagnostic pop |
442 | #endif |
443 | |
444 | static void |
445 | arm64_platform_error(arm_saved_state_t *state, uint32_t esr, vm_offset_t far, platform_error_source_t source) |
446 | { |
447 | #if CONFIG_KERNEL_INTEGRITY |
448 | kernel_integrity_error_handler(esr, far); |
449 | #endif |
450 | |
451 | (void)source; |
452 | cpu_data_t *cdp = getCpuDatap(); |
453 | |
454 | if (PE_handle_platform_error(far)) { |
455 | return; |
456 | } else if (cdp->platform_error_handler != NULL) { |
457 | cdp->platform_error_handler(cdp->cpu_id, far); |
458 | } else { |
459 | arm64_implementation_specific_error(state, esr, far); |
460 | } |
461 | } |
462 | |
463 | void |
464 | panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss) |
465 | { |
466 | boolean_t ss_valid; |
467 | |
468 | ss_valid = is_saved_state64(iss: ss); |
469 | arm_saved_state64_t *state = saved_state64(iss: ss); |
470 | |
471 | os_atomic_cmpxchg(&original_faulting_state, NULL, state, seq_cst); |
472 | |
473 | // rdar://80659177 |
474 | // Read SoCD tracepoints up to twice — once the first time we call panic and |
475 | // another time if we encounter a nested panic after that. |
476 | static int twice = 2; |
477 | if (twice > 0) { |
478 | twice--; |
479 | SOCD_TRACE_XNU(KERNEL_STATE_PANIC, ADDR(state->pc), |
480 | PACK_LSB(VALUE(state->lr), VALUE(ss_valid)), |
481 | PACK_2X32(VALUE(state->esr), VALUE(state->cpsr)), |
482 | VALUE(state->far)); |
483 | } |
484 | |
485 | |
486 | panic_plain("%s at pc 0x%016llx, lr 0x%016llx (saved state: %p%s)\n" |
487 | "\t x0: 0x%016llx x1: 0x%016llx x2: 0x%016llx x3: 0x%016llx\n" |
488 | "\t x4: 0x%016llx x5: 0x%016llx x6: 0x%016llx x7: 0x%016llx\n" |
489 | "\t x8: 0x%016llx x9: 0x%016llx x10: 0x%016llx x11: 0x%016llx\n" |
490 | "\t x12: 0x%016llx x13: 0x%016llx x14: 0x%016llx x15: 0x%016llx\n" |
491 | "\t x16: 0x%016llx x17: 0x%016llx x18: 0x%016llx x19: 0x%016llx\n" |
492 | "\t x20: 0x%016llx x21: 0x%016llx x22: 0x%016llx x23: 0x%016llx\n" |
493 | "\t x24: 0x%016llx x25: 0x%016llx x26: 0x%016llx x27: 0x%016llx\n" |
494 | "\t x28: 0x%016llx fp: 0x%016llx lr: 0x%016llx sp: 0x%016llx\n" |
495 | "\t pc: 0x%016llx cpsr: 0x%08x esr: 0x%08x far: 0x%016llx\n" , |
496 | msg, state->pc, state->lr, ss, (ss_valid ? "" : " INVALID" ), |
497 | state->x[0], state->x[1], state->x[2], state->x[3], |
498 | state->x[4], state->x[5], state->x[6], state->x[7], |
499 | state->x[8], state->x[9], state->x[10], state->x[11], |
500 | state->x[12], state->x[13], state->x[14], state->x[15], |
501 | state->x[16], state->x[17], state->x[18], state->x[19], |
502 | state->x[20], state->x[21], state->x[22], state->x[23], |
503 | state->x[24], state->x[25], state->x[26], state->x[27], |
504 | state->x[28], state->fp, state->lr, state->sp, |
505 | state->pc, state->cpsr, state->esr, state->far); |
506 | } |
507 | |
508 | void |
509 | sleh_synchronous_sp1(arm_context_t *context, uint32_t esr, vm_offset_t far __unused) |
510 | { |
511 | esr_exception_class_t class = ESR_EC(esr); |
512 | arm_saved_state_t * state = &context->ss; |
513 | |
514 | switch (class) { |
515 | case ESR_EC_UNCATEGORIZED: |
516 | { |
517 | #if (DEVELOPMENT || DEBUG) |
518 | uint32_t instr = *((uint32_t*)get_saved_state_pc(state)); |
519 | if (IS_ARM_GDB_TRAP(instr)) { |
520 | DebuggerCall(EXC_BREAKPOINT, state); |
521 | } |
522 | OS_FALLTHROUGH; // panic if we return from the debugger |
523 | #else |
524 | panic_with_thread_kernel_state(msg: "Unexpected debugger trap while SP1 selected" , ss: state); |
525 | #endif /* (DEVELOPMENT || DEBUG) */ |
526 | } |
527 | default: |
528 | panic_with_thread_kernel_state(msg: "Synchronous exception taken while SP1 selected" , ss: state); |
529 | } |
530 | } |
531 | |
532 | |
533 | __attribute__((noreturn)) |
534 | void |
535 | thread_exception_return() |
536 | { |
537 | thread_t thread = current_thread(); |
538 | if (thread->machine.exception_trace_code != 0) { |
539 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, |
540 | MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_END, 0, 0, 0, 0, 0); |
541 | thread->machine.exception_trace_code = 0; |
542 | } |
543 | |
544 | #if KASAN_TBI |
545 | kasan_unpoison_curstack(true); |
546 | #endif /* KASAN_TBI */ |
547 | arm64_thread_exception_return(); |
548 | __builtin_unreachable(); |
549 | } |
550 | |
551 | /* |
552 | * check whether task vtimers are running and set thread and CPU BSD AST |
553 | * |
554 | * must be called with interrupts masked so updates of fields are atomic |
555 | * must be emitted inline to avoid generating an FBT probe on the exception path |
556 | * |
557 | */ |
558 | __attribute__((__always_inline__)) |
559 | static inline void |
560 | task_vtimer_check(thread_t thread) |
561 | { |
562 | task_t task = get_threadtask_early(thread); |
563 | |
564 | if (__improbable(task != NULL && task->vtimers)) { |
565 | thread_ast_set(thread, AST_BSD); |
566 | thread->machine.CpuDatap->cpu_pending_ast |= AST_BSD; |
567 | } |
568 | } |
569 | |
570 | #if MACH_ASSERT |
571 | /** |
572 | * A version of get_preemption_level() that works in early boot. |
573 | * |
574 | * If an exception is raised in early boot before the initial thread has been |
575 | * set up, then calling get_preemption_level() in the SLEH will trigger an |
576 | * infinitely-recursing exception. This function handles this edge case. |
577 | */ |
578 | static inline int |
579 | sleh_get_preemption_level(void) |
580 | { |
581 | if (__improbable(current_thread() == NULL)) { |
582 | return 0; |
583 | } |
584 | return get_preemption_level(); |
585 | } |
586 | #endif // MACH_ASSERT |
587 | |
588 | static inline bool |
589 | is_platform_error(uint32_t esr) |
590 | { |
591 | esr_exception_class_t class = ESR_EC(esr); |
592 | uint32_t iss = ESR_ISS(esr); |
593 | fault_status_t fault_code; |
594 | |
595 | if (class == ESR_EC_DABORT_EL0 || class == ESR_EC_DABORT_EL1) { |
596 | fault_code = ISS_DA_FSC(iss); |
597 | } else if (class == ESR_EC_IABORT_EL0 || class == ESR_EC_IABORT_EL1) { |
598 | fault_code = ISS_IA_FSC(iss); |
599 | } else { |
600 | return false; |
601 | } |
602 | |
603 | return fault_code == FSC_SYNC_PARITY; |
604 | } |
605 | |
606 | void |
607 | sleh_synchronous(arm_context_t *context, uint32_t esr, vm_offset_t far, __unused bool did_initiate_panic_lockdown) |
608 | { |
609 | esr_exception_class_t class = ESR_EC(esr); |
610 | arm_saved_state_t * state = &context->ss; |
611 | thread_t thread = current_thread(); |
612 | #if MACH_ASSERT |
613 | int preemption_level = sleh_get_preemption_level(); |
614 | #endif |
615 | expected_fault_handler_t expected_fault_handler = NULL; |
616 | #ifdef CONFIG_XNUPOST |
617 | expected_fault_handler_t saved_expected_fault_handler = NULL; |
618 | uintptr_t saved_expected_fault_addr = 0; |
619 | uintptr_t saved_expected_fault_pc = 0; |
620 | #endif /* CONFIG_XNUPOST */ |
621 | |
622 | ASSERT_CONTEXT_SANITY(context); |
623 | |
624 | task_vtimer_check(thread); |
625 | |
626 | #if CONFIG_DTRACE |
627 | /* |
628 | * Handle kernel DTrace probes as early as possible to minimize the likelihood |
629 | * that this path will itself trigger a DTrace probe, which would lead to infinite |
630 | * probe recursion. |
631 | */ |
632 | if (__improbable((class == ESR_EC_UNCATEGORIZED) && tempDTraceTrapHook && |
633 | (tempDTraceTrapHook(EXC_BAD_INSTRUCTION, state, 0, 0) == KERN_SUCCESS))) { |
634 | #if CONFIG_SPTM |
635 | if (__improbable(did_initiate_panic_lockdown)) { |
636 | panic("Unexpectedly initiated lockdown for DTrace probe?" ); |
637 | } |
638 | #endif |
639 | return; |
640 | } |
641 | #endif |
642 | bool is_user = PSR64_IS_USER(get_saved_state_cpsr(state)); |
643 | |
644 | #if CONFIG_SPTM |
645 | // Lockdown should only be initiated for kernel exceptions |
646 | assert(!(is_user && did_initiate_panic_lockdown)); |
647 | #endif /* CONFIG_SPTM */ |
648 | |
649 | /* |
650 | * Use KERNEL_DEBUG_CONSTANT_IST here to avoid producing tracepoints |
651 | * that would disclose the behavior of PT_DENY_ATTACH processes. |
652 | */ |
653 | if (is_user) { |
654 | /* Sanitize FAR (but only if the exception was taken from userspace) */ |
655 | switch (class) { |
656 | case ESR_EC_IABORT_EL1: |
657 | case ESR_EC_IABORT_EL0: |
658 | /* If this is a SEA, since we can't trust FnV, just clear FAR from the save area. */ |
659 | if (ISS_IA_FSC(ESR_ISS(esr)) == FSC_SYNC_EXT_ABORT) { |
660 | saved_state64(iss: state)->far = 0; |
661 | } |
662 | break; |
663 | case ESR_EC_DABORT_EL1: |
664 | case ESR_EC_DABORT_EL0: |
665 | /* If this is a SEA, since we can't trust FnV, just clear FAR from the save area. */ |
666 | if (ISS_DA_FSC(ESR_ISS(esr)) == FSC_SYNC_EXT_ABORT) { |
667 | saved_state64(iss: state)->far = 0; |
668 | } |
669 | break; |
670 | case ESR_EC_WATCHPT_MATCH_EL1: |
671 | case ESR_EC_WATCHPT_MATCH_EL0: |
672 | case ESR_EC_PC_ALIGN: |
673 | break; /* FAR_ELx is valid */ |
674 | default: |
675 | saved_state64(iss: state)->far = 0; |
676 | break; |
677 | } |
678 | |
679 | thread->machine.exception_trace_code = (uint16_t)(ARM64_KDBG_CODE_USER | class); |
680 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, |
681 | MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_START, |
682 | esr, far, get_saved_state_pc(state), 0, 0); |
683 | } else { |
684 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, |
685 | MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, ARM64_KDBG_CODE_KERNEL | class) | DBG_FUNC_START, |
686 | esr, VM_KERNEL_ADDRHIDE(far), VM_KERNEL_UNSLIDE(get_saved_state_pc(state)), 0, 0); |
687 | } |
688 | |
689 | if (__improbable(ESR_INSTR_IS_2BYTES(esr))) { |
690 | /* |
691 | * We no longer support 32-bit, which means no 2-byte |
692 | * instructions. |
693 | */ |
694 | if (is_user) { |
695 | panic("Exception on 2-byte instruction, " |
696 | "context=%p, esr=%#x, far=%p" , |
697 | context, esr, (void *)far); |
698 | } else { |
699 | panic_with_thread_kernel_state(msg: "Exception on 2-byte instruction" , ss: state); |
700 | } |
701 | } |
702 | |
703 | #ifdef CONFIG_XNUPOST |
704 | if (thread->machine.expected_fault_handler != NULL) { |
705 | bool matching_fault_pc = false; |
706 | saved_expected_fault_handler = thread->machine.expected_fault_handler; |
707 | saved_expected_fault_addr = thread->machine.expected_fault_addr; |
708 | saved_expected_fault_pc = thread->machine.expected_fault_pc; |
709 | |
710 | thread->machine.expected_fault_handler = NULL; |
711 | thread->machine.expected_fault_addr = 0; |
712 | thread->machine.expected_fault_pc = 0; |
713 | |
714 | #if __has_feature(ptrauth_calls) |
715 | /* |
716 | * Compare only the bits of PC which make up the virtual address. |
717 | * This ignores the upper bits, which may have been corrupted by HW in |
718 | * platform dependent ways to signal pointer authentication fault. |
719 | */ |
720 | uint64_t fault_addr_mask = fault_addr_bitmask(0, 64 - T1SZ_BOOT - 1); |
721 | uint64_t masked_expected_pc = saved_expected_fault_pc & fault_addr_mask; |
722 | uint64_t masked_saved_pc = get_saved_state_pc(state) & fault_addr_mask; |
723 | matching_fault_pc = masked_expected_pc == masked_saved_pc; |
724 | #else |
725 | matching_fault_pc = |
726 | (saved_expected_fault_pc == get_saved_state_pc(state)); |
727 | #endif /* ptrauth_call */ |
728 | if (saved_expected_fault_addr == far || |
729 | matching_fault_pc) { |
730 | expected_fault_handler = saved_expected_fault_handler; |
731 | } |
732 | } |
733 | #endif /* CONFIG_XNUPOST */ |
734 | |
735 | if (__improbable(is_platform_error(esr))) { |
736 | /* |
737 | * Must gather error info in platform error handler before |
738 | * thread is preempted to another core/cluster to guarantee |
739 | * accurate error details |
740 | */ |
741 | |
742 | arm64_platform_error(state, esr, far, source: PLAT_ERR_SRC_SYNC); |
743 | #if CONFIG_SPTM |
744 | if (__improbable(did_initiate_panic_lockdown)) { |
745 | panic("Panic lockdown initiated for platform error" ); |
746 | } |
747 | #endif |
748 | return; |
749 | } |
750 | |
751 | if (is_user && class == ESR_EC_DABORT_EL0) { |
752 | thread_reset_pcs_will_fault(thread); |
753 | } |
754 | |
755 | /* Inherit the interrupt masks from previous context */ |
756 | if (SPSR_INTERRUPTS_ENABLED(get_saved_state_cpsr(state))) { |
757 | ml_set_interrupts_enabled(TRUE); |
758 | } |
759 | |
760 | switch (class) { |
761 | case ESR_EC_SVC_64: |
762 | if (!is_saved_state64(iss: state) || !is_user) { |
763 | panic("Invalid SVC_64 context" ); |
764 | } |
765 | |
766 | handle_svc(state); |
767 | break; |
768 | |
769 | case ESR_EC_DABORT_EL0: |
770 | handle_abort(state, esr, far, inspect_data_abort, handle_user_abort, expected_fault_handler); |
771 | break; |
772 | |
773 | case ESR_EC_MSR_TRAP: |
774 | handle_msr_trap(state, esr); |
775 | break; |
776 | /** |
777 | * Some APPLEVIRTUALPLATFORM targets do not specify armv8.6, but it's still possible for |
778 | * them to be hosted by a host that implements ARM_FPAC. There's no way for such a host |
779 | * to disable it or trap it without substantial performance penalty. Therefore, the FPAC |
780 | * handler here needs to be built into the guest kernels to prevent the exception to fall |
781 | * through. |
782 | */ |
783 | #if __has_feature(ptrauth_calls) |
784 | case ESR_EC_PAC_FAIL: |
785 | #ifdef CONFIG_XNUPOST |
786 | if (expected_fault_handler != NULL && expected_fault_handler(state)) { |
787 | break; |
788 | } |
789 | #endif /* CONFIG_XNUPOST */ |
790 | handle_pac_fail(state, esr); |
791 | __builtin_unreachable(); |
792 | |
793 | #endif /* __has_feature(ptrauth_calls) */ |
794 | |
795 | |
796 | case ESR_EC_IABORT_EL0: |
797 | handle_abort(state, esr, far, inspect_instruction_abort, handle_user_abort, expected_fault_handler); |
798 | break; |
799 | |
800 | case ESR_EC_IABORT_EL1: |
801 | #ifdef CONFIG_XNUPOST |
802 | if ((expected_fault_handler != NULL) && expected_fault_handler(state)) { |
803 | break; |
804 | } |
805 | #endif /* CONFIG_XNUPOST */ |
806 | |
807 | panic_with_thread_kernel_state(msg: "Kernel instruction fetch abort" , ss: state); |
808 | |
809 | case ESR_EC_PC_ALIGN: |
810 | handle_pc_align(ss: state); |
811 | __builtin_unreachable(); |
812 | |
813 | case ESR_EC_DABORT_EL1: |
814 | handle_abort(state, esr, far, inspect_data_abort, handle_kernel_abort, expected_fault_handler); |
815 | break; |
816 | |
817 | case ESR_EC_UNCATEGORIZED: |
818 | assert(!ESR_ISS(esr)); |
819 | |
820 | #if CONFIG_XNUPOST |
821 | if (!is_user && (expected_fault_handler != NULL) && expected_fault_handler(state)) { |
822 | /* |
823 | * The fault handler accepted the exception and handled it on its |
824 | * own. Don't trap to the debugger/panic. |
825 | */ |
826 | break; |
827 | } |
828 | #endif /* CONFIG_XNUPOST */ |
829 | handle_uncategorized(&context->ss); |
830 | break; |
831 | |
832 | case ESR_EC_SP_ALIGN: |
833 | handle_sp_align(ss: state); |
834 | __builtin_unreachable(); |
835 | |
836 | case ESR_EC_BKPT_AARCH32: |
837 | handle_breakpoint(state, esr); |
838 | __builtin_unreachable(); |
839 | |
840 | case ESR_EC_BRK_AARCH64: |
841 | #ifdef CONFIG_XNUPOST |
842 | if ((expected_fault_handler != NULL) && expected_fault_handler(state)) { |
843 | break; |
844 | } |
845 | #endif /* CONFIG_XNUPOST */ |
846 | if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { |
847 | handle_kernel_breakpoint(state, esr); |
848 | break; |
849 | } else { |
850 | handle_breakpoint(state, esr); |
851 | __builtin_unreachable(); |
852 | } |
853 | |
854 | case ESR_EC_BKPT_REG_MATCH_EL0: |
855 | if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) { |
856 | handle_breakpoint(state, esr); |
857 | } |
858 | panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p" , |
859 | class, state, class, esr, (void *)far); |
860 | __builtin_unreachable(); |
861 | |
862 | case ESR_EC_BKPT_REG_MATCH_EL1: |
863 | panic_with_thread_kernel_state(msg: "Hardware Breakpoint Debug exception from kernel. Panic (by design)" , ss: state); |
864 | __builtin_unreachable(); |
865 | |
866 | case ESR_EC_SW_STEP_DEBUG_EL0: |
867 | if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) { |
868 | handle_sw_step_debug(ss: state); |
869 | } |
870 | panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p" , |
871 | class, state, class, esr, (void *)far); |
872 | __builtin_unreachable(); |
873 | |
874 | case ESR_EC_SW_STEP_DEBUG_EL1: |
875 | panic_with_thread_kernel_state(msg: "Software Step Debug exception from kernel. Panic (by design)" , ss: state); |
876 | __builtin_unreachable(); |
877 | |
878 | case ESR_EC_WATCHPT_MATCH_EL0: |
879 | if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) { |
880 | handle_watchpoint(fault_addr: far); |
881 | } |
882 | panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p" , |
883 | class, state, class, esr, (void *)far); |
884 | __builtin_unreachable(); |
885 | |
886 | case ESR_EC_WATCHPT_MATCH_EL1: |
887 | /* |
888 | * If we hit a watchpoint in kernel mode, probably in a copyin/copyout which we don't want to |
889 | * abort. Turn off watchpoints and keep going; we'll turn them back on in return_from_exception.. |
890 | */ |
891 | if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) { |
892 | arm_debug_set(NULL); |
893 | break; /* return to first level handler */ |
894 | } |
895 | panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p" , |
896 | class, state, class, esr, (void *)far); |
897 | __builtin_unreachable(); |
898 | |
899 | case ESR_EC_TRAP_SIMD_FP: |
900 | handle_simd_trap(state, esr); |
901 | __builtin_unreachable(); |
902 | |
903 | case ESR_EC_ILLEGAL_INSTR_SET: |
904 | panic("Illegal instruction set exception. state=%p class=%u esr=%u far=%p spsr=0x%x" , |
905 | state, class, esr, (void *)far, get_saved_state_cpsr(state)); |
906 | __builtin_unreachable(); |
907 | |
908 | case ESR_EC_MCR_MRC_CP15_TRAP: |
909 | case ESR_EC_MCRR_MRRC_CP15_TRAP: |
910 | case ESR_EC_MCR_MRC_CP14_TRAP: |
911 | case ESR_EC_LDC_STC_CP14_TRAP: |
912 | case ESR_EC_MCRR_MRRC_CP14_TRAP: |
913 | handle_user_trapped_instruction32(state, esr); |
914 | __builtin_unreachable(); |
915 | |
916 | case ESR_EC_WFI_WFE: |
917 | // Use of WFI or WFE instruction when they have been disabled for EL0 |
918 | handle_wf_trap(ss: state); |
919 | __builtin_unreachable(); |
920 | |
921 | case ESR_EC_FLOATING_POINT_64: |
922 | handle_fp_trap(ss: state, esr); |
923 | __builtin_unreachable(); |
924 | |
925 | default: |
926 | handle_uncategorized(state); |
927 | } |
928 | |
929 | #ifdef CONFIG_XNUPOST |
930 | if (saved_expected_fault_handler != NULL) { |
931 | thread->machine.expected_fault_handler = saved_expected_fault_handler; |
932 | thread->machine.expected_fault_addr = saved_expected_fault_addr; |
933 | thread->machine.expected_fault_pc = saved_expected_fault_pc; |
934 | } |
935 | #endif /* CONFIG_XNUPOST */ |
936 | |
937 | if (is_user) { |
938 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, |
939 | MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_END, |
940 | esr, far, get_saved_state_pc(state), 0, 0); |
941 | thread->machine.exception_trace_code = 0; |
942 | } else { |
943 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, |
944 | MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, ARM64_KDBG_CODE_KERNEL | class) | DBG_FUNC_END, |
945 | esr, VM_KERNEL_ADDRHIDE(far), VM_KERNEL_UNSLIDE(get_saved_state_pc(state)), 0, 0); |
946 | } |
947 | #if MACH_ASSERT |
948 | if (preemption_level != sleh_get_preemption_level()) { |
949 | panic("synchronous exception changed preemption level from %d to %d" , preemption_level, sleh_get_preemption_level()); |
950 | } |
951 | #endif |
952 | |
953 | #if CONFIG_SPTM |
954 | if (did_initiate_panic_lockdown |
955 | #if CONFIG_XNUPOST |
956 | /* Do not engage the panic interlock if we matched a fault handler */ |
957 | && !expected_fault_handler |
958 | #endif /* CONFIG_XNUPOST */ |
959 | ) { |
960 | /* |
961 | * fleh already triggered a lockdown but we, for whatever reason, didn't |
962 | * end up finding a reason to panic. Catch all panic in this case. |
963 | * Note that the panic here has no security benefit as the system is already |
964 | * hosed, this is merely for telemetry. |
965 | */ |
966 | panic_with_thread_kernel_state("Panic lockdown initiated" , state); |
967 | } |
968 | #endif /* CONFIG_SPTM */ |
969 | } |
970 | |
971 | /* |
972 | * Uncategorized exceptions are a catch-all for general execution errors. |
973 | * ARM64_TODO: For now, we assume this is for undefined instruction exceptions. |
974 | */ |
975 | static void |
976 | handle_uncategorized(arm_saved_state_t *state) |
977 | { |
978 | exception_type_t exception = EXC_BAD_INSTRUCTION; |
979 | mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED}; |
980 | mach_msg_type_number_t numcodes = 2; |
981 | uint32_t instr = 0; |
982 | |
983 | COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr)); |
984 | |
985 | #if CONFIG_DTRACE |
986 | |
987 | if (PSR64_IS_USER64(get_saved_state_cpsr(state))) { |
988 | /* |
989 | * For a 64bit user process, we care about all 4 bytes of the |
990 | * instr. |
991 | */ |
992 | if (instr == FASTTRAP_ARM64_INSTR || instr == FASTTRAP_ARM64_RET_INSTR) { |
993 | if (dtrace_user_probe(regs: state) == KERN_SUCCESS) { |
994 | return; |
995 | } |
996 | } |
997 | } else if (PSR64_IS_USER32(get_saved_state_cpsr(state))) { |
998 | /* |
999 | * For a 32bit user process, we check for thumb mode, in |
1000 | * which case we only care about a 2 byte instruction length. |
1001 | * For non-thumb mode, we care about all 4 bytes of the instructin. |
1002 | */ |
1003 | if (get_saved_state_cpsr(iss: state) & PSR64_MODE_USER32_THUMB) { |
1004 | if (((uint16_t)instr == FASTTRAP_THUMB32_INSTR) || |
1005 | ((uint16_t)instr == FASTTRAP_THUMB32_RET_INSTR)) { |
1006 | if (dtrace_user_probe(regs: state) == KERN_SUCCESS) { |
1007 | return; |
1008 | } |
1009 | } |
1010 | } else { |
1011 | if ((instr == FASTTRAP_ARM32_INSTR) || |
1012 | (instr == FASTTRAP_ARM32_RET_INSTR)) { |
1013 | if (dtrace_user_probe(regs: state) == KERN_SUCCESS) { |
1014 | return; |
1015 | } |
1016 | } |
1017 | } |
1018 | } |
1019 | |
1020 | #endif /* CONFIG_DTRACE */ |
1021 | |
1022 | if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { |
1023 | if (IS_ARM_GDB_TRAP(instr)) { |
1024 | boolean_t interrupt_state; |
1025 | exception = EXC_BREAKPOINT; |
1026 | |
1027 | interrupt_state = ml_set_interrupts_enabled(FALSE); |
1028 | |
1029 | /* Save off the context here (so that the debug logic |
1030 | * can see the original state of this thread). |
1031 | */ |
1032 | current_thread()->machine.kpcb = state; |
1033 | |
1034 | /* Hop into the debugger (typically either due to a |
1035 | * fatal exception, an explicit panic, or a stackshot |
1036 | * request. |
1037 | */ |
1038 | DebuggerCall(reason: exception, ctx: state); |
1039 | |
1040 | current_thread()->machine.kpcb = NULL; |
1041 | (void) ml_set_interrupts_enabled(enable: interrupt_state); |
1042 | return; |
1043 | } else { |
1044 | panic("Undefined kernel instruction: pc=%p instr=%x" , (void*)get_saved_state_pc(state), instr); |
1045 | } |
1046 | } |
1047 | |
1048 | /* |
1049 | * Check for GDB breakpoint via illegal opcode. |
1050 | */ |
1051 | if (IS_ARM_GDB_TRAP(instr)) { |
1052 | exception = EXC_BREAKPOINT; |
1053 | codes[0] = EXC_ARM_BREAKPOINT; |
1054 | codes[1] = instr; |
1055 | } else { |
1056 | codes[1] = instr; |
1057 | } |
1058 | |
1059 | exception_triage(exception, code: codes, codeCnt: numcodes); |
1060 | __builtin_unreachable(); |
1061 | } |
1062 | |
1063 | #if __has_feature(ptrauth_calls) |
1064 | static const uint16_t PTRAUTH_TRAP_START = 0xC470; |
1065 | static inline bool |
1066 | brk_comment_is_ptrauth(uint16_t comment) |
1067 | { |
1068 | return comment >= PTRAUTH_TRAP_START && |
1069 | comment <= PTRAUTH_TRAP_START + ptrauth_key_asdb; |
1070 | } |
1071 | |
1072 | static inline const char * |
1073 | ptrauth_key_to_string(ptrauth_key key) |
1074 | { |
1075 | switch (key) { |
1076 | case ptrauth_key_asia: |
1077 | return "IA" ; |
1078 | case ptrauth_key_asib: |
1079 | return "IB" ; |
1080 | case ptrauth_key_asda: |
1081 | return "DA" ; |
1082 | case ptrauth_key_asdb: |
1083 | return "DB" ; |
1084 | default: |
1085 | __builtin_unreachable(); |
1086 | } |
1087 | } |
1088 | |
1089 | static void __attribute__((noreturn)) |
1090 | ptrauth_handle_brk_trap(void *tstate, uint16_t comment) |
1091 | { |
1092 | arm_saved_state_t *state = (arm_saved_state_t *)tstate; |
1093 | #define MSG_FMT "Break 0x%04X instruction exception from kernel. Ptrauth failure with %s key resulted in 0x%016llx" |
1094 | char msg[strlen(MSG_FMT) |
1095 | - strlen("0x%04X" ) + strlen("0xFFFF" ) |
1096 | - strlen("%s" ) + strlen("IA" ) |
1097 | - strlen("0x%016llx" ) + strlen("0xFFFFFFFFFFFFFFFF" ) |
1098 | + 1]; |
1099 | ptrauth_key key = (ptrauth_key)(comment - PTRAUTH_TRAP_START); |
1100 | const char *key_str = ptrauth_key_to_string(key); |
1101 | snprintf(msg, sizeof(msg), MSG_FMT, comment, key_str, saved_state64(state)->x[16]); |
1102 | #undef MSG_FMT |
1103 | |
1104 | panic_with_thread_kernel_state(msg, state); |
1105 | __builtin_unreachable(); |
1106 | } |
1107 | #endif /* __has_feature(ptrauth_calls) */ |
1108 | |
1109 | #if HAS_TELEMETRY_KERNEL_BRK |
1110 | static uint32_t bound_chk_violations_event; |
1111 | |
1112 | static void |
1113 | xnu_soft_trap_handle_breakpoint( |
1114 | void *tstate, |
1115 | uint16_t comment) |
1116 | { |
1117 | #if CONFIG_UBSAN_MINIMAL |
1118 | if (comment == UBSAN_SOFT_TRAP_SIGNED_OF) { |
1119 | ubsan_handle_brk_trap(tstate, comment); |
1120 | } |
1121 | #else |
1122 | (void)tstate; |
1123 | #endif |
1124 | |
1125 | if (comment == CLANG_SOFT_TRAP_BOUND_CHK) { |
1126 | os_atomic_inc(&bound_chk_violations_event, relaxed); |
1127 | } |
1128 | } |
1129 | #endif /* HAS_TELEMETRY_KERNEL_BRK */ |
1130 | |
1131 | static void |
1132 | xnu_hard_trap_handle_breakpoint(void *tstate, uint16_t ) |
1133 | { |
1134 | switch (comment) { |
1135 | case XNU_HARD_TRAP_STRING_CHK: |
1136 | panic_with_thread_kernel_state(msg: "panic: string operation caused an overflow" , ss: tstate); |
1137 | default: |
1138 | break; |
1139 | } |
1140 | } |
1141 | |
1142 | #if __has_feature(ptrauth_calls) |
1143 | KERNEL_BRK_DESCRIPTOR_DEFINE(ptrauth_desc, |
1144 | .type = KERNEL_BRK_TYPE_PTRAUTH, |
1145 | .base = PTRAUTH_TRAP_START, |
1146 | .max = PTRAUTH_TRAP_START + ptrauth_key_asdb, |
1147 | .options = KERNEL_BRK_UNRECOVERABLE, |
1148 | .handle_breakpoint = ptrauth_handle_brk_trap); |
1149 | #endif |
1150 | |
1151 | KERNEL_BRK_DESCRIPTOR_DEFINE(clang_desc, |
1152 | .type = KERNEL_BRK_TYPE_CLANG, |
1153 | .base = CLANG_ARM_TRAP_START, |
1154 | .max = CLANG_ARM_TRAP_END, |
1155 | .options = KERNEL_BRK_UNRECOVERABLE, |
1156 | .handle_breakpoint = NULL); |
1157 | |
1158 | KERNEL_BRK_DESCRIPTOR_DEFINE(libcxx_desc, |
1159 | .type = KERNEL_BRK_TYPE_LIBCXX, |
1160 | .base = LIBCXX_TRAP_START, |
1161 | .max = LIBCXX_TRAP_END, |
1162 | .options = KERNEL_BRK_UNRECOVERABLE, |
1163 | .handle_breakpoint = NULL); |
1164 | |
1165 | #if HAS_TELEMETRY_KERNEL_BRK |
1166 | KERNEL_BRK_DESCRIPTOR_DEFINE(xnu_soft_traps_desc, |
1167 | .type = KERNEL_BRK_TYPE_TELEMETRY, |
1168 | .base = XNU_SOFT_TRAP_START, |
1169 | .max = XNU_SOFT_TRAP_END, |
1170 | .options = KERNEL_BRK_RECOVERABLE | KERNEL_BRK_CORE_ANALYTICS, |
1171 | .handle_breakpoint = xnu_soft_trap_handle_breakpoint); |
1172 | #endif /* HAS_TELEMETRY_KERNEL_BRK */ |
1173 | |
1174 | KERNEL_BRK_DESCRIPTOR_DEFINE(xnu_hard_traps_desc, |
1175 | .type = KERNEL_BRK_TYPE_XNU, |
1176 | .base = XNU_HARD_TRAP_START, |
1177 | .max = XNU_HARD_TRAP_END, |
1178 | .options = KERNEL_BRK_UNRECOVERABLE, |
1179 | .handle_breakpoint = xnu_hard_trap_handle_breakpoint); |
1180 | |
1181 | static void |
1182 | #if !HAS_TELEMETRY_KERNEL_BRK |
1183 | __attribute__((noreturn)) |
1184 | #endif |
1185 | handle_kernel_breakpoint(arm_saved_state_t *state, uint32_t esr) |
1186 | { |
1187 | uint16_t = ISS_BRK_COMMENT(esr); |
1188 | const struct kernel_brk_descriptor *desc; |
1189 | |
1190 | #define MSG_FMT "Break 0x%04X instruction exception from kernel. Panic (by design)" |
1191 | char msg[strlen(MSG_FMT) - strlen(s: "0x%04X" ) + strlen(s: "0xFFFF" ) + 1]; |
1192 | |
1193 | desc = find_brk_descriptor_by_comment(comment); |
1194 | |
1195 | if (!desc) { |
1196 | goto brk_out; |
1197 | } |
1198 | |
1199 | #if HAS_TELEMETRY_KERNEL_BRK |
1200 | if (desc->options & KERNEL_BRK_TELEMETRY_OPTIONS) { |
1201 | telemetry_kernel_brk(desc->type, desc->options, (void *)state, comment); |
1202 | } |
1203 | #endif |
1204 | |
1205 | if (desc->handle_breakpoint) { |
1206 | desc->handle_breakpoint(state, comment); /* May trigger panic */ |
1207 | } |
1208 | |
1209 | #if HAS_TELEMETRY_KERNEL_BRK |
1210 | /* Still alive? Check if we should recover. */ |
1211 | if (desc->options & KERNEL_BRK_RECOVERABLE) { |
1212 | add_saved_state_pc(state, 4); |
1213 | return; |
1214 | } |
1215 | #endif |
1216 | |
1217 | brk_out: |
1218 | snprintf(msg, sizeof(msg), MSG_FMT, comment); |
1219 | |
1220 | panic_with_thread_kernel_state(msg, ss: state); |
1221 | __builtin_unreachable(); |
1222 | #undef MSG_FMT |
1223 | } |
1224 | |
1225 | static void |
1226 | handle_breakpoint(arm_saved_state_t *state, uint32_t esr __unused) |
1227 | { |
1228 | exception_type_t exception = EXC_BREAKPOINT; |
1229 | mach_exception_data_type_t codes[2] = {EXC_ARM_BREAKPOINT}; |
1230 | mach_msg_type_number_t numcodes = 2; |
1231 | |
1232 | #if __has_feature(ptrauth_calls) |
1233 | if (ESR_EC(esr) == ESR_EC_BRK_AARCH64 && |
1234 | brk_comment_is_ptrauth(ISS_BRK_COMMENT(esr))) { |
1235 | exception |= EXC_PTRAUTH_BIT; |
1236 | } |
1237 | #endif /* __has_feature(ptrauth_calls) */ |
1238 | |
1239 | codes[1] = get_saved_state_pc(iss: state); |
1240 | exception_triage(exception, code: codes, codeCnt: numcodes); |
1241 | __builtin_unreachable(); |
1242 | } |
1243 | |
1244 | static void |
1245 | handle_watchpoint(vm_offset_t fault_addr) |
1246 | { |
1247 | exception_type_t exception = EXC_BREAKPOINT; |
1248 | mach_exception_data_type_t codes[2] = {EXC_ARM_DA_DEBUG}; |
1249 | mach_msg_type_number_t numcodes = 2; |
1250 | |
1251 | codes[1] = fault_addr; |
1252 | exception_triage(exception, code: codes, codeCnt: numcodes); |
1253 | __builtin_unreachable(); |
1254 | } |
1255 | |
1256 | static void |
1257 | handle_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr, |
1258 | abort_inspector_t inspect_abort, abort_handler_t handler, expected_fault_handler_t expected_fault_handler) |
1259 | { |
1260 | fault_status_t fault_code; |
1261 | vm_prot_t fault_type; |
1262 | |
1263 | inspect_abort(ESR_ISS(esr), &fault_code, &fault_type); |
1264 | handler(state, esr, fault_addr, fault_code, fault_type, expected_fault_handler); |
1265 | } |
1266 | |
1267 | static void |
1268 | inspect_instruction_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type) |
1269 | { |
1270 | getCpuDatap()->cpu_stat.instr_ex_cnt++; |
1271 | *fault_code = ISS_IA_FSC(iss); |
1272 | *fault_type = (VM_PROT_READ | VM_PROT_EXECUTE); |
1273 | } |
1274 | |
1275 | static void |
1276 | inspect_data_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type) |
1277 | { |
1278 | getCpuDatap()->cpu_stat.data_ex_cnt++; |
1279 | *fault_code = ISS_DA_FSC(iss); |
1280 | |
1281 | /* |
1282 | * Cache maintenance operations always report faults as write access. |
1283 | * Change these to read access, unless they report a permission fault. |
1284 | * Only certain cache maintenance operations (e.g. 'dc ivac') require write |
1285 | * access to the mapping, but if a cache maintenance operation that only requires |
1286 | * read access generates a permission fault, then we will not be able to handle |
1287 | * the fault regardless of whether we treat it as a read or write fault. |
1288 | */ |
1289 | if ((iss & ISS_DA_WNR) && (!(iss & ISS_DA_CM) || is_permission_fault(status: *fault_code))) { |
1290 | *fault_type = (VM_PROT_READ | VM_PROT_WRITE); |
1291 | } else { |
1292 | *fault_type = (VM_PROT_READ); |
1293 | } |
1294 | } |
1295 | |
1296 | #if __has_feature(ptrauth_calls) |
1297 | static inline uint64_t |
1298 | fault_addr_bitmask(unsigned int bit_from, unsigned int bit_to) |
1299 | { |
1300 | return ((1ULL << (bit_to - bit_from + 1)) - 1) << bit_from; |
1301 | } |
1302 | |
1303 | static inline bool |
1304 | fault_addr_bit(vm_offset_t fault_addr, unsigned int bit) |
1305 | { |
1306 | return (bool)((fault_addr >> bit) & 1); |
1307 | } |
1308 | |
1309 | extern int gARM_FEAT_PAuth2; |
1310 | |
1311 | /** |
1312 | * Determines whether a fault address taken at EL0 contains a PAC error code |
1313 | * corresponding to the specified kind of ptrauth key. |
1314 | */ |
1315 | static bool |
1316 | user_fault_addr_matches_pac_error_code(vm_offset_t fault_addr, bool data_key) |
1317 | { |
1318 | bool instruction_tbi = !(get_tcr() & TCR_TBID0_TBI_DATA_ONLY); |
1319 | bool tbi = data_key || __improbable(instruction_tbi); |
1320 | |
1321 | if (gARM_FEAT_PAuth2) { |
1322 | /* |
1323 | * EnhancedPAC2 CPUs don't encode error codes at fixed positions, so |
1324 | * treat all non-canonical address bits like potential poison bits. |
1325 | */ |
1326 | uint64_t mask = fault_addr_bitmask(64 - T0SZ_BOOT, 54); |
1327 | if (!tbi) { |
1328 | mask |= fault_addr_bitmask(56, 63); |
1329 | } |
1330 | return (fault_addr & mask) != 0; |
1331 | } else { |
1332 | unsigned int poison_shift; |
1333 | if (tbi) { |
1334 | poison_shift = 53; |
1335 | } else { |
1336 | poison_shift = 61; |
1337 | } |
1338 | |
1339 | /* PAC error codes are always in the form key_number:NOT(key_number) */ |
1340 | bool poison_bit_1 = fault_addr_bit(fault_addr, poison_shift); |
1341 | bool poison_bit_2 = fault_addr_bit(fault_addr, poison_shift + 1); |
1342 | return poison_bit_1 != poison_bit_2; |
1343 | } |
1344 | } |
1345 | #endif /* __has_feature(ptrauth_calls) */ |
1346 | |
1347 | /** |
1348 | * Determines whether the userland thread has a JIT region in RW mode, TPRO |
1349 | * in RW mode, or JCTL_EL0 in pointer signing mode. A fault in any of these trusted |
1350 | * code paths may indicate an attack on WebKit. Rather than letting a |
1351 | * potentially-compromised process try to handle the exception, it will be killed |
1352 | * by the kernel and a crash report will be generated. |
1353 | */ |
1354 | static bool |
1355 | user_fault_in_self_restrict_mode(thread_t thread __unused) |
1356 | { |
1357 | |
1358 | return false; |
1359 | } |
1360 | |
1361 | static void |
1362 | handle_pc_align(arm_saved_state_t *ss) |
1363 | { |
1364 | exception_type_t exc; |
1365 | mach_exception_data_type_t codes[2]; |
1366 | mach_msg_type_number_t numcodes = 2; |
1367 | |
1368 | if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) { |
1369 | panic_with_thread_kernel_state(msg: "PC alignment exception from kernel." , ss); |
1370 | } |
1371 | |
1372 | exc = EXC_BAD_ACCESS; |
1373 | #if __has_feature(ptrauth_calls) |
1374 | if (user_fault_addr_matches_pac_error_code(get_saved_state_pc(ss), false)) { |
1375 | exc |= EXC_PTRAUTH_BIT; |
1376 | } |
1377 | #endif /* __has_feature(ptrauth_calls) */ |
1378 | |
1379 | codes[0] = EXC_ARM_DA_ALIGN; |
1380 | codes[1] = get_saved_state_pc(iss: ss); |
1381 | |
1382 | exception_triage(exception: exc, code: codes, codeCnt: numcodes); |
1383 | __builtin_unreachable(); |
1384 | } |
1385 | |
1386 | static void |
1387 | handle_sp_align(arm_saved_state_t *ss) |
1388 | { |
1389 | exception_type_t exc; |
1390 | mach_exception_data_type_t codes[2]; |
1391 | mach_msg_type_number_t numcodes = 2; |
1392 | |
1393 | if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) { |
1394 | panic_with_thread_kernel_state(msg: "SP alignment exception from kernel." , ss); |
1395 | } |
1396 | |
1397 | exc = EXC_BAD_ACCESS; |
1398 | #if __has_feature(ptrauth_calls) |
1399 | if (user_fault_addr_matches_pac_error_code(get_saved_state_sp(ss), true)) { |
1400 | exc |= EXC_PTRAUTH_BIT; |
1401 | } |
1402 | #endif /* __has_feature(ptrauth_calls) */ |
1403 | |
1404 | codes[0] = EXC_ARM_SP_ALIGN; |
1405 | codes[1] = get_saved_state_sp(iss: ss); |
1406 | |
1407 | exception_triage(exception: exc, code: codes, codeCnt: numcodes); |
1408 | __builtin_unreachable(); |
1409 | } |
1410 | |
1411 | static void |
1412 | handle_wf_trap(arm_saved_state_t *state) |
1413 | { |
1414 | exception_type_t exc; |
1415 | mach_exception_data_type_t codes[2]; |
1416 | mach_msg_type_number_t numcodes = 2; |
1417 | uint32_t instr = 0; |
1418 | |
1419 | COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr)); |
1420 | |
1421 | exc = EXC_BAD_INSTRUCTION; |
1422 | codes[0] = EXC_ARM_UNDEFINED; |
1423 | codes[1] = instr; |
1424 | |
1425 | exception_triage(exception: exc, code: codes, codeCnt: numcodes); |
1426 | __builtin_unreachable(); |
1427 | } |
1428 | |
1429 | static void |
1430 | handle_fp_trap(arm_saved_state_t *state, uint32_t esr) |
1431 | { |
1432 | exception_type_t exc = EXC_ARITHMETIC; |
1433 | mach_exception_data_type_t codes[2]; |
1434 | mach_msg_type_number_t numcodes = 2; |
1435 | uint32_t instr = 0; |
1436 | |
1437 | if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { |
1438 | panic_with_thread_kernel_state(msg: "Floating point exception from kernel" , ss: state); |
1439 | } |
1440 | |
1441 | COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr)); |
1442 | codes[1] = instr; |
1443 | |
1444 | /* The floating point trap flags are only valid if TFV is set. */ |
1445 | if (!fp_exceptions_enabled) { |
1446 | exc = EXC_BAD_INSTRUCTION; |
1447 | codes[0] = EXC_ARM_UNDEFINED; |
1448 | } else if (!(esr & ISS_FP_TFV)) { |
1449 | codes[0] = EXC_ARM_FP_UNDEFINED; |
1450 | } else if (esr & ISS_FP_UFF) { |
1451 | codes[0] = EXC_ARM_FP_UF; |
1452 | } else if (esr & ISS_FP_OFF) { |
1453 | codes[0] = EXC_ARM_FP_OF; |
1454 | } else if (esr & ISS_FP_IOF) { |
1455 | codes[0] = EXC_ARM_FP_IO; |
1456 | } else if (esr & ISS_FP_DZF) { |
1457 | codes[0] = EXC_ARM_FP_DZ; |
1458 | } else if (esr & ISS_FP_IDF) { |
1459 | codes[0] = EXC_ARM_FP_ID; |
1460 | } else if (esr & ISS_FP_IXF) { |
1461 | codes[0] = EXC_ARM_FP_IX; |
1462 | } else { |
1463 | panic("Unrecognized floating point exception, state=%p, esr=%#x" , state, esr); |
1464 | } |
1465 | |
1466 | exception_triage(exception: exc, code: codes, codeCnt: numcodes); |
1467 | __builtin_unreachable(); |
1468 | } |
1469 | |
1470 | |
1471 | |
1472 | /* |
1473 | * handle_alignment_fault_from_user: |
1474 | * state: Saved state |
1475 | * |
1476 | * Attempts to deal with an alignment fault from userspace (possibly by |
1477 | * emulating the faulting instruction). If emulation failed due to an |
1478 | * unservicable fault, the ESR for that fault will be stored in the |
1479 | * recovery_esr field of the thread by the exception code. |
1480 | * |
1481 | * Returns: |
1482 | * -1: Emulation failed (emulation of state/instr not supported) |
1483 | * 0: Successfully emulated the instruction |
1484 | * EFAULT: Emulation failed (probably due to permissions) |
1485 | * EINVAL: Emulation failed (probably due to a bad address) |
1486 | */ |
1487 | |
1488 | |
1489 | static int |
1490 | handle_alignment_fault_from_user(arm_saved_state_t *state, kern_return_t *vmfr) |
1491 | { |
1492 | int ret = -1; |
1493 | |
1494 | #pragma unused (state) |
1495 | #pragma unused (vmfr) |
1496 | |
1497 | return ret; |
1498 | } |
1499 | |
1500 | |
1501 | |
1502 | |
1503 | static void |
1504 | handle_sw_step_debug(arm_saved_state_t *state) |
1505 | { |
1506 | thread_t thread = current_thread(); |
1507 | exception_type_t exc; |
1508 | mach_exception_data_type_t codes[2]; |
1509 | mach_msg_type_number_t numcodes = 2; |
1510 | |
1511 | if (!PSR64_IS_USER(get_saved_state_cpsr(state))) { |
1512 | panic_with_thread_kernel_state(msg: "SW_STEP_DEBUG exception from kernel." , ss: state); |
1513 | } |
1514 | |
1515 | // Disable single step and unmask interrupts (in the saved state, anticipating next exception return) |
1516 | if (thread->machine.DebugData != NULL) { |
1517 | thread->machine.DebugData->uds.ds64.mdscr_el1 &= ~0x1; |
1518 | } else { |
1519 | panic_with_thread_kernel_state(msg: "SW_STEP_DEBUG exception thread DebugData is NULL." , ss: state); |
1520 | } |
1521 | |
1522 | mask_user_saved_state_cpsr(iss: thread->machine.upcb, set_bits: 0, PSR64_SS | DAIF_ALL); |
1523 | |
1524 | // Special encoding for gdb single step event on ARM |
1525 | exc = EXC_BREAKPOINT; |
1526 | codes[0] = 1; |
1527 | codes[1] = 0; |
1528 | |
1529 | exception_triage(exception: exc, code: codes, codeCnt: numcodes); |
1530 | __builtin_unreachable(); |
1531 | } |
1532 | |
1533 | static void |
1534 | handle_user_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr, |
1535 | fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler) |
1536 | { |
1537 | exception_type_t exc = EXC_BAD_ACCESS; |
1538 | mach_exception_data_type_t codes[2]; |
1539 | mach_msg_type_number_t numcodes = 2; |
1540 | thread_t thread = current_thread(); |
1541 | |
1542 | (void)esr; |
1543 | (void)expected_fault_handler; |
1544 | |
1545 | if (ml_at_interrupt_context()) { |
1546 | panic_with_thread_kernel_state(msg: "Apparently on interrupt stack when taking user abort!\n" , ss: state); |
1547 | } |
1548 | |
1549 | thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling abort from userspace */ |
1550 | |
1551 | if (!is_vm_fault(status: fault_code) && |
1552 | thread->t_rr_state.trr_fault_state != TRR_FAULT_NONE) { |
1553 | thread_reset_pcs_done_faulting(thread); |
1554 | } |
1555 | |
1556 | if (is_vm_fault(status: fault_code)) { |
1557 | vm_map_t map = thread->map; |
1558 | vm_offset_t vm_fault_addr = fault_addr; |
1559 | kern_return_t result = KERN_FAILURE; |
1560 | |
1561 | assert(map != kernel_map); |
1562 | |
1563 | if (!(fault_type & VM_PROT_EXECUTE)) { |
1564 | vm_fault_addr = VM_USER_STRIP_TBI(fault_addr); |
1565 | } |
1566 | |
1567 | /* check to see if it is just a pmap ref/modify fault */ |
1568 | if (!is_translation_fault(status: fault_code)) { |
1569 | result = arm_fast_fault(map->pmap, |
1570 | vm_fault_addr, |
1571 | fault_type, (fault_code == FSC_ACCESS_FLAG_FAULT_L3), TRUE); |
1572 | } |
1573 | if (result != KERN_SUCCESS) { |
1574 | |
1575 | { |
1576 | /* We have to fault the page in */ |
1577 | result = vm_fault(map, vaddr: vm_fault_addr, fault_type, |
1578 | /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, THREAD_ABORTSAFE, |
1579 | /* caller_pmap */ NULL, /* caller_pmap_addr */ pmap_addr: 0); |
1580 | } |
1581 | } |
1582 | if (thread->t_rr_state.trr_fault_state != TRR_FAULT_NONE) { |
1583 | thread_reset_pcs_done_faulting(thread); |
1584 | } |
1585 | if (result == KERN_SUCCESS || result == KERN_ABORTED) { |
1586 | return; |
1587 | } |
1588 | |
1589 | /* |
1590 | * vm_fault() should never return KERN_FAILURE for page faults from user space. |
1591 | * If it does, we're leaking preemption disables somewhere in the kernel. |
1592 | */ |
1593 | if (__improbable(result == KERN_FAILURE)) { |
1594 | panic("vm_fault() KERN_FAILURE from user fault on thread %p" , thread); |
1595 | } |
1596 | |
1597 | codes[0] = result; |
1598 | } else if (is_alignment_fault(status: fault_code)) { |
1599 | kern_return_t vmfkr = KERN_SUCCESS; |
1600 | thread->machine.recover_esr = 0; |
1601 | thread->machine.recover_far = 0; |
1602 | int result = handle_alignment_fault_from_user(state, vmfr: &vmfkr); |
1603 | if (result == 0) { |
1604 | /* Successfully emulated, or instruction |
1605 | * copyin() for decode/emulation failed. |
1606 | * Continue, or redrive instruction. |
1607 | */ |
1608 | thread_exception_return(); |
1609 | } else if (((result == EFAULT) || (result == EINVAL)) && |
1610 | (thread->machine.recover_esr == 0)) { |
1611 | /* |
1612 | * If we didn't actually take a fault, but got one of |
1613 | * these errors, then we failed basic sanity checks of |
1614 | * the fault address. Treat this as an invalid |
1615 | * address. |
1616 | */ |
1617 | codes[0] = KERN_INVALID_ADDRESS; |
1618 | } else if ((result == EFAULT) && |
1619 | (thread->machine.recover_esr)) { |
1620 | /* |
1621 | * Since alignment aborts are prioritized |
1622 | * ahead of translation aborts, the misaligned |
1623 | * atomic emulation flow may have triggered a |
1624 | * VM pagefault, which the VM could not resolve. |
1625 | * Report the VM fault error in codes[] |
1626 | */ |
1627 | |
1628 | codes[0] = vmfkr; |
1629 | assertf(vmfkr != KERN_SUCCESS, "Unexpected vmfkr 0x%x" , vmfkr); |
1630 | /* Cause ESR_EC to reflect an EL0 abort */ |
1631 | thread->machine.recover_esr &= ~ESR_EC_MASK; |
1632 | thread->machine.recover_esr |= (ESR_EC_DABORT_EL0 << ESR_EC_SHIFT); |
1633 | set_saved_state_esr(iss: thread->machine.upcb, esr: thread->machine.recover_esr); |
1634 | set_saved_state_far(iss: thread->machine.upcb, far: thread->machine.recover_far); |
1635 | fault_addr = thread->machine.recover_far; |
1636 | } else { |
1637 | /* This was just an unsupported alignment |
1638 | * exception. Misaligned atomic emulation |
1639 | * timeouts fall in this category. |
1640 | */ |
1641 | codes[0] = EXC_ARM_DA_ALIGN; |
1642 | } |
1643 | } else if (is_parity_error(status: fault_code)) { |
1644 | #if defined(APPLE_ARM64_ARCH_FAMILY) |
1645 | /* |
1646 | * Platform errors are handled in sleh_sync before interrupts are enabled. |
1647 | */ |
1648 | #else |
1649 | panic("User parity error." ); |
1650 | #endif |
1651 | } else { |
1652 | codes[0] = KERN_FAILURE; |
1653 | } |
1654 | |
1655 | #if CODE_SIGNING_MONITOR |
1656 | /* |
1657 | * If the code reaches here, it means we weren't able to resolve the fault and we're |
1658 | * going to be sending the task an exception. On systems which have the code signing |
1659 | * monitor enabled, an execute fault which cannot be handled must result in sending |
1660 | * a SIGKILL to the task. |
1661 | */ |
1662 | if (is_vm_fault(fault_code) && (fault_type & VM_PROT_EXECUTE)) { |
1663 | csm_code_signing_violation(current_proc(), fault_addr); |
1664 | } |
1665 | #endif |
1666 | |
1667 | codes[1] = fault_addr; |
1668 | #if __has_feature(ptrauth_calls) |
1669 | bool is_data_abort = (ESR_EC(esr) == ESR_EC_DABORT_EL0); |
1670 | if (user_fault_addr_matches_pac_error_code(fault_addr, is_data_abort)) { |
1671 | exc |= EXC_PTRAUTH_BIT; |
1672 | } |
1673 | #endif /* __has_feature(ptrauth_calls) */ |
1674 | |
1675 | if (user_fault_in_self_restrict_mode(thread) && |
1676 | task_is_jit_exception_fatal(task: get_threadtask(thread))) { |
1677 | extern int exit_with_jit_exception(proc_t p); |
1678 | exit_with_jit_exception(p: current_proc()); |
1679 | } |
1680 | |
1681 | exception_triage(exception: exc, code: codes, codeCnt: numcodes); |
1682 | __builtin_unreachable(); |
1683 | } |
1684 | |
1685 | /** |
1686 | * Panic because the kernel abort handler tried to apply a recovery handler that |
1687 | * isn't inside copyio_recover_table[]. |
1688 | * |
1689 | * @param state original saved-state |
1690 | * @param recover invalid recovery handler |
1691 | */ |
1692 | __attribute__((noreturn, used)) |
1693 | static void |
1694 | panic_on_invalid_recovery_handler(arm_saved_state_t *state, struct copyio_recovery_entry *recover) |
1695 | { |
1696 | panic("attempt to set invalid recovery handler %p on kernel saved-state %p" , recover, state); |
1697 | } |
1698 | |
1699 | static void |
1700 | handle_kernel_abort_recover( |
1701 | arm_saved_state_t *state, |
1702 | uint32_t esr, |
1703 | vm_offset_t fault_addr, |
1704 | thread_t thread, |
1705 | struct copyio_recovery_entry *_Nonnull recover) |
1706 | { |
1707 | thread->machine.recover_esr = esr; |
1708 | thread->machine.recover_far = fault_addr; |
1709 | #if defined(HAS_APPLE_PAC) |
1710 | MANIPULATE_SIGNED_THREAD_STATE(state, |
1711 | "adrp x6, _copyio_recover_table_end@page \n" |
1712 | "add x6, x6, _copyio_recover_table_end@pageoff \n" |
1713 | "cmp %[recover], x6 \n" |
1714 | "b.lt 1f \n" |
1715 | "bl _panic_on_invalid_recovery_handler \n" |
1716 | "brk #0 \n" |
1717 | "1: \n" |
1718 | "adrp x6, _copyio_recover_table@page \n" |
1719 | "add x6, x6, _copyio_recover_table@pageoff \n" |
1720 | "cmp %[recover], x6 \n" |
1721 | "b.ge 1f \n" |
1722 | "bl _panic_on_invalid_recovery_handler \n" |
1723 | "brk #0 \n" |
1724 | "1: \n" |
1725 | "ldr x1, [%[recover], %[CRE_RECOVERY]] \n" |
1726 | "add x1, x1, x6 \n" |
1727 | "str x1, [x0, %[SS64_PC]] \n" , |
1728 | [recover] "r" (recover), |
1729 | [CRE_RECOVERY] "i" (offsetof(struct copyio_recovery_entry, cre_recovery)) |
1730 | ); |
1731 | #else |
1732 | if ((uintptr_t)recover < (uintptr_t)copyio_recover_table || |
1733 | (uintptr_t)recover >= (uintptr_t)copyio_recover_table_end) { |
1734 | panic_on_invalid_recovery_handler(state, recover); |
1735 | } |
1736 | saved_state64(state)->pc = copyio_recovery_addr(recover->cre_recovery); |
1737 | #endif |
1738 | } |
1739 | |
1740 | static void |
1741 | handle_kernel_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr, |
1742 | fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler) |
1743 | { |
1744 | thread_t thread = current_thread(); |
1745 | struct copyio_recovery_entry *recover = find_copyio_recovery_entry(state); |
1746 | |
1747 | #ifndef CONFIG_XNUPOST |
1748 | (void)expected_fault_handler; |
1749 | #endif /* CONFIG_XNUPOST */ |
1750 | |
1751 | #if CONFIG_DTRACE |
1752 | if (is_vm_fault(status: fault_code) && thread->t_dtrace_inprobe) { /* Executing under dtrace_probe? */ |
1753 | if (dtrace_tally_fault(fault_addr)) { /* Should a fault under dtrace be ignored? */ |
1754 | /* |
1755 | * Point to next instruction, or recovery handler if set. |
1756 | */ |
1757 | if (recover) { |
1758 | handle_kernel_abort_recover(state, esr, VM_USER_STRIP_PTR(fault_addr), thread, recover); |
1759 | } else { |
1760 | add_saved_state_pc(iss: state, diff: 4); |
1761 | } |
1762 | return; |
1763 | } else { |
1764 | panic_with_thread_kernel_state(msg: "Unexpected page fault under dtrace_probe" , ss: state); |
1765 | } |
1766 | } |
1767 | #endif |
1768 | |
1769 | if (ml_at_interrupt_context()) { |
1770 | panic_with_thread_kernel_state(msg: "Unexpected abort while on interrupt stack." , ss: state); |
1771 | } |
1772 | |
1773 | if (is_vm_fault(status: fault_code)) { |
1774 | kern_return_t result = KERN_FAILURE; |
1775 | vm_map_t map; |
1776 | int interruptible; |
1777 | |
1778 | /* |
1779 | * Ensure no faults in the physical aperture. This could happen if |
1780 | * a page table is incorrectly allocated from the read only region |
1781 | * when running with KTRR. |
1782 | */ |
1783 | |
1784 | #ifdef CONFIG_XNUPOST |
1785 | if (expected_fault_handler && expected_fault_handler(state)) { |
1786 | return; |
1787 | } |
1788 | #endif /* CONFIG_XNUPOST */ |
1789 | |
1790 | if (fault_addr >= gVirtBase && fault_addr < static_memory_end) { |
1791 | panic_with_thread_kernel_state(msg: "Unexpected fault in kernel static region\n" , ss: state); |
1792 | } |
1793 | |
1794 | if (VM_KERNEL_ADDRESS(fault_addr) || thread == THREAD_NULL || recover == 0) { |
1795 | /* |
1796 | * If no recovery handler is supplied, always drive the fault against |
1797 | * the kernel map. If the fault was taken against a userspace VA, indicating |
1798 | * an unprotected access to user address space, vm_fault() should fail and |
1799 | * ultimately lead to a panic here. |
1800 | */ |
1801 | map = kernel_map; |
1802 | interruptible = THREAD_UNINT; |
1803 | |
1804 | #if CONFIG_KERNEL_TAGGING |
1805 | /* |
1806 | * If kernel tagging is enabled, canonicalize the address here, so that we have a |
1807 | * chance to find it in the VM ranges. Do not mess with exec fault cases. |
1808 | */ |
1809 | if (!((fault_type) & VM_PROT_EXECUTE)) { |
1810 | fault_addr = vm_memtag_canonicalize_address(fault_addr); |
1811 | } |
1812 | #endif /* CONFIG_KERNEL_TAGGING */ |
1813 | } else { |
1814 | map = thread->map; |
1815 | |
1816 | /** |
1817 | * In the case that the recovery handler is set (e.g., during copyio |
1818 | * and dtrace probes), we don't want the vm_fault() operation to be |
1819 | * aborted early. Those code paths can't handle restarting the |
1820 | * vm_fault() operation so don't allow it to return early without |
1821 | * creating the wanted mapping. |
1822 | */ |
1823 | interruptible = (recover) ? THREAD_UNINT : THREAD_ABORTSAFE; |
1824 | |
1825 | } |
1826 | |
1827 | if (fault_addr >= gVirtBase && fault_addr < static_memory_end) { |
1828 | panic_with_thread_kernel_state(msg: "Unexpected fault in kernel static region\n" , ss: state); |
1829 | } |
1830 | |
1831 | /* check to see if it is just a pmap ref/modify fault */ |
1832 | if (!is_translation_fault(status: fault_code)) { |
1833 | result = arm_fast_fault(map->pmap, |
1834 | fault_addr, |
1835 | fault_type, (fault_code == FSC_ACCESS_FLAG_FAULT_L3), FALSE); |
1836 | if (result == KERN_SUCCESS) { |
1837 | return; |
1838 | } |
1839 | } |
1840 | |
1841 | if (result != KERN_PROTECTION_FAILURE) { |
1842 | /* |
1843 | * We have to "fault" the page in. |
1844 | */ |
1845 | result = vm_fault(map, vaddr: fault_addr, fault_type, |
1846 | /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, interruptible, |
1847 | /* caller_pmap */ NULL, /* caller_pmap_addr */ pmap_addr: 0); |
1848 | } |
1849 | |
1850 | if (result == KERN_SUCCESS) { |
1851 | return; |
1852 | } |
1853 | |
1854 | /* |
1855 | * If we have a recover handler, invoke it now. |
1856 | */ |
1857 | if (recover) { |
1858 | handle_kernel_abort_recover(state, esr, fault_addr, thread, recover); |
1859 | return; |
1860 | } |
1861 | |
1862 | panic_fault_address = fault_addr; |
1863 | } else if (is_alignment_fault(status: fault_code)) { |
1864 | if (recover) { |
1865 | handle_kernel_abort_recover(state, esr, fault_addr, thread, recover); |
1866 | return; |
1867 | } |
1868 | panic_with_thread_kernel_state(msg: "Unaligned kernel data abort." , ss: state); |
1869 | } else if (is_parity_error(status: fault_code)) { |
1870 | #if defined(APPLE_ARM64_ARCH_FAMILY) |
1871 | /* |
1872 | * Platform errors are handled in sleh_sync before interrupts are enabled. |
1873 | */ |
1874 | #else |
1875 | panic_with_thread_kernel_state("Kernel parity error." , state); |
1876 | #endif |
1877 | } else { |
1878 | kprintf(fmt: "Unclassified kernel abort (fault_code=0x%x)\n" , fault_code); |
1879 | } |
1880 | |
1881 | panic_with_thread_kernel_state(msg: "Kernel data abort." , ss: state); |
1882 | } |
1883 | |
1884 | extern void syscall_trace(struct arm_saved_state * regs); |
1885 | |
1886 | static void |
1887 | handle_svc(arm_saved_state_t *state) |
1888 | { |
1889 | int trap_no = get_saved_state_svc_number(iss: state); |
1890 | thread_t thread = current_thread(); |
1891 | struct proc *p; |
1892 | |
1893 | #define handle_svc_kprintf(x...) /* kprintf("handle_svc: " x) */ |
1894 | |
1895 | #define TRACE_SYSCALL 1 |
1896 | #if TRACE_SYSCALL |
1897 | syscall_trace(regs: state); |
1898 | #endif |
1899 | |
1900 | thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling SVC from userspace */ |
1901 | |
1902 | if (trap_no == (int)PLATFORM_SYSCALL_TRAP_NO) { |
1903 | platform_syscall(state); |
1904 | panic("Returned from platform_syscall()?" ); |
1905 | } |
1906 | |
1907 | current_cached_proc_cred_update(); |
1908 | |
1909 | if (trap_no < 0) { |
1910 | switch (trap_no) { |
1911 | case MACH_ARM_TRAP_ABSTIME: |
1912 | handle_mach_absolute_time_trap(state); |
1913 | return; |
1914 | case MACH_ARM_TRAP_CONTTIME: |
1915 | handle_mach_continuous_time_trap(state); |
1916 | return; |
1917 | } |
1918 | |
1919 | /* Counting perhaps better in the handler, but this is how it's been done */ |
1920 | thread->syscalls_mach++; |
1921 | mach_syscall(state); |
1922 | } else { |
1923 | /* Counting perhaps better in the handler, but this is how it's been done */ |
1924 | thread->syscalls_unix++; |
1925 | p = get_bsdthreadtask_info(thread); |
1926 | |
1927 | assert(p); |
1928 | |
1929 | unix_syscall(regs: state, thread_act: thread, proc: p); |
1930 | } |
1931 | } |
1932 | |
1933 | static void |
1934 | handle_mach_absolute_time_trap(arm_saved_state_t *state) |
1935 | { |
1936 | uint64_t now = mach_absolute_time(); |
1937 | saved_state64(iss: state)->x[0] = now; |
1938 | } |
1939 | |
1940 | static void |
1941 | handle_mach_continuous_time_trap(arm_saved_state_t *state) |
1942 | { |
1943 | uint64_t now = mach_continuous_time(); |
1944 | saved_state64(iss: state)->x[0] = now; |
1945 | } |
1946 | |
1947 | |
1948 | __attribute__((noreturn)) |
1949 | static void |
1950 | handle_msr_trap(arm_saved_state_t *state, uint32_t esr) |
1951 | { |
1952 | exception_type_t exception = EXC_BAD_INSTRUCTION; |
1953 | mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED}; |
1954 | mach_msg_type_number_t numcodes = 2; |
1955 | uint32_t instr = 0; |
1956 | |
1957 | if (!is_saved_state64(iss: state)) { |
1958 | panic("MSR/MRS trap (ESR 0x%x) from 32-bit state" , esr); |
1959 | } |
1960 | |
1961 | if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { |
1962 | panic("MSR/MRS trap (ESR 0x%x) from kernel" , esr); |
1963 | } |
1964 | |
1965 | COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr)); |
1966 | codes[1] = instr; |
1967 | |
1968 | exception_triage(exception, code: codes, codeCnt: numcodes); |
1969 | __builtin_unreachable(); |
1970 | } |
1971 | |
1972 | #if __has_feature(ptrauth_calls) |
1973 | static void |
1974 | stringify_gpr(unsigned int r, char reg[4]) |
1975 | { |
1976 | switch (r) { |
1977 | case 29: |
1978 | strncpy(reg, "fp" , 4); |
1979 | return; |
1980 | |
1981 | case 30: |
1982 | strncpy(reg, "lr" , 4); |
1983 | return; |
1984 | |
1985 | case 31: |
1986 | strncpy(reg, "xzr" , 4); |
1987 | return; |
1988 | |
1989 | default: |
1990 | snprintf(reg, 4, "x%u" , r); |
1991 | return; |
1992 | } |
1993 | } |
1994 | |
1995 | static void |
1996 | autxx_instruction_extract_reg(uint32_t instr, char reg[4]) |
1997 | { |
1998 | unsigned int rd = ARM64_INSTR_AUTxx_RD_GET(instr); |
1999 | stringify_gpr(rd, reg); |
2000 | } |
2001 | |
2002 | static const char * |
2003 | autix_system_instruction_extract_reg(uint32_t instr) |
2004 | { |
2005 | unsigned int crm_op2 = ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_GET(instr); |
2006 | if (crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIA1716 || |
2007 | crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIB1716) { |
2008 | return "x17" ; |
2009 | } else { |
2010 | return "lr" ; |
2011 | } |
2012 | } |
2013 | |
2014 | |
2015 | static void |
2016 | handle_pac_fail(arm_saved_state_t *state, uint32_t esr) |
2017 | { |
2018 | exception_type_t exception = EXC_BAD_ACCESS | EXC_PTRAUTH_BIT; |
2019 | mach_exception_data_type_t codes[2] = {EXC_ARM_PAC_FAIL}; |
2020 | mach_msg_type_number_t numcodes = 2; |
2021 | uint32_t instr = 0; |
2022 | |
2023 | if (!is_saved_state64(state)) { |
2024 | panic("PAC failure (ESR 0x%x) from 32-bit state" , esr); |
2025 | } |
2026 | |
2027 | COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr)); |
2028 | |
2029 | if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { |
2030 | #define GENERIC_PAC_FAILURE_MSG_FMT "PAC failure from kernel with %s key" |
2031 | #define AUTXX_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT " while authing %s" |
2032 | #define GENERIC_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT |
2033 | #define MAX_PAC_MSG_FMT AUTXX_MSG_FMT |
2034 | |
2035 | char msg[strlen(MAX_PAC_MSG_FMT) |
2036 | - strlen("%s" ) + strlen("IA" ) |
2037 | - strlen("%s" ) + strlen("xzr" ) |
2038 | + 1]; |
2039 | ptrauth_key key = (ptrauth_key)(esr & 0x3); |
2040 | const char *key_str = ptrauth_key_to_string(key); |
2041 | |
2042 | if (ARM64_INSTR_IS_AUTxx(instr)) { |
2043 | char reg[4]; |
2044 | autxx_instruction_extract_reg(instr, reg); |
2045 | snprintf(msg, sizeof(msg), AUTXX_MSG_FMT, key_str, reg); |
2046 | } else if (ARM64_INSTR_IS_AUTIx_SYSTEM(instr)) { |
2047 | const char *reg = autix_system_instruction_extract_reg(instr); |
2048 | snprintf(msg, sizeof(msg), AUTXX_MSG_FMT, key_str, reg); |
2049 | } else { |
2050 | snprintf(msg, sizeof(msg), GENERIC_MSG_FMT, key_str); |
2051 | } |
2052 | panic_with_thread_kernel_state(msg, state); |
2053 | } |
2054 | |
2055 | codes[1] = instr; |
2056 | |
2057 | exception_triage(exception, codes, numcodes); |
2058 | __builtin_unreachable(); |
2059 | } |
2060 | #endif /* __has_feature(ptrauth_calls) */ |
2061 | |
2062 | static void |
2063 | handle_user_trapped_instruction32(arm_saved_state_t *state, uint32_t esr) |
2064 | { |
2065 | exception_type_t exception = EXC_BAD_INSTRUCTION; |
2066 | mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED}; |
2067 | mach_msg_type_number_t numcodes = 2; |
2068 | uint32_t instr; |
2069 | |
2070 | if (is_saved_state64(iss: state)) { |
2071 | panic("ESR (0x%x) for instruction trapped from U32, but saved state is 64-bit." , esr); |
2072 | } |
2073 | |
2074 | if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { |
2075 | panic("ESR (0x%x) for instruction trapped from U32, actually came from kernel?" , esr); |
2076 | } |
2077 | |
2078 | COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr)); |
2079 | codes[1] = instr; |
2080 | |
2081 | exception_triage(exception, code: codes, codeCnt: numcodes); |
2082 | __builtin_unreachable(); |
2083 | } |
2084 | |
2085 | static void |
2086 | handle_simd_trap(arm_saved_state_t *state, uint32_t esr) |
2087 | { |
2088 | exception_type_t exception = EXC_BAD_INSTRUCTION; |
2089 | mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED}; |
2090 | mach_msg_type_number_t numcodes = 2; |
2091 | uint32_t instr = 0; |
2092 | |
2093 | if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { |
2094 | panic("ESR (0x%x) for SIMD trap from userland, actually came from kernel?" , esr); |
2095 | } |
2096 | |
2097 | COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr)); |
2098 | codes[1] = instr; |
2099 | |
2100 | exception_triage(exception, code: codes, codeCnt: numcodes); |
2101 | __builtin_unreachable(); |
2102 | } |
2103 | |
2104 | void |
2105 | sleh_irq(arm_saved_state_t *state) |
2106 | { |
2107 | cpu_data_t * cdp __unused = getCpuDatap(); |
2108 | #if MACH_ASSERT |
2109 | int preemption_level = sleh_get_preemption_level(); |
2110 | #endif |
2111 | |
2112 | |
2113 | sleh_interrupt_handler_prologue(state, DBG_INTR_TYPE_OTHER); |
2114 | |
2115 | #if USE_APPLEARMSMP |
2116 | PE_handle_ext_interrupt(); |
2117 | #else |
2118 | /* Run the registered interrupt handler. */ |
2119 | cdp->interrupt_handler(cdp->interrupt_target, |
2120 | cdp->interrupt_refCon, |
2121 | cdp->interrupt_nub, |
2122 | cdp->interrupt_source); |
2123 | #endif |
2124 | |
2125 | entropy_collect(); |
2126 | |
2127 | |
2128 | sleh_interrupt_handler_epilogue(); |
2129 | #if MACH_ASSERT |
2130 | if (preemption_level != sleh_get_preemption_level()) { |
2131 | panic("irq handler %p changed preemption level from %d to %d" , cdp->interrupt_handler, preemption_level, sleh_get_preemption_level()); |
2132 | } |
2133 | #endif |
2134 | } |
2135 | |
2136 | void |
2137 | sleh_fiq(arm_saved_state_t *state) |
2138 | { |
2139 | unsigned int type = DBG_INTR_TYPE_UNKNOWN; |
2140 | #if MACH_ASSERT |
2141 | int preemption_level = sleh_get_preemption_level(); |
2142 | #endif |
2143 | |
2144 | #if MONOTONIC_FIQ |
2145 | uint64_t pmcr0 = 0, upmsr = 0; |
2146 | #endif /* MONOTONIC_FIQ */ |
2147 | |
2148 | #if defined(HAS_IPI) |
2149 | boolean_t is_ipi = FALSE; |
2150 | uint64_t ipi_sr = 0; |
2151 | |
2152 | if (gFastIPI) { |
2153 | MRS(ipi_sr, "S3_5_C15_C1_1" ); |
2154 | |
2155 | if (ipi_sr & ARM64_IPISR_IPI_PENDING) { |
2156 | is_ipi = TRUE; |
2157 | } |
2158 | } |
2159 | |
2160 | if (is_ipi) { |
2161 | type = DBG_INTR_TYPE_IPI; |
2162 | } else |
2163 | #endif /* defined(HAS_IPI) */ |
2164 | if (ml_get_timer_pending()) { |
2165 | type = DBG_INTR_TYPE_TIMER; |
2166 | } |
2167 | #if MONOTONIC_FIQ |
2168 | /* Consult the PMI sysregs last, after IPI/timer |
2169 | * classification. |
2170 | */ |
2171 | else if (mt_pmi_pending(&pmcr0, &upmsr)) { |
2172 | type = DBG_INTR_TYPE_PMI; |
2173 | } |
2174 | #endif /* MONOTONIC_FIQ */ |
2175 | |
2176 | sleh_interrupt_handler_prologue(state, type); |
2177 | |
2178 | #if APPLEVIRTUALPLATFORM |
2179 | uint64_t iar = __builtin_arm_rsr64("ICC_IAR0_EL1" ); |
2180 | #endif |
2181 | |
2182 | #if defined(HAS_IPI) |
2183 | if (type == DBG_INTR_TYPE_IPI) { |
2184 | /* |
2185 | * Order is important here: we must ack the IPI by writing IPI_SR |
2186 | * before we call cpu_signal_handler(). Otherwise, there will be |
2187 | * a window between the completion of pending-signal processing in |
2188 | * cpu_signal_handler() and the ack during which a newly-issued |
2189 | * IPI to this CPU may be lost. ISB is required to ensure the msr |
2190 | * is retired before execution of cpu_signal_handler(). |
2191 | */ |
2192 | MSR("S3_5_C15_C1_1" , ARM64_IPISR_IPI_PENDING); |
2193 | __builtin_arm_isb(ISB_SY); |
2194 | cpu_signal_handler(); |
2195 | } else |
2196 | #endif /* defined(HAS_IPI) */ |
2197 | #if MONOTONIC_FIQ |
2198 | if (type == DBG_INTR_TYPE_PMI) { |
2199 | INTERRUPT_MASKED_DEBUG_START(mt_fiq, DBG_INTR_TYPE_PMI); |
2200 | mt_fiq(getCpuDatap(), pmcr0, upmsr); |
2201 | INTERRUPT_MASKED_DEBUG_END(); |
2202 | } else |
2203 | #endif /* MONOTONIC_FIQ */ |
2204 | { |
2205 | /* |
2206 | * We don't know that this is a timer, but we don't have insight into |
2207 | * the other interrupts that go down this path. |
2208 | */ |
2209 | |
2210 | cpu_data_t *cdp = getCpuDatap(); |
2211 | |
2212 | cdp->cpu_decrementer = -1; /* Large */ |
2213 | |
2214 | /* |
2215 | * ARM64_TODO: whether we're coming from userland is ignored right now. |
2216 | * We can easily thread it through, but not bothering for the |
2217 | * moment (AArch32 doesn't either). |
2218 | */ |
2219 | INTERRUPT_MASKED_DEBUG_START(rtclock_intr, DBG_INTR_TYPE_TIMER); |
2220 | rtclock_intr(TRUE); |
2221 | INTERRUPT_MASKED_DEBUG_END(); |
2222 | } |
2223 | |
2224 | #if APPLEVIRTUALPLATFORM |
2225 | if (iar != GIC_SPURIOUS_IRQ) { |
2226 | __builtin_arm_wsr64("ICC_EOIR0_EL1" , iar); |
2227 | __builtin_arm_isb(ISB_SY); |
2228 | } |
2229 | #endif |
2230 | |
2231 | sleh_interrupt_handler_epilogue(); |
2232 | #if MACH_ASSERT |
2233 | if (preemption_level != sleh_get_preemption_level()) { |
2234 | panic("fiq type %u changed preemption level from %d to %d" , type, preemption_level, sleh_get_preemption_level()); |
2235 | } |
2236 | #endif |
2237 | } |
2238 | |
2239 | void |
2240 | sleh_serror(arm_context_t *context, uint32_t esr, vm_offset_t far) |
2241 | { |
2242 | task_vtimer_check(thread: current_thread()); |
2243 | |
2244 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_START, |
2245 | esr, VM_KERNEL_ADDRHIDE(far)); |
2246 | arm_saved_state_t *state = &context->ss; |
2247 | #if MACH_ASSERT |
2248 | int preemption_level = sleh_get_preemption_level(); |
2249 | #endif |
2250 | |
2251 | if (PSR64_IS_USER(get_saved_state_cpsr(state))) { |
2252 | /* Sanitize FAR (only if we came from userspace) */ |
2253 | saved_state64(iss: state)->far = 0; |
2254 | } |
2255 | |
2256 | ASSERT_CONTEXT_SANITY(context); |
2257 | arm64_platform_error(state, esr, far, source: PLAT_ERR_SRC_ASYNC); |
2258 | #if MACH_ASSERT |
2259 | if (preemption_level != sleh_get_preemption_level()) { |
2260 | panic("serror changed preemption level from %d to %d" , preemption_level, sleh_get_preemption_level()); |
2261 | } |
2262 | #endif |
2263 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_END, |
2264 | esr, VM_KERNEL_ADDRHIDE(far)); |
2265 | } |
2266 | |
2267 | void |
2268 | mach_syscall_trace_exit(unsigned int retval, |
2269 | unsigned int call_number) |
2270 | { |
2271 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, |
2272 | MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) | |
2273 | DBG_FUNC_END, retval, 0, 0, 0, 0); |
2274 | } |
2275 | |
2276 | __attribute__((noreturn)) |
2277 | void |
2278 | thread_syscall_return(kern_return_t error) |
2279 | { |
2280 | thread_t thread; |
2281 | struct arm_saved_state *state; |
2282 | |
2283 | thread = current_thread(); |
2284 | state = get_user_regs(thread); |
2285 | |
2286 | assert(is_saved_state64(state)); |
2287 | saved_state64(iss: state)->x[0] = error; |
2288 | |
2289 | #if MACH_ASSERT |
2290 | kern_allocation_name_t |
2291 | prior __assert_only = thread_get_kernel_state(thread)->allocation_name; |
2292 | assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared" , kern_allocation_get_name(prior)); |
2293 | #endif /* MACH_ASSERT */ |
2294 | |
2295 | if (kdebug_enable) { |
2296 | /* Invert syscall number (negative for a mach syscall) */ |
2297 | mach_syscall_trace_exit(retval: error, call_number: (-1) * get_saved_state_svc_number(iss: state)); |
2298 | } |
2299 | |
2300 | thread_exception_return(); |
2301 | } |
2302 | |
2303 | void |
2304 | syscall_trace( |
2305 | struct arm_saved_state * regs __unused) |
2306 | { |
2307 | /* kprintf("syscall: %d\n", saved_state64(regs)->x[16]); */ |
2308 | } |
2309 | |
2310 | static void |
2311 | sleh_interrupt_handler_prologue(arm_saved_state_t *state, unsigned int type) |
2312 | { |
2313 | const bool is_user = PSR64_IS_USER(get_saved_state_cpsr(state)); |
2314 | |
2315 | if (is_user == true) { |
2316 | /* Sanitize FAR (only if the interrupt occurred while the CPU was in usermode) */ |
2317 | saved_state64(iss: state)->far = 0; |
2318 | } |
2319 | |
2320 | recount_enter_interrupt(); |
2321 | |
2322 | task_vtimer_check(thread: current_thread()); |
2323 | |
2324 | uint64_t pc = is_user ? get_saved_state_pc(iss: state) : |
2325 | VM_KERNEL_UNSLIDE(get_saved_state_pc(state)); |
2326 | |
2327 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START, |
2328 | 0, pc, is_user, type); |
2329 | |
2330 | #if CONFIG_TELEMETRY |
2331 | if (telemetry_needs_record) { |
2332 | telemetry_mark_curthread(interrupted_userspace: is_user, FALSE); |
2333 | } |
2334 | #endif /* CONFIG_TELEMETRY */ |
2335 | } |
2336 | |
2337 | static void |
2338 | sleh_interrupt_handler_epilogue(void) |
2339 | { |
2340 | #if KPERF |
2341 | kperf_interrupt(); |
2342 | #endif /* KPERF */ |
2343 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END); |
2344 | recount_leave_interrupt(); |
2345 | } |
2346 | |
2347 | void |
2348 | sleh_invalid_stack(arm_context_t *context, uint32_t esr __unused, vm_offset_t far __unused) |
2349 | { |
2350 | thread_t thread = current_thread(); |
2351 | vm_offset_t kernel_stack_bottom, sp; |
2352 | |
2353 | sp = get_saved_state_sp(iss: &context->ss); |
2354 | vm_offset_t kstackptr = (vm_offset_t)thread->machine.kstackptr; |
2355 | kernel_stack_bottom = round_page(x: kstackptr) - KERNEL_STACK_SIZE; |
2356 | |
2357 | if ((sp < kernel_stack_bottom) && (sp >= (kernel_stack_bottom - PAGE_SIZE))) { |
2358 | panic_with_thread_kernel_state(msg: "Invalid kernel stack pointer (probable overflow)." , ss: &context->ss); |
2359 | } |
2360 | |
2361 | panic_with_thread_kernel_state(msg: "Invalid kernel stack pointer (probable corruption)." , ss: &context->ss); |
2362 | } |
2363 | |
2364 | |
2365 | #if DEVELOPMENT || DEBUG |
2366 | static int trap_handled; |
2367 | static void |
2368 | handle_recoverable_kernel_trap( |
2369 | __unused void *tstate, |
2370 | uint16_t comment) |
2371 | { |
2372 | assert(comment == TEST_RECOVERABLE_SOFT_TRAP); |
2373 | |
2374 | printf("Recoverable trap handled.\n" ); |
2375 | trap_handled = 1; |
2376 | } |
2377 | |
2378 | KERNEL_BRK_DESCRIPTOR_DEFINE(test_desc, |
2379 | .type = KERNEL_BRK_TYPE_TEST, |
2380 | .base = TEST_RECOVERABLE_SOFT_TRAP, |
2381 | .max = TEST_RECOVERABLE_SOFT_TRAP, |
2382 | .options = KERNEL_BRK_RECOVERABLE, |
2383 | .handle_breakpoint = handle_recoverable_kernel_trap); |
2384 | |
2385 | static int |
2386 | recoverable_kernel_trap_test(__unused int64_t in, int64_t *out) |
2387 | { |
2388 | ml_recoverable_trap(TEST_RECOVERABLE_SOFT_TRAP); |
2389 | |
2390 | *out = trap_handled; |
2391 | return 0; |
2392 | } |
2393 | |
2394 | SYSCTL_TEST_REGISTER(recoverable_kernel_trap, recoverable_kernel_trap_test); |
2395 | |
2396 | #endif |
2397 | |