1/*
2 * Copyright (c) 2012-2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <arm/caches_internal.h>
30#include <arm/cpu_data.h>
31#include <arm/cpu_data_internal.h>
32#include <arm/misc_protos.h>
33#include <arm/thread.h>
34#include <arm/rtclock.h>
35#include <arm/trap_internal.h> /* for IS_ARM_GDB_TRAP() et al */
36#include <arm64/proc_reg.h>
37#include <arm64/machine_machdep.h>
38#include <arm64/monotonic.h>
39#include <arm64/instructions.h>
40
41#include <kern/debug.h>
42#include <kern/restartable.h>
43#include <kern/socd_client.h>
44#include <kern/task.h>
45#include <kern/thread.h>
46#include <kern/zalloc_internal.h>
47#include <mach/exception.h>
48#include <mach/arm/traps.h>
49#include <mach/vm_types.h>
50#include <mach/machine/thread_status.h>
51
52#include <machine/atomic.h>
53#include <machine/limits.h>
54
55#include <pexpert/arm/protos.h>
56#include <pexpert/arm64/apple_arm64_regs.h>
57#include <pexpert/arm64/board_config.h>
58
59#include <vm/vm_page.h>
60#include <vm/pmap.h>
61#include <vm/vm_fault.h>
62#include <vm/vm_kern.h>
63
64#include <sys/errno.h>
65#include <sys/kdebug.h>
66#include <sys/code_signing.h>
67#include <kperf/kperf.h>
68
69#include <kern/policy_internal.h>
70#if CONFIG_TELEMETRY
71#include <kern/telemetry.h>
72#endif
73
74#include <prng/entropy.h>
75
76
77
78
79#include <arm64/platform_error_handler.h>
80
81#if KASAN_TBI
82#include <san/kasan.h>
83#endif /* KASAN_TBI */
84
85#if CONFIG_UBSAN_MINIMAL
86#include <san/ubsan_minimal.h>
87#endif
88
89
90#ifndef __arm64__
91#error Should only be compiling for arm64.
92#endif
93
94#if DEBUG || DEVELOPMENT
95#define HAS_TELEMETRY_KERNEL_BRK 1
96#endif
97
98
99#define TEST_CONTEXT32_SANITY(context) \
100 (context->ss.ash.flavor == ARM_SAVED_STATE32 && context->ss.ash.count == ARM_SAVED_STATE32_COUNT && \
101 context->ns.nsh.flavor == ARM_NEON_SAVED_STATE32 && context->ns.nsh.count == ARM_NEON_SAVED_STATE32_COUNT)
102
103#define TEST_CONTEXT64_SANITY(context) \
104 (context->ss.ash.flavor == ARM_SAVED_STATE64 && context->ss.ash.count == ARM_SAVED_STATE64_COUNT && \
105 context->ns.nsh.flavor == ARM_NEON_SAVED_STATE64 && context->ns.nsh.count == ARM_NEON_SAVED_STATE64_COUNT)
106
107#define ASSERT_CONTEXT_SANITY(context) \
108 assert(TEST_CONTEXT32_SANITY(context) || TEST_CONTEXT64_SANITY(context))
109
110
111#define COPYIN(src, dst, size) \
112 (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \
113 copyin_kern(src, dst, size) : \
114 copyin(src, dst, size)
115
116#define COPYOUT(src, dst, size) \
117 (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \
118 copyout_kern(src, dst, size) : \
119 copyout(src, dst, size)
120
121// Below is for concatenating a string param to a string literal
122#define STR1(x) #x
123#define STR(x) STR1(x)
124
125#define ARM64_KDBG_CODE_KERNEL (0 << 8)
126#define ARM64_KDBG_CODE_USER (1 << 8)
127#define ARM64_KDBG_CODE_GUEST (2 << 8)
128
129_Static_assert(ARM64_KDBG_CODE_GUEST <= KDBG_CODE_MAX, "arm64 KDBG trace codes out of range");
130_Static_assert(ARM64_KDBG_CODE_GUEST <= UINT16_MAX, "arm64 KDBG trace codes out of range");
131
132void panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss) __abortlike;
133
134void sleh_synchronous_sp1(arm_context_t *, uint32_t, vm_offset_t) __abortlike;
135void sleh_synchronous(arm_context_t *, uint32_t, vm_offset_t, bool);
136
137
138
139void sleh_irq(arm_saved_state_t *);
140void sleh_fiq(arm_saved_state_t *);
141void sleh_serror(arm_context_t *context, uint32_t esr, vm_offset_t far);
142void sleh_invalid_stack(arm_context_t *context, uint32_t esr, vm_offset_t far) __dead2;
143
144static void sleh_interrupt_handler_prologue(arm_saved_state_t *, unsigned int type);
145static void sleh_interrupt_handler_epilogue(void);
146
147static void handle_svc(arm_saved_state_t *);
148static void handle_mach_absolute_time_trap(arm_saved_state_t *);
149static void handle_mach_continuous_time_trap(arm_saved_state_t *);
150
151static void handle_msr_trap(arm_saved_state_t *state, uint32_t esr);
152#if __has_feature(ptrauth_calls)
153static void handle_pac_fail(arm_saved_state_t *state, uint32_t esr) __dead2;
154static inline uint64_t fault_addr_bitmask(unsigned int bit_from, unsigned int bit_to);
155#endif
156
157extern kern_return_t arm_fast_fault(pmap_t, vm_map_address_t, vm_prot_t, bool, bool);
158
159static void handle_uncategorized(arm_saved_state_t *);
160
161static void handle_kernel_breakpoint(arm_saved_state_t *, uint32_t);
162
163static void handle_breakpoint(arm_saved_state_t *, uint32_t) __dead2;
164
165typedef void (*abort_inspector_t)(uint32_t, fault_status_t *, vm_prot_t *);
166static void inspect_instruction_abort(uint32_t, fault_status_t *, vm_prot_t *);
167static void inspect_data_abort(uint32_t, fault_status_t *, vm_prot_t *);
168
169static int is_vm_fault(fault_status_t);
170static int is_translation_fault(fault_status_t);
171static int is_alignment_fault(fault_status_t);
172
173typedef void (*abort_handler_t)(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
174static void handle_user_abort(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
175static void handle_kernel_abort(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
176
177static void handle_pc_align(arm_saved_state_t *ss) __dead2;
178static void handle_sp_align(arm_saved_state_t *ss) __dead2;
179static void handle_sw_step_debug(arm_saved_state_t *ss) __dead2;
180static void handle_wf_trap(arm_saved_state_t *ss) __dead2;
181static void handle_fp_trap(arm_saved_state_t *ss, uint32_t esr) __dead2;
182
183static void handle_watchpoint(vm_offset_t fault_addr) __dead2;
184
185static void handle_abort(arm_saved_state_t *, uint32_t, vm_offset_t, abort_inspector_t, abort_handler_t, expected_fault_handler_t);
186
187static void handle_user_trapped_instruction32(arm_saved_state_t *, uint32_t esr) __dead2;
188
189static void handle_simd_trap(arm_saved_state_t *, uint32_t esr) __dead2;
190
191extern void current_cached_proc_cred_update(void);
192void mach_syscall_trace_exit(unsigned int retval, unsigned int call_number);
193
194struct proc;
195
196typedef uint32_t arm64_instr_t;
197
198extern void
199unix_syscall(struct arm_saved_state * regs, thread_t thread_act, struct proc * proc);
200
201extern void
202mach_syscall(struct arm_saved_state*);
203
204#if CONFIG_DTRACE
205extern kern_return_t dtrace_user_probe(arm_saved_state_t* regs);
206extern boolean_t dtrace_tally_fault(user_addr_t);
207
208/*
209 * Traps for userland processing. Can't include bsd/sys/fasttrap_isa.h, so copy
210 * and paste the trap instructions
211 * over from that file. Need to keep these in sync!
212 */
213#define FASTTRAP_ARM32_INSTR 0xe7ffdefc
214#define FASTTRAP_THUMB32_INSTR 0xdefc
215#define FASTTRAP_ARM64_INSTR 0xe7eeee7e
216
217#define FASTTRAP_ARM32_RET_INSTR 0xe7ffdefb
218#define FASTTRAP_THUMB32_RET_INSTR 0xdefb
219#define FASTTRAP_ARM64_RET_INSTR 0xe7eeee7d
220
221/* See <rdar://problem/4613924> */
222perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */
223#endif
224
225
226
227extern void arm64_thread_exception_return(void) __dead2;
228
229#if defined(APPLETYPHOON)
230#define CPU_NAME "Typhoon"
231#elif defined(APPLETWISTER)
232#define CPU_NAME "Twister"
233#elif defined(APPLEHURRICANE)
234#define CPU_NAME "Hurricane"
235#elif defined(APPLELIGHTNING)
236#define CPU_NAME "Lightning"
237#else
238#define CPU_NAME "Unknown"
239#endif
240
241#if (CONFIG_KERNEL_INTEGRITY && defined(KERNEL_INTEGRITY_WT))
242#define ESR_WT_SERROR(esr) (((esr) & 0xffffff00) == 0xbf575400)
243#define ESR_WT_REASON(esr) ((esr) & 0xff)
244
245#define WT_REASON_NONE 0
246#define WT_REASON_INTEGRITY_FAIL 1
247#define WT_REASON_BAD_SYSCALL 2
248#define WT_REASON_NOT_LOCKED 3
249#define WT_REASON_ALREADY_LOCKED 4
250#define WT_REASON_SW_REQ 5
251#define WT_REASON_PT_INVALID 6
252#define WT_REASON_PT_VIOLATION 7
253#define WT_REASON_REG_VIOLATION 8
254#endif
255
256#if defined(HAS_IPI)
257void cpu_signal_handler(void);
258extern unsigned int gFastIPI;
259#endif /* defined(HAS_IPI) */
260
261static arm_saved_state64_t *original_faulting_state = NULL;
262
263
264TUNABLE(bool, fp_exceptions_enabled, "-fp_exceptions", false);
265
266extern vm_offset_t static_memory_end;
267
268/*
269 * Fault copyio_recovery_entry in copyin/copyout routines.
270 *
271 * Offets are expressed in bytes from &copy_recovery_table
272 */
273struct copyio_recovery_entry {
274 ptrdiff_t cre_start;
275 ptrdiff_t cre_end;
276 ptrdiff_t cre_recovery;
277};
278
279extern struct copyio_recovery_entry copyio_recover_table[];
280extern struct copyio_recovery_entry copyio_recover_table_end[];
281
282static inline ptrdiff_t
283copyio_recovery_offset(uintptr_t addr)
284{
285 return (ptrdiff_t)(addr - (uintptr_t)copyio_recover_table);
286}
287
288#if !HAS_APPLE_PAC
289static inline uintptr_t
290copyio_recovery_addr(ptrdiff_t offset)
291{
292 return (uintptr_t)copyio_recover_table + (uintptr_t)offset;
293}
294#endif
295
296static inline struct copyio_recovery_entry *
297find_copyio_recovery_entry(arm_saved_state_t *state)
298{
299 ptrdiff_t offset = copyio_recovery_offset(addr: get_saved_state_pc(iss: state));
300 struct copyio_recovery_entry *e;
301
302 for (e = copyio_recover_table; e < copyio_recover_table_end; e++) {
303 if (offset >= e->cre_start && offset < e->cre_end) {
304 return e;
305 }
306 }
307
308 return NULL;
309}
310
311static inline int
312is_vm_fault(fault_status_t status)
313{
314 switch (status) {
315 case FSC_TRANSLATION_FAULT_L0:
316 case FSC_TRANSLATION_FAULT_L1:
317 case FSC_TRANSLATION_FAULT_L2:
318 case FSC_TRANSLATION_FAULT_L3:
319 case FSC_ACCESS_FLAG_FAULT_L1:
320 case FSC_ACCESS_FLAG_FAULT_L2:
321 case FSC_ACCESS_FLAG_FAULT_L3:
322 case FSC_PERMISSION_FAULT_L1:
323 case FSC_PERMISSION_FAULT_L2:
324 case FSC_PERMISSION_FAULT_L3:
325 return TRUE;
326 default:
327 return FALSE;
328 }
329}
330
331static inline int
332is_translation_fault(fault_status_t status)
333{
334 switch (status) {
335 case FSC_TRANSLATION_FAULT_L0:
336 case FSC_TRANSLATION_FAULT_L1:
337 case FSC_TRANSLATION_FAULT_L2:
338 case FSC_TRANSLATION_FAULT_L3:
339 return TRUE;
340 default:
341 return FALSE;
342 }
343}
344
345static inline int
346is_permission_fault(fault_status_t status)
347{
348 switch (status) {
349 case FSC_PERMISSION_FAULT_L1:
350 case FSC_PERMISSION_FAULT_L2:
351 case FSC_PERMISSION_FAULT_L3:
352 return TRUE;
353 default:
354 return FALSE;
355 }
356}
357
358static inline int
359is_alignment_fault(fault_status_t status)
360{
361 return status == FSC_ALIGNMENT_FAULT;
362}
363
364static inline int
365is_parity_error(fault_status_t status)
366{
367 switch (status) {
368#if defined(ARM64_BOARD_CONFIG_T6020)
369 /*
370 * H14 Erratum (rdar://61553243): Despite having FEAT_RAS implemented,
371 * FSC_SYNC_PARITY_X can be reported for data and instruction aborts
372 * and should be interpreted as FSC_SYNC_EXT_ABORT_x
373 */
374#else
375 /*
376 * TODO: According to ARM ARM, Async Parity (0b011001) is a DFSC that is
377 * only applicable to AArch32 HSR register. Can this be removed?
378 */
379 case FSC_ASYNC_PARITY:
380 case FSC_SYNC_PARITY:
381 case FSC_SYNC_PARITY_TT_L1:
382 case FSC_SYNC_PARITY_TT_L2:
383 case FSC_SYNC_PARITY_TT_L3:
384 return TRUE;
385#endif
386 default:
387 return FALSE;
388 }
389}
390
391
392
393__dead2 __unused
394static void
395arm64_implementation_specific_error(arm_saved_state_t *state, uint32_t esr, vm_offset_t far)
396{
397#pragma unused (state, esr, far)
398 panic_plain("Unhandled implementation specific error\n");
399}
400
401#if CONFIG_KERNEL_INTEGRITY
402#pragma clang diagnostic push
403#pragma clang diagnostic ignored "-Wunused-parameter"
404static void
405kernel_integrity_error_handler(uint32_t esr, vm_offset_t far)
406{
407#if defined(KERNEL_INTEGRITY_WT)
408#if (DEVELOPMENT || DEBUG)
409 if (ESR_WT_SERROR(esr)) {
410 switch (ESR_WT_REASON(esr)) {
411 case WT_REASON_INTEGRITY_FAIL:
412 panic_plain("Kernel integrity, violation in frame 0x%016lx.", far);
413 case WT_REASON_BAD_SYSCALL:
414 panic_plain("Kernel integrity, bad syscall.");
415 case WT_REASON_NOT_LOCKED:
416 panic_plain("Kernel integrity, not locked.");
417 case WT_REASON_ALREADY_LOCKED:
418 panic_plain("Kernel integrity, already locked.");
419 case WT_REASON_SW_REQ:
420 panic_plain("Kernel integrity, software request.");
421 case WT_REASON_PT_INVALID:
422 panic_plain("Kernel integrity, encountered invalid TTE/PTE while "
423 "walking 0x%016lx.", far);
424 case WT_REASON_PT_VIOLATION:
425 panic_plain("Kernel integrity, violation in mapping 0x%016lx.",
426 far);
427 case WT_REASON_REG_VIOLATION:
428 panic_plain("Kernel integrity, violation in system register %d.",
429 (unsigned) far);
430 default:
431 panic_plain("Kernel integrity, unknown (esr=0x%08x).", esr);
432 }
433 }
434#else
435 if (ESR_WT_SERROR(esr)) {
436 panic_plain("SError esr: 0x%08x far: 0x%016lx.", esr, far);
437 }
438#endif
439#endif
440}
441#pragma clang diagnostic pop
442#endif
443
444static void
445arm64_platform_error(arm_saved_state_t *state, uint32_t esr, vm_offset_t far, platform_error_source_t source)
446{
447#if CONFIG_KERNEL_INTEGRITY
448 kernel_integrity_error_handler(esr, far);
449#endif
450
451 (void)source;
452 cpu_data_t *cdp = getCpuDatap();
453
454 if (PE_handle_platform_error(far)) {
455 return;
456 } else if (cdp->platform_error_handler != NULL) {
457 cdp->platform_error_handler(cdp->cpu_id, far);
458 } else {
459 arm64_implementation_specific_error(state, esr, far);
460 }
461}
462
463void
464panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss)
465{
466 boolean_t ss_valid;
467
468 ss_valid = is_saved_state64(iss: ss);
469 arm_saved_state64_t *state = saved_state64(iss: ss);
470
471 os_atomic_cmpxchg(&original_faulting_state, NULL, state, seq_cst);
472
473 // rdar://80659177
474 // Read SoCD tracepoints up to twice — once the first time we call panic and
475 // another time if we encounter a nested panic after that.
476 static int twice = 2;
477 if (twice > 0) {
478 twice--;
479 SOCD_TRACE_XNU(KERNEL_STATE_PANIC, ADDR(state->pc),
480 PACK_LSB(VALUE(state->lr), VALUE(ss_valid)),
481 PACK_2X32(VALUE(state->esr), VALUE(state->cpsr)),
482 VALUE(state->far));
483 }
484
485
486 panic_plain("%s at pc 0x%016llx, lr 0x%016llx (saved state: %p%s)\n"
487 "\t x0: 0x%016llx x1: 0x%016llx x2: 0x%016llx x3: 0x%016llx\n"
488 "\t x4: 0x%016llx x5: 0x%016llx x6: 0x%016llx x7: 0x%016llx\n"
489 "\t x8: 0x%016llx x9: 0x%016llx x10: 0x%016llx x11: 0x%016llx\n"
490 "\t x12: 0x%016llx x13: 0x%016llx x14: 0x%016llx x15: 0x%016llx\n"
491 "\t x16: 0x%016llx x17: 0x%016llx x18: 0x%016llx x19: 0x%016llx\n"
492 "\t x20: 0x%016llx x21: 0x%016llx x22: 0x%016llx x23: 0x%016llx\n"
493 "\t x24: 0x%016llx x25: 0x%016llx x26: 0x%016llx x27: 0x%016llx\n"
494 "\t x28: 0x%016llx fp: 0x%016llx lr: 0x%016llx sp: 0x%016llx\n"
495 "\t pc: 0x%016llx cpsr: 0x%08x esr: 0x%08x far: 0x%016llx\n",
496 msg, state->pc, state->lr, ss, (ss_valid ? "" : " INVALID"),
497 state->x[0], state->x[1], state->x[2], state->x[3],
498 state->x[4], state->x[5], state->x[6], state->x[7],
499 state->x[8], state->x[9], state->x[10], state->x[11],
500 state->x[12], state->x[13], state->x[14], state->x[15],
501 state->x[16], state->x[17], state->x[18], state->x[19],
502 state->x[20], state->x[21], state->x[22], state->x[23],
503 state->x[24], state->x[25], state->x[26], state->x[27],
504 state->x[28], state->fp, state->lr, state->sp,
505 state->pc, state->cpsr, state->esr, state->far);
506}
507
508void
509sleh_synchronous_sp1(arm_context_t *context, uint32_t esr, vm_offset_t far __unused)
510{
511 esr_exception_class_t class = ESR_EC(esr);
512 arm_saved_state_t * state = &context->ss;
513
514 switch (class) {
515 case ESR_EC_UNCATEGORIZED:
516 {
517#if (DEVELOPMENT || DEBUG)
518 uint32_t instr = *((uint32_t*)get_saved_state_pc(state));
519 if (IS_ARM_GDB_TRAP(instr)) {
520 DebuggerCall(EXC_BREAKPOINT, state);
521 }
522 OS_FALLTHROUGH; // panic if we return from the debugger
523#else
524 panic_with_thread_kernel_state(msg: "Unexpected debugger trap while SP1 selected", ss: state);
525#endif /* (DEVELOPMENT || DEBUG) */
526 }
527 default:
528 panic_with_thread_kernel_state(msg: "Synchronous exception taken while SP1 selected", ss: state);
529 }
530}
531
532
533__attribute__((noreturn))
534void
535thread_exception_return()
536{
537 thread_t thread = current_thread();
538 if (thread->machine.exception_trace_code != 0) {
539 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
540 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_END, 0, 0, 0, 0, 0);
541 thread->machine.exception_trace_code = 0;
542 }
543
544#if KASAN_TBI
545 kasan_unpoison_curstack(true);
546#endif /* KASAN_TBI */
547 arm64_thread_exception_return();
548 __builtin_unreachable();
549}
550
551/*
552 * check whether task vtimers are running and set thread and CPU BSD AST
553 *
554 * must be called with interrupts masked so updates of fields are atomic
555 * must be emitted inline to avoid generating an FBT probe on the exception path
556 *
557 */
558__attribute__((__always_inline__))
559static inline void
560task_vtimer_check(thread_t thread)
561{
562 task_t task = get_threadtask_early(thread);
563
564 if (__improbable(task != NULL && task->vtimers)) {
565 thread_ast_set(thread, AST_BSD);
566 thread->machine.CpuDatap->cpu_pending_ast |= AST_BSD;
567 }
568}
569
570#if MACH_ASSERT
571/**
572 * A version of get_preemption_level() that works in early boot.
573 *
574 * If an exception is raised in early boot before the initial thread has been
575 * set up, then calling get_preemption_level() in the SLEH will trigger an
576 * infinitely-recursing exception. This function handles this edge case.
577 */
578static inline int
579sleh_get_preemption_level(void)
580{
581 if (__improbable(current_thread() == NULL)) {
582 return 0;
583 }
584 return get_preemption_level();
585}
586#endif // MACH_ASSERT
587
588static inline bool
589is_platform_error(uint32_t esr)
590{
591 esr_exception_class_t class = ESR_EC(esr);
592 uint32_t iss = ESR_ISS(esr);
593 fault_status_t fault_code;
594
595 if (class == ESR_EC_DABORT_EL0 || class == ESR_EC_DABORT_EL1) {
596 fault_code = ISS_DA_FSC(iss);
597 } else if (class == ESR_EC_IABORT_EL0 || class == ESR_EC_IABORT_EL1) {
598 fault_code = ISS_IA_FSC(iss);
599 } else {
600 return false;
601 }
602
603 return fault_code == FSC_SYNC_PARITY;
604}
605
606void
607sleh_synchronous(arm_context_t *context, uint32_t esr, vm_offset_t far, __unused bool did_initiate_panic_lockdown)
608{
609 esr_exception_class_t class = ESR_EC(esr);
610 arm_saved_state_t * state = &context->ss;
611 thread_t thread = current_thread();
612#if MACH_ASSERT
613 int preemption_level = sleh_get_preemption_level();
614#endif
615 expected_fault_handler_t expected_fault_handler = NULL;
616#ifdef CONFIG_XNUPOST
617 expected_fault_handler_t saved_expected_fault_handler = NULL;
618 uintptr_t saved_expected_fault_addr = 0;
619 uintptr_t saved_expected_fault_pc = 0;
620#endif /* CONFIG_XNUPOST */
621
622 ASSERT_CONTEXT_SANITY(context);
623
624 task_vtimer_check(thread);
625
626#if CONFIG_DTRACE
627 /*
628 * Handle kernel DTrace probes as early as possible to minimize the likelihood
629 * that this path will itself trigger a DTrace probe, which would lead to infinite
630 * probe recursion.
631 */
632 if (__improbable((class == ESR_EC_UNCATEGORIZED) && tempDTraceTrapHook &&
633 (tempDTraceTrapHook(EXC_BAD_INSTRUCTION, state, 0, 0) == KERN_SUCCESS))) {
634#if CONFIG_SPTM
635 if (__improbable(did_initiate_panic_lockdown)) {
636 panic("Unexpectedly initiated lockdown for DTrace probe?");
637 }
638#endif
639 return;
640 }
641#endif
642 bool is_user = PSR64_IS_USER(get_saved_state_cpsr(state));
643
644#if CONFIG_SPTM
645 // Lockdown should only be initiated for kernel exceptions
646 assert(!(is_user && did_initiate_panic_lockdown));
647#endif /* CONFIG_SPTM */
648
649 /*
650 * Use KERNEL_DEBUG_CONSTANT_IST here to avoid producing tracepoints
651 * that would disclose the behavior of PT_DENY_ATTACH processes.
652 */
653 if (is_user) {
654 /* Sanitize FAR (but only if the exception was taken from userspace) */
655 switch (class) {
656 case ESR_EC_IABORT_EL1:
657 case ESR_EC_IABORT_EL0:
658 /* If this is a SEA, since we can't trust FnV, just clear FAR from the save area. */
659 if (ISS_IA_FSC(ESR_ISS(esr)) == FSC_SYNC_EXT_ABORT) {
660 saved_state64(iss: state)->far = 0;
661 }
662 break;
663 case ESR_EC_DABORT_EL1:
664 case ESR_EC_DABORT_EL0:
665 /* If this is a SEA, since we can't trust FnV, just clear FAR from the save area. */
666 if (ISS_DA_FSC(ESR_ISS(esr)) == FSC_SYNC_EXT_ABORT) {
667 saved_state64(iss: state)->far = 0;
668 }
669 break;
670 case ESR_EC_WATCHPT_MATCH_EL1:
671 case ESR_EC_WATCHPT_MATCH_EL0:
672 case ESR_EC_PC_ALIGN:
673 break; /* FAR_ELx is valid */
674 default:
675 saved_state64(iss: state)->far = 0;
676 break;
677 }
678
679 thread->machine.exception_trace_code = (uint16_t)(ARM64_KDBG_CODE_USER | class);
680 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
681 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_START,
682 esr, far, get_saved_state_pc(state), 0, 0);
683 } else {
684 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
685 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, ARM64_KDBG_CODE_KERNEL | class) | DBG_FUNC_START,
686 esr, VM_KERNEL_ADDRHIDE(far), VM_KERNEL_UNSLIDE(get_saved_state_pc(state)), 0, 0);
687 }
688
689 if (__improbable(ESR_INSTR_IS_2BYTES(esr))) {
690 /*
691 * We no longer support 32-bit, which means no 2-byte
692 * instructions.
693 */
694 if (is_user) {
695 panic("Exception on 2-byte instruction, "
696 "context=%p, esr=%#x, far=%p",
697 context, esr, (void *)far);
698 } else {
699 panic_with_thread_kernel_state(msg: "Exception on 2-byte instruction", ss: state);
700 }
701 }
702
703#ifdef CONFIG_XNUPOST
704 if (thread->machine.expected_fault_handler != NULL) {
705 bool matching_fault_pc = false;
706 saved_expected_fault_handler = thread->machine.expected_fault_handler;
707 saved_expected_fault_addr = thread->machine.expected_fault_addr;
708 saved_expected_fault_pc = thread->machine.expected_fault_pc;
709
710 thread->machine.expected_fault_handler = NULL;
711 thread->machine.expected_fault_addr = 0;
712 thread->machine.expected_fault_pc = 0;
713
714#if __has_feature(ptrauth_calls)
715 /*
716 * Compare only the bits of PC which make up the virtual address.
717 * This ignores the upper bits, which may have been corrupted by HW in
718 * platform dependent ways to signal pointer authentication fault.
719 */
720 uint64_t fault_addr_mask = fault_addr_bitmask(0, 64 - T1SZ_BOOT - 1);
721 uint64_t masked_expected_pc = saved_expected_fault_pc & fault_addr_mask;
722 uint64_t masked_saved_pc = get_saved_state_pc(state) & fault_addr_mask;
723 matching_fault_pc = masked_expected_pc == masked_saved_pc;
724#else
725 matching_fault_pc =
726 (saved_expected_fault_pc == get_saved_state_pc(state));
727#endif /* ptrauth_call */
728 if (saved_expected_fault_addr == far ||
729 matching_fault_pc) {
730 expected_fault_handler = saved_expected_fault_handler;
731 }
732 }
733#endif /* CONFIG_XNUPOST */
734
735 if (__improbable(is_platform_error(esr))) {
736 /*
737 * Must gather error info in platform error handler before
738 * thread is preempted to another core/cluster to guarantee
739 * accurate error details
740 */
741
742 arm64_platform_error(state, esr, far, source: PLAT_ERR_SRC_SYNC);
743#if CONFIG_SPTM
744 if (__improbable(did_initiate_panic_lockdown)) {
745 panic("Panic lockdown initiated for platform error");
746 }
747#endif
748 return;
749 }
750
751 if (is_user && class == ESR_EC_DABORT_EL0) {
752 thread_reset_pcs_will_fault(thread);
753 }
754
755 /* Inherit the interrupt masks from previous context */
756 if (SPSR_INTERRUPTS_ENABLED(get_saved_state_cpsr(state))) {
757 ml_set_interrupts_enabled(TRUE);
758 }
759
760 switch (class) {
761 case ESR_EC_SVC_64:
762 if (!is_saved_state64(iss: state) || !is_user) {
763 panic("Invalid SVC_64 context");
764 }
765
766 handle_svc(state);
767 break;
768
769 case ESR_EC_DABORT_EL0:
770 handle_abort(state, esr, far, inspect_data_abort, handle_user_abort, expected_fault_handler);
771 break;
772
773 case ESR_EC_MSR_TRAP:
774 handle_msr_trap(state, esr);
775 break;
776/**
777 * Some APPLEVIRTUALPLATFORM targets do not specify armv8.6, but it's still possible for
778 * them to be hosted by a host that implements ARM_FPAC. There's no way for such a host
779 * to disable it or trap it without substantial performance penalty. Therefore, the FPAC
780 * handler here needs to be built into the guest kernels to prevent the exception to fall
781 * through.
782 */
783#if __has_feature(ptrauth_calls)
784 case ESR_EC_PAC_FAIL:
785#ifdef CONFIG_XNUPOST
786 if (expected_fault_handler != NULL && expected_fault_handler(state)) {
787 break;
788 }
789#endif /* CONFIG_XNUPOST */
790 handle_pac_fail(state, esr);
791 __builtin_unreachable();
792
793#endif /* __has_feature(ptrauth_calls) */
794
795
796 case ESR_EC_IABORT_EL0:
797 handle_abort(state, esr, far, inspect_instruction_abort, handle_user_abort, expected_fault_handler);
798 break;
799
800 case ESR_EC_IABORT_EL1:
801#ifdef CONFIG_XNUPOST
802 if ((expected_fault_handler != NULL) && expected_fault_handler(state)) {
803 break;
804 }
805#endif /* CONFIG_XNUPOST */
806
807 panic_with_thread_kernel_state(msg: "Kernel instruction fetch abort", ss: state);
808
809 case ESR_EC_PC_ALIGN:
810 handle_pc_align(ss: state);
811 __builtin_unreachable();
812
813 case ESR_EC_DABORT_EL1:
814 handle_abort(state, esr, far, inspect_data_abort, handle_kernel_abort, expected_fault_handler);
815 break;
816
817 case ESR_EC_UNCATEGORIZED:
818 assert(!ESR_ISS(esr));
819
820#if CONFIG_XNUPOST
821 if (!is_user && (expected_fault_handler != NULL) && expected_fault_handler(state)) {
822 /*
823 * The fault handler accepted the exception and handled it on its
824 * own. Don't trap to the debugger/panic.
825 */
826 break;
827 }
828#endif /* CONFIG_XNUPOST */
829 handle_uncategorized(&context->ss);
830 break;
831
832 case ESR_EC_SP_ALIGN:
833 handle_sp_align(ss: state);
834 __builtin_unreachable();
835
836 case ESR_EC_BKPT_AARCH32:
837 handle_breakpoint(state, esr);
838 __builtin_unreachable();
839
840 case ESR_EC_BRK_AARCH64:
841#ifdef CONFIG_XNUPOST
842 if ((expected_fault_handler != NULL) && expected_fault_handler(state)) {
843 break;
844 }
845#endif /* CONFIG_XNUPOST */
846 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
847 handle_kernel_breakpoint(state, esr);
848 break;
849 } else {
850 handle_breakpoint(state, esr);
851 __builtin_unreachable();
852 }
853
854 case ESR_EC_BKPT_REG_MATCH_EL0:
855 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
856 handle_breakpoint(state, esr);
857 }
858 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
859 class, state, class, esr, (void *)far);
860 __builtin_unreachable();
861
862 case ESR_EC_BKPT_REG_MATCH_EL1:
863 panic_with_thread_kernel_state(msg: "Hardware Breakpoint Debug exception from kernel. Panic (by design)", ss: state);
864 __builtin_unreachable();
865
866 case ESR_EC_SW_STEP_DEBUG_EL0:
867 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
868 handle_sw_step_debug(ss: state);
869 }
870 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
871 class, state, class, esr, (void *)far);
872 __builtin_unreachable();
873
874 case ESR_EC_SW_STEP_DEBUG_EL1:
875 panic_with_thread_kernel_state(msg: "Software Step Debug exception from kernel. Panic (by design)", ss: state);
876 __builtin_unreachable();
877
878 case ESR_EC_WATCHPT_MATCH_EL0:
879 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
880 handle_watchpoint(fault_addr: far);
881 }
882 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
883 class, state, class, esr, (void *)far);
884 __builtin_unreachable();
885
886 case ESR_EC_WATCHPT_MATCH_EL1:
887 /*
888 * If we hit a watchpoint in kernel mode, probably in a copyin/copyout which we don't want to
889 * abort. Turn off watchpoints and keep going; we'll turn them back on in return_from_exception..
890 */
891 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
892 arm_debug_set(NULL);
893 break; /* return to first level handler */
894 }
895 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
896 class, state, class, esr, (void *)far);
897 __builtin_unreachable();
898
899 case ESR_EC_TRAP_SIMD_FP:
900 handle_simd_trap(state, esr);
901 __builtin_unreachable();
902
903 case ESR_EC_ILLEGAL_INSTR_SET:
904 panic("Illegal instruction set exception. state=%p class=%u esr=%u far=%p spsr=0x%x",
905 state, class, esr, (void *)far, get_saved_state_cpsr(state));
906 __builtin_unreachable();
907
908 case ESR_EC_MCR_MRC_CP15_TRAP:
909 case ESR_EC_MCRR_MRRC_CP15_TRAP:
910 case ESR_EC_MCR_MRC_CP14_TRAP:
911 case ESR_EC_LDC_STC_CP14_TRAP:
912 case ESR_EC_MCRR_MRRC_CP14_TRAP:
913 handle_user_trapped_instruction32(state, esr);
914 __builtin_unreachable();
915
916 case ESR_EC_WFI_WFE:
917 // Use of WFI or WFE instruction when they have been disabled for EL0
918 handle_wf_trap(ss: state);
919 __builtin_unreachable();
920
921 case ESR_EC_FLOATING_POINT_64:
922 handle_fp_trap(ss: state, esr);
923 __builtin_unreachable();
924
925 default:
926 handle_uncategorized(state);
927 }
928
929#ifdef CONFIG_XNUPOST
930 if (saved_expected_fault_handler != NULL) {
931 thread->machine.expected_fault_handler = saved_expected_fault_handler;
932 thread->machine.expected_fault_addr = saved_expected_fault_addr;
933 thread->machine.expected_fault_pc = saved_expected_fault_pc;
934 }
935#endif /* CONFIG_XNUPOST */
936
937 if (is_user) {
938 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
939 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_END,
940 esr, far, get_saved_state_pc(state), 0, 0);
941 thread->machine.exception_trace_code = 0;
942 } else {
943 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
944 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, ARM64_KDBG_CODE_KERNEL | class) | DBG_FUNC_END,
945 esr, VM_KERNEL_ADDRHIDE(far), VM_KERNEL_UNSLIDE(get_saved_state_pc(state)), 0, 0);
946 }
947#if MACH_ASSERT
948 if (preemption_level != sleh_get_preemption_level()) {
949 panic("synchronous exception changed preemption level from %d to %d", preemption_level, sleh_get_preemption_level());
950 }
951#endif
952
953#if CONFIG_SPTM
954 if (did_initiate_panic_lockdown
955#if CONFIG_XNUPOST
956 /* Do not engage the panic interlock if we matched a fault handler */
957 && !expected_fault_handler
958#endif /* CONFIG_XNUPOST */
959 ) {
960 /*
961 * fleh already triggered a lockdown but we, for whatever reason, didn't
962 * end up finding a reason to panic. Catch all panic in this case.
963 * Note that the panic here has no security benefit as the system is already
964 * hosed, this is merely for telemetry.
965 */
966 panic_with_thread_kernel_state("Panic lockdown initiated", state);
967 }
968#endif /* CONFIG_SPTM */
969}
970
971/*
972 * Uncategorized exceptions are a catch-all for general execution errors.
973 * ARM64_TODO: For now, we assume this is for undefined instruction exceptions.
974 */
975static void
976handle_uncategorized(arm_saved_state_t *state)
977{
978 exception_type_t exception = EXC_BAD_INSTRUCTION;
979 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
980 mach_msg_type_number_t numcodes = 2;
981 uint32_t instr = 0;
982
983 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
984
985#if CONFIG_DTRACE
986
987 if (PSR64_IS_USER64(get_saved_state_cpsr(state))) {
988 /*
989 * For a 64bit user process, we care about all 4 bytes of the
990 * instr.
991 */
992 if (instr == FASTTRAP_ARM64_INSTR || instr == FASTTRAP_ARM64_RET_INSTR) {
993 if (dtrace_user_probe(regs: state) == KERN_SUCCESS) {
994 return;
995 }
996 }
997 } else if (PSR64_IS_USER32(get_saved_state_cpsr(state))) {
998 /*
999 * For a 32bit user process, we check for thumb mode, in
1000 * which case we only care about a 2 byte instruction length.
1001 * For non-thumb mode, we care about all 4 bytes of the instructin.
1002 */
1003 if (get_saved_state_cpsr(iss: state) & PSR64_MODE_USER32_THUMB) {
1004 if (((uint16_t)instr == FASTTRAP_THUMB32_INSTR) ||
1005 ((uint16_t)instr == FASTTRAP_THUMB32_RET_INSTR)) {
1006 if (dtrace_user_probe(regs: state) == KERN_SUCCESS) {
1007 return;
1008 }
1009 }
1010 } else {
1011 if ((instr == FASTTRAP_ARM32_INSTR) ||
1012 (instr == FASTTRAP_ARM32_RET_INSTR)) {
1013 if (dtrace_user_probe(regs: state) == KERN_SUCCESS) {
1014 return;
1015 }
1016 }
1017 }
1018 }
1019
1020#endif /* CONFIG_DTRACE */
1021
1022 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1023 if (IS_ARM_GDB_TRAP(instr)) {
1024 boolean_t interrupt_state;
1025 exception = EXC_BREAKPOINT;
1026
1027 interrupt_state = ml_set_interrupts_enabled(FALSE);
1028
1029 /* Save off the context here (so that the debug logic
1030 * can see the original state of this thread).
1031 */
1032 current_thread()->machine.kpcb = state;
1033
1034 /* Hop into the debugger (typically either due to a
1035 * fatal exception, an explicit panic, or a stackshot
1036 * request.
1037 */
1038 DebuggerCall(reason: exception, ctx: state);
1039
1040 current_thread()->machine.kpcb = NULL;
1041 (void) ml_set_interrupts_enabled(enable: interrupt_state);
1042 return;
1043 } else {
1044 panic("Undefined kernel instruction: pc=%p instr=%x", (void*)get_saved_state_pc(state), instr);
1045 }
1046 }
1047
1048 /*
1049 * Check for GDB breakpoint via illegal opcode.
1050 */
1051 if (IS_ARM_GDB_TRAP(instr)) {
1052 exception = EXC_BREAKPOINT;
1053 codes[0] = EXC_ARM_BREAKPOINT;
1054 codes[1] = instr;
1055 } else {
1056 codes[1] = instr;
1057 }
1058
1059 exception_triage(exception, code: codes, codeCnt: numcodes);
1060 __builtin_unreachable();
1061}
1062
1063#if __has_feature(ptrauth_calls)
1064static const uint16_t PTRAUTH_TRAP_START = 0xC470;
1065static inline bool
1066brk_comment_is_ptrauth(uint16_t comment)
1067{
1068 return comment >= PTRAUTH_TRAP_START &&
1069 comment <= PTRAUTH_TRAP_START + ptrauth_key_asdb;
1070}
1071
1072static inline const char *
1073ptrauth_key_to_string(ptrauth_key key)
1074{
1075 switch (key) {
1076 case ptrauth_key_asia:
1077 return "IA";
1078 case ptrauth_key_asib:
1079 return "IB";
1080 case ptrauth_key_asda:
1081 return "DA";
1082 case ptrauth_key_asdb:
1083 return "DB";
1084 default:
1085 __builtin_unreachable();
1086 }
1087}
1088
1089static void __attribute__((noreturn))
1090ptrauth_handle_brk_trap(void *tstate, uint16_t comment)
1091{
1092 arm_saved_state_t *state = (arm_saved_state_t *)tstate;
1093#define MSG_FMT "Break 0x%04X instruction exception from kernel. Ptrauth failure with %s key resulted in 0x%016llx"
1094 char msg[strlen(MSG_FMT)
1095 - strlen("0x%04X") + strlen("0xFFFF")
1096 - strlen("%s") + strlen("IA")
1097 - strlen("0x%016llx") + strlen("0xFFFFFFFFFFFFFFFF")
1098 + 1];
1099 ptrauth_key key = (ptrauth_key)(comment - PTRAUTH_TRAP_START);
1100 const char *key_str = ptrauth_key_to_string(key);
1101 snprintf(msg, sizeof(msg), MSG_FMT, comment, key_str, saved_state64(state)->x[16]);
1102#undef MSG_FMT
1103
1104 panic_with_thread_kernel_state(msg, state);
1105 __builtin_unreachable();
1106}
1107#endif /* __has_feature(ptrauth_calls) */
1108
1109#if HAS_TELEMETRY_KERNEL_BRK
1110static uint32_t bound_chk_violations_event;
1111
1112static void
1113xnu_soft_trap_handle_breakpoint(
1114 void *tstate,
1115 uint16_t comment)
1116{
1117#if CONFIG_UBSAN_MINIMAL
1118 if (comment == UBSAN_SOFT_TRAP_SIGNED_OF) {
1119 ubsan_handle_brk_trap(tstate, comment);
1120 }
1121#else
1122 (void)tstate;
1123#endif
1124
1125 if (comment == CLANG_SOFT_TRAP_BOUND_CHK) {
1126 os_atomic_inc(&bound_chk_violations_event, relaxed);
1127 }
1128}
1129#endif /* HAS_TELEMETRY_KERNEL_BRK */
1130
1131static void
1132xnu_hard_trap_handle_breakpoint(void *tstate, uint16_t comment)
1133{
1134 switch (comment) {
1135 case XNU_HARD_TRAP_STRING_CHK:
1136 panic_with_thread_kernel_state(msg: "panic: string operation caused an overflow", ss: tstate);
1137 default:
1138 break;
1139 }
1140}
1141
1142#if __has_feature(ptrauth_calls)
1143KERNEL_BRK_DESCRIPTOR_DEFINE(ptrauth_desc,
1144 .type = KERNEL_BRK_TYPE_PTRAUTH,
1145 .base = PTRAUTH_TRAP_START,
1146 .max = PTRAUTH_TRAP_START + ptrauth_key_asdb,
1147 .options = KERNEL_BRK_UNRECOVERABLE,
1148 .handle_breakpoint = ptrauth_handle_brk_trap);
1149#endif
1150
1151KERNEL_BRK_DESCRIPTOR_DEFINE(clang_desc,
1152 .type = KERNEL_BRK_TYPE_CLANG,
1153 .base = CLANG_ARM_TRAP_START,
1154 .max = CLANG_ARM_TRAP_END,
1155 .options = KERNEL_BRK_UNRECOVERABLE,
1156 .handle_breakpoint = NULL);
1157
1158KERNEL_BRK_DESCRIPTOR_DEFINE(libcxx_desc,
1159 .type = KERNEL_BRK_TYPE_LIBCXX,
1160 .base = LIBCXX_TRAP_START,
1161 .max = LIBCXX_TRAP_END,
1162 .options = KERNEL_BRK_UNRECOVERABLE,
1163 .handle_breakpoint = NULL);
1164
1165#if HAS_TELEMETRY_KERNEL_BRK
1166KERNEL_BRK_DESCRIPTOR_DEFINE(xnu_soft_traps_desc,
1167 .type = KERNEL_BRK_TYPE_TELEMETRY,
1168 .base = XNU_SOFT_TRAP_START,
1169 .max = XNU_SOFT_TRAP_END,
1170 .options = KERNEL_BRK_RECOVERABLE | KERNEL_BRK_CORE_ANALYTICS,
1171 .handle_breakpoint = xnu_soft_trap_handle_breakpoint);
1172#endif /* HAS_TELEMETRY_KERNEL_BRK */
1173
1174KERNEL_BRK_DESCRIPTOR_DEFINE(xnu_hard_traps_desc,
1175 .type = KERNEL_BRK_TYPE_XNU,
1176 .base = XNU_HARD_TRAP_START,
1177 .max = XNU_HARD_TRAP_END,
1178 .options = KERNEL_BRK_UNRECOVERABLE,
1179 .handle_breakpoint = xnu_hard_trap_handle_breakpoint);
1180
1181static void
1182#if !HAS_TELEMETRY_KERNEL_BRK
1183__attribute__((noreturn))
1184#endif
1185handle_kernel_breakpoint(arm_saved_state_t *state, uint32_t esr)
1186{
1187 uint16_t comment = ISS_BRK_COMMENT(esr);
1188 const struct kernel_brk_descriptor *desc;
1189
1190#define MSG_FMT "Break 0x%04X instruction exception from kernel. Panic (by design)"
1191 char msg[strlen(MSG_FMT) - strlen(s: "0x%04X") + strlen(s: "0xFFFF") + 1];
1192
1193 desc = find_brk_descriptor_by_comment(comment);
1194
1195 if (!desc) {
1196 goto brk_out;
1197 }
1198
1199#if HAS_TELEMETRY_KERNEL_BRK
1200 if (desc->options & KERNEL_BRK_TELEMETRY_OPTIONS) {
1201 telemetry_kernel_brk(desc->type, desc->options, (void *)state, comment);
1202 }
1203#endif
1204
1205 if (desc->handle_breakpoint) {
1206 desc->handle_breakpoint(state, comment); /* May trigger panic */
1207 }
1208
1209#if HAS_TELEMETRY_KERNEL_BRK
1210 /* Still alive? Check if we should recover. */
1211 if (desc->options & KERNEL_BRK_RECOVERABLE) {
1212 add_saved_state_pc(state, 4);
1213 return;
1214 }
1215#endif
1216
1217brk_out:
1218 snprintf(msg, sizeof(msg), MSG_FMT, comment);
1219
1220 panic_with_thread_kernel_state(msg, ss: state);
1221 __builtin_unreachable();
1222#undef MSG_FMT
1223}
1224
1225static void
1226handle_breakpoint(arm_saved_state_t *state, uint32_t esr __unused)
1227{
1228 exception_type_t exception = EXC_BREAKPOINT;
1229 mach_exception_data_type_t codes[2] = {EXC_ARM_BREAKPOINT};
1230 mach_msg_type_number_t numcodes = 2;
1231
1232#if __has_feature(ptrauth_calls)
1233 if (ESR_EC(esr) == ESR_EC_BRK_AARCH64 &&
1234 brk_comment_is_ptrauth(ISS_BRK_COMMENT(esr))) {
1235 exception |= EXC_PTRAUTH_BIT;
1236 }
1237#endif /* __has_feature(ptrauth_calls) */
1238
1239 codes[1] = get_saved_state_pc(iss: state);
1240 exception_triage(exception, code: codes, codeCnt: numcodes);
1241 __builtin_unreachable();
1242}
1243
1244static void
1245handle_watchpoint(vm_offset_t fault_addr)
1246{
1247 exception_type_t exception = EXC_BREAKPOINT;
1248 mach_exception_data_type_t codes[2] = {EXC_ARM_DA_DEBUG};
1249 mach_msg_type_number_t numcodes = 2;
1250
1251 codes[1] = fault_addr;
1252 exception_triage(exception, code: codes, codeCnt: numcodes);
1253 __builtin_unreachable();
1254}
1255
1256static void
1257handle_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
1258 abort_inspector_t inspect_abort, abort_handler_t handler, expected_fault_handler_t expected_fault_handler)
1259{
1260 fault_status_t fault_code;
1261 vm_prot_t fault_type;
1262
1263 inspect_abort(ESR_ISS(esr), &fault_code, &fault_type);
1264 handler(state, esr, fault_addr, fault_code, fault_type, expected_fault_handler);
1265}
1266
1267static void
1268inspect_instruction_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type)
1269{
1270 getCpuDatap()->cpu_stat.instr_ex_cnt++;
1271 *fault_code = ISS_IA_FSC(iss);
1272 *fault_type = (VM_PROT_READ | VM_PROT_EXECUTE);
1273}
1274
1275static void
1276inspect_data_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type)
1277{
1278 getCpuDatap()->cpu_stat.data_ex_cnt++;
1279 *fault_code = ISS_DA_FSC(iss);
1280
1281 /*
1282 * Cache maintenance operations always report faults as write access.
1283 * Change these to read access, unless they report a permission fault.
1284 * Only certain cache maintenance operations (e.g. 'dc ivac') require write
1285 * access to the mapping, but if a cache maintenance operation that only requires
1286 * read access generates a permission fault, then we will not be able to handle
1287 * the fault regardless of whether we treat it as a read or write fault.
1288 */
1289 if ((iss & ISS_DA_WNR) && (!(iss & ISS_DA_CM) || is_permission_fault(status: *fault_code))) {
1290 *fault_type = (VM_PROT_READ | VM_PROT_WRITE);
1291 } else {
1292 *fault_type = (VM_PROT_READ);
1293 }
1294}
1295
1296#if __has_feature(ptrauth_calls)
1297static inline uint64_t
1298fault_addr_bitmask(unsigned int bit_from, unsigned int bit_to)
1299{
1300 return ((1ULL << (bit_to - bit_from + 1)) - 1) << bit_from;
1301}
1302
1303static inline bool
1304fault_addr_bit(vm_offset_t fault_addr, unsigned int bit)
1305{
1306 return (bool)((fault_addr >> bit) & 1);
1307}
1308
1309extern int gARM_FEAT_PAuth2;
1310
1311/**
1312 * Determines whether a fault address taken at EL0 contains a PAC error code
1313 * corresponding to the specified kind of ptrauth key.
1314 */
1315static bool
1316user_fault_addr_matches_pac_error_code(vm_offset_t fault_addr, bool data_key)
1317{
1318 bool instruction_tbi = !(get_tcr() & TCR_TBID0_TBI_DATA_ONLY);
1319 bool tbi = data_key || __improbable(instruction_tbi);
1320
1321 if (gARM_FEAT_PAuth2) {
1322 /*
1323 * EnhancedPAC2 CPUs don't encode error codes at fixed positions, so
1324 * treat all non-canonical address bits like potential poison bits.
1325 */
1326 uint64_t mask = fault_addr_bitmask(64 - T0SZ_BOOT, 54);
1327 if (!tbi) {
1328 mask |= fault_addr_bitmask(56, 63);
1329 }
1330 return (fault_addr & mask) != 0;
1331 } else {
1332 unsigned int poison_shift;
1333 if (tbi) {
1334 poison_shift = 53;
1335 } else {
1336 poison_shift = 61;
1337 }
1338
1339 /* PAC error codes are always in the form key_number:NOT(key_number) */
1340 bool poison_bit_1 = fault_addr_bit(fault_addr, poison_shift);
1341 bool poison_bit_2 = fault_addr_bit(fault_addr, poison_shift + 1);
1342 return poison_bit_1 != poison_bit_2;
1343 }
1344}
1345#endif /* __has_feature(ptrauth_calls) */
1346
1347/**
1348 * Determines whether the userland thread has a JIT region in RW mode, TPRO
1349 * in RW mode, or JCTL_EL0 in pointer signing mode. A fault in any of these trusted
1350 * code paths may indicate an attack on WebKit. Rather than letting a
1351 * potentially-compromised process try to handle the exception, it will be killed
1352 * by the kernel and a crash report will be generated.
1353 */
1354static bool
1355user_fault_in_self_restrict_mode(thread_t thread __unused)
1356{
1357
1358 return false;
1359}
1360
1361static void
1362handle_pc_align(arm_saved_state_t *ss)
1363{
1364 exception_type_t exc;
1365 mach_exception_data_type_t codes[2];
1366 mach_msg_type_number_t numcodes = 2;
1367
1368 if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) {
1369 panic_with_thread_kernel_state(msg: "PC alignment exception from kernel.", ss);
1370 }
1371
1372 exc = EXC_BAD_ACCESS;
1373#if __has_feature(ptrauth_calls)
1374 if (user_fault_addr_matches_pac_error_code(get_saved_state_pc(ss), false)) {
1375 exc |= EXC_PTRAUTH_BIT;
1376 }
1377#endif /* __has_feature(ptrauth_calls) */
1378
1379 codes[0] = EXC_ARM_DA_ALIGN;
1380 codes[1] = get_saved_state_pc(iss: ss);
1381
1382 exception_triage(exception: exc, code: codes, codeCnt: numcodes);
1383 __builtin_unreachable();
1384}
1385
1386static void
1387handle_sp_align(arm_saved_state_t *ss)
1388{
1389 exception_type_t exc;
1390 mach_exception_data_type_t codes[2];
1391 mach_msg_type_number_t numcodes = 2;
1392
1393 if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) {
1394 panic_with_thread_kernel_state(msg: "SP alignment exception from kernel.", ss);
1395 }
1396
1397 exc = EXC_BAD_ACCESS;
1398#if __has_feature(ptrauth_calls)
1399 if (user_fault_addr_matches_pac_error_code(get_saved_state_sp(ss), true)) {
1400 exc |= EXC_PTRAUTH_BIT;
1401 }
1402#endif /* __has_feature(ptrauth_calls) */
1403
1404 codes[0] = EXC_ARM_SP_ALIGN;
1405 codes[1] = get_saved_state_sp(iss: ss);
1406
1407 exception_triage(exception: exc, code: codes, codeCnt: numcodes);
1408 __builtin_unreachable();
1409}
1410
1411static void
1412handle_wf_trap(arm_saved_state_t *state)
1413{
1414 exception_type_t exc;
1415 mach_exception_data_type_t codes[2];
1416 mach_msg_type_number_t numcodes = 2;
1417 uint32_t instr = 0;
1418
1419 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1420
1421 exc = EXC_BAD_INSTRUCTION;
1422 codes[0] = EXC_ARM_UNDEFINED;
1423 codes[1] = instr;
1424
1425 exception_triage(exception: exc, code: codes, codeCnt: numcodes);
1426 __builtin_unreachable();
1427}
1428
1429static void
1430handle_fp_trap(arm_saved_state_t *state, uint32_t esr)
1431{
1432 exception_type_t exc = EXC_ARITHMETIC;
1433 mach_exception_data_type_t codes[2];
1434 mach_msg_type_number_t numcodes = 2;
1435 uint32_t instr = 0;
1436
1437 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1438 panic_with_thread_kernel_state(msg: "Floating point exception from kernel", ss: state);
1439 }
1440
1441 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1442 codes[1] = instr;
1443
1444 /* The floating point trap flags are only valid if TFV is set. */
1445 if (!fp_exceptions_enabled) {
1446 exc = EXC_BAD_INSTRUCTION;
1447 codes[0] = EXC_ARM_UNDEFINED;
1448 } else if (!(esr & ISS_FP_TFV)) {
1449 codes[0] = EXC_ARM_FP_UNDEFINED;
1450 } else if (esr & ISS_FP_UFF) {
1451 codes[0] = EXC_ARM_FP_UF;
1452 } else if (esr & ISS_FP_OFF) {
1453 codes[0] = EXC_ARM_FP_OF;
1454 } else if (esr & ISS_FP_IOF) {
1455 codes[0] = EXC_ARM_FP_IO;
1456 } else if (esr & ISS_FP_DZF) {
1457 codes[0] = EXC_ARM_FP_DZ;
1458 } else if (esr & ISS_FP_IDF) {
1459 codes[0] = EXC_ARM_FP_ID;
1460 } else if (esr & ISS_FP_IXF) {
1461 codes[0] = EXC_ARM_FP_IX;
1462 } else {
1463 panic("Unrecognized floating point exception, state=%p, esr=%#x", state, esr);
1464 }
1465
1466 exception_triage(exception: exc, code: codes, codeCnt: numcodes);
1467 __builtin_unreachable();
1468}
1469
1470
1471
1472/*
1473 * handle_alignment_fault_from_user:
1474 * state: Saved state
1475 *
1476 * Attempts to deal with an alignment fault from userspace (possibly by
1477 * emulating the faulting instruction). If emulation failed due to an
1478 * unservicable fault, the ESR for that fault will be stored in the
1479 * recovery_esr field of the thread by the exception code.
1480 *
1481 * Returns:
1482 * -1: Emulation failed (emulation of state/instr not supported)
1483 * 0: Successfully emulated the instruction
1484 * EFAULT: Emulation failed (probably due to permissions)
1485 * EINVAL: Emulation failed (probably due to a bad address)
1486 */
1487
1488
1489static int
1490handle_alignment_fault_from_user(arm_saved_state_t *state, kern_return_t *vmfr)
1491{
1492 int ret = -1;
1493
1494#pragma unused (state)
1495#pragma unused (vmfr)
1496
1497 return ret;
1498}
1499
1500
1501
1502
1503static void
1504handle_sw_step_debug(arm_saved_state_t *state)
1505{
1506 thread_t thread = current_thread();
1507 exception_type_t exc;
1508 mach_exception_data_type_t codes[2];
1509 mach_msg_type_number_t numcodes = 2;
1510
1511 if (!PSR64_IS_USER(get_saved_state_cpsr(state))) {
1512 panic_with_thread_kernel_state(msg: "SW_STEP_DEBUG exception from kernel.", ss: state);
1513 }
1514
1515 // Disable single step and unmask interrupts (in the saved state, anticipating next exception return)
1516 if (thread->machine.DebugData != NULL) {
1517 thread->machine.DebugData->uds.ds64.mdscr_el1 &= ~0x1;
1518 } else {
1519 panic_with_thread_kernel_state(msg: "SW_STEP_DEBUG exception thread DebugData is NULL.", ss: state);
1520 }
1521
1522 mask_user_saved_state_cpsr(iss: thread->machine.upcb, set_bits: 0, PSR64_SS | DAIF_ALL);
1523
1524 // Special encoding for gdb single step event on ARM
1525 exc = EXC_BREAKPOINT;
1526 codes[0] = 1;
1527 codes[1] = 0;
1528
1529 exception_triage(exception: exc, code: codes, codeCnt: numcodes);
1530 __builtin_unreachable();
1531}
1532
1533static void
1534handle_user_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
1535 fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler)
1536{
1537 exception_type_t exc = EXC_BAD_ACCESS;
1538 mach_exception_data_type_t codes[2];
1539 mach_msg_type_number_t numcodes = 2;
1540 thread_t thread = current_thread();
1541
1542 (void)esr;
1543 (void)expected_fault_handler;
1544
1545 if (ml_at_interrupt_context()) {
1546 panic_with_thread_kernel_state(msg: "Apparently on interrupt stack when taking user abort!\n", ss: state);
1547 }
1548
1549 thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling abort from userspace */
1550
1551 if (!is_vm_fault(status: fault_code) &&
1552 thread->t_rr_state.trr_fault_state != TRR_FAULT_NONE) {
1553 thread_reset_pcs_done_faulting(thread);
1554 }
1555
1556 if (is_vm_fault(status: fault_code)) {
1557 vm_map_t map = thread->map;
1558 vm_offset_t vm_fault_addr = fault_addr;
1559 kern_return_t result = KERN_FAILURE;
1560
1561 assert(map != kernel_map);
1562
1563 if (!(fault_type & VM_PROT_EXECUTE)) {
1564 vm_fault_addr = VM_USER_STRIP_TBI(fault_addr);
1565 }
1566
1567 /* check to see if it is just a pmap ref/modify fault */
1568 if (!is_translation_fault(status: fault_code)) {
1569 result = arm_fast_fault(map->pmap,
1570 vm_fault_addr,
1571 fault_type, (fault_code == FSC_ACCESS_FLAG_FAULT_L3), TRUE);
1572 }
1573 if (result != KERN_SUCCESS) {
1574
1575 {
1576 /* We have to fault the page in */
1577 result = vm_fault(map, vaddr: vm_fault_addr, fault_type,
1578 /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, THREAD_ABORTSAFE,
1579 /* caller_pmap */ NULL, /* caller_pmap_addr */ pmap_addr: 0);
1580 }
1581 }
1582 if (thread->t_rr_state.trr_fault_state != TRR_FAULT_NONE) {
1583 thread_reset_pcs_done_faulting(thread);
1584 }
1585 if (result == KERN_SUCCESS || result == KERN_ABORTED) {
1586 return;
1587 }
1588
1589 /*
1590 * vm_fault() should never return KERN_FAILURE for page faults from user space.
1591 * If it does, we're leaking preemption disables somewhere in the kernel.
1592 */
1593 if (__improbable(result == KERN_FAILURE)) {
1594 panic("vm_fault() KERN_FAILURE from user fault on thread %p", thread);
1595 }
1596
1597 codes[0] = result;
1598 } else if (is_alignment_fault(status: fault_code)) {
1599 kern_return_t vmfkr = KERN_SUCCESS;
1600 thread->machine.recover_esr = 0;
1601 thread->machine.recover_far = 0;
1602 int result = handle_alignment_fault_from_user(state, vmfr: &vmfkr);
1603 if (result == 0) {
1604 /* Successfully emulated, or instruction
1605 * copyin() for decode/emulation failed.
1606 * Continue, or redrive instruction.
1607 */
1608 thread_exception_return();
1609 } else if (((result == EFAULT) || (result == EINVAL)) &&
1610 (thread->machine.recover_esr == 0)) {
1611 /*
1612 * If we didn't actually take a fault, but got one of
1613 * these errors, then we failed basic sanity checks of
1614 * the fault address. Treat this as an invalid
1615 * address.
1616 */
1617 codes[0] = KERN_INVALID_ADDRESS;
1618 } else if ((result == EFAULT) &&
1619 (thread->machine.recover_esr)) {
1620 /*
1621 * Since alignment aborts are prioritized
1622 * ahead of translation aborts, the misaligned
1623 * atomic emulation flow may have triggered a
1624 * VM pagefault, which the VM could not resolve.
1625 * Report the VM fault error in codes[]
1626 */
1627
1628 codes[0] = vmfkr;
1629 assertf(vmfkr != KERN_SUCCESS, "Unexpected vmfkr 0x%x", vmfkr);
1630 /* Cause ESR_EC to reflect an EL0 abort */
1631 thread->machine.recover_esr &= ~ESR_EC_MASK;
1632 thread->machine.recover_esr |= (ESR_EC_DABORT_EL0 << ESR_EC_SHIFT);
1633 set_saved_state_esr(iss: thread->machine.upcb, esr: thread->machine.recover_esr);
1634 set_saved_state_far(iss: thread->machine.upcb, far: thread->machine.recover_far);
1635 fault_addr = thread->machine.recover_far;
1636 } else {
1637 /* This was just an unsupported alignment
1638 * exception. Misaligned atomic emulation
1639 * timeouts fall in this category.
1640 */
1641 codes[0] = EXC_ARM_DA_ALIGN;
1642 }
1643 } else if (is_parity_error(status: fault_code)) {
1644#if defined(APPLE_ARM64_ARCH_FAMILY)
1645 /*
1646 * Platform errors are handled in sleh_sync before interrupts are enabled.
1647 */
1648#else
1649 panic("User parity error.");
1650#endif
1651 } else {
1652 codes[0] = KERN_FAILURE;
1653 }
1654
1655#if CODE_SIGNING_MONITOR
1656 /*
1657 * If the code reaches here, it means we weren't able to resolve the fault and we're
1658 * going to be sending the task an exception. On systems which have the code signing
1659 * monitor enabled, an execute fault which cannot be handled must result in sending
1660 * a SIGKILL to the task.
1661 */
1662 if (is_vm_fault(fault_code) && (fault_type & VM_PROT_EXECUTE)) {
1663 csm_code_signing_violation(current_proc(), fault_addr);
1664 }
1665#endif
1666
1667 codes[1] = fault_addr;
1668#if __has_feature(ptrauth_calls)
1669 bool is_data_abort = (ESR_EC(esr) == ESR_EC_DABORT_EL0);
1670 if (user_fault_addr_matches_pac_error_code(fault_addr, is_data_abort)) {
1671 exc |= EXC_PTRAUTH_BIT;
1672 }
1673#endif /* __has_feature(ptrauth_calls) */
1674
1675 if (user_fault_in_self_restrict_mode(thread) &&
1676 task_is_jit_exception_fatal(task: get_threadtask(thread))) {
1677 extern int exit_with_jit_exception(proc_t p);
1678 exit_with_jit_exception(p: current_proc());
1679 }
1680
1681 exception_triage(exception: exc, code: codes, codeCnt: numcodes);
1682 __builtin_unreachable();
1683}
1684
1685/**
1686 * Panic because the kernel abort handler tried to apply a recovery handler that
1687 * isn't inside copyio_recover_table[].
1688 *
1689 * @param state original saved-state
1690 * @param recover invalid recovery handler
1691 */
1692__attribute__((noreturn, used))
1693static void
1694panic_on_invalid_recovery_handler(arm_saved_state_t *state, struct copyio_recovery_entry *recover)
1695{
1696 panic("attempt to set invalid recovery handler %p on kernel saved-state %p", recover, state);
1697}
1698
1699static void
1700handle_kernel_abort_recover(
1701 arm_saved_state_t *state,
1702 uint32_t esr,
1703 vm_offset_t fault_addr,
1704 thread_t thread,
1705 struct copyio_recovery_entry *_Nonnull recover)
1706{
1707 thread->machine.recover_esr = esr;
1708 thread->machine.recover_far = fault_addr;
1709#if defined(HAS_APPLE_PAC)
1710 MANIPULATE_SIGNED_THREAD_STATE(state,
1711 "adrp x6, _copyio_recover_table_end@page \n"
1712 "add x6, x6, _copyio_recover_table_end@pageoff \n"
1713 "cmp %[recover], x6 \n"
1714 "b.lt 1f \n"
1715 "bl _panic_on_invalid_recovery_handler \n"
1716 "brk #0 \n"
1717 "1: \n"
1718 "adrp x6, _copyio_recover_table@page \n"
1719 "add x6, x6, _copyio_recover_table@pageoff \n"
1720 "cmp %[recover], x6 \n"
1721 "b.ge 1f \n"
1722 "bl _panic_on_invalid_recovery_handler \n"
1723 "brk #0 \n"
1724 "1: \n"
1725 "ldr x1, [%[recover], %[CRE_RECOVERY]] \n"
1726 "add x1, x1, x6 \n"
1727 "str x1, [x0, %[SS64_PC]] \n",
1728 [recover] "r"(recover),
1729 [CRE_RECOVERY] "i"(offsetof(struct copyio_recovery_entry, cre_recovery))
1730 );
1731#else
1732 if ((uintptr_t)recover < (uintptr_t)copyio_recover_table ||
1733 (uintptr_t)recover >= (uintptr_t)copyio_recover_table_end) {
1734 panic_on_invalid_recovery_handler(state, recover);
1735 }
1736 saved_state64(state)->pc = copyio_recovery_addr(recover->cre_recovery);
1737#endif
1738}
1739
1740static void
1741handle_kernel_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
1742 fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler)
1743{
1744 thread_t thread = current_thread();
1745 struct copyio_recovery_entry *recover = find_copyio_recovery_entry(state);
1746
1747#ifndef CONFIG_XNUPOST
1748 (void)expected_fault_handler;
1749#endif /* CONFIG_XNUPOST */
1750
1751#if CONFIG_DTRACE
1752 if (is_vm_fault(status: fault_code) && thread->t_dtrace_inprobe) { /* Executing under dtrace_probe? */
1753 if (dtrace_tally_fault(fault_addr)) { /* Should a fault under dtrace be ignored? */
1754 /*
1755 * Point to next instruction, or recovery handler if set.
1756 */
1757 if (recover) {
1758 handle_kernel_abort_recover(state, esr, VM_USER_STRIP_PTR(fault_addr), thread, recover);
1759 } else {
1760 add_saved_state_pc(iss: state, diff: 4);
1761 }
1762 return;
1763 } else {
1764 panic_with_thread_kernel_state(msg: "Unexpected page fault under dtrace_probe", ss: state);
1765 }
1766 }
1767#endif
1768
1769 if (ml_at_interrupt_context()) {
1770 panic_with_thread_kernel_state(msg: "Unexpected abort while on interrupt stack.", ss: state);
1771 }
1772
1773 if (is_vm_fault(status: fault_code)) {
1774 kern_return_t result = KERN_FAILURE;
1775 vm_map_t map;
1776 int interruptible;
1777
1778 /*
1779 * Ensure no faults in the physical aperture. This could happen if
1780 * a page table is incorrectly allocated from the read only region
1781 * when running with KTRR.
1782 */
1783
1784#ifdef CONFIG_XNUPOST
1785 if (expected_fault_handler && expected_fault_handler(state)) {
1786 return;
1787 }
1788#endif /* CONFIG_XNUPOST */
1789
1790 if (fault_addr >= gVirtBase && fault_addr < static_memory_end) {
1791 panic_with_thread_kernel_state(msg: "Unexpected fault in kernel static region\n", ss: state);
1792 }
1793
1794 if (VM_KERNEL_ADDRESS(fault_addr) || thread == THREAD_NULL || recover == 0) {
1795 /*
1796 * If no recovery handler is supplied, always drive the fault against
1797 * the kernel map. If the fault was taken against a userspace VA, indicating
1798 * an unprotected access to user address space, vm_fault() should fail and
1799 * ultimately lead to a panic here.
1800 */
1801 map = kernel_map;
1802 interruptible = THREAD_UNINT;
1803
1804#if CONFIG_KERNEL_TAGGING
1805 /*
1806 * If kernel tagging is enabled, canonicalize the address here, so that we have a
1807 * chance to find it in the VM ranges. Do not mess with exec fault cases.
1808 */
1809 if (!((fault_type) & VM_PROT_EXECUTE)) {
1810 fault_addr = vm_memtag_canonicalize_address(fault_addr);
1811 }
1812#endif /* CONFIG_KERNEL_TAGGING */
1813 } else {
1814 map = thread->map;
1815
1816 /**
1817 * In the case that the recovery handler is set (e.g., during copyio
1818 * and dtrace probes), we don't want the vm_fault() operation to be
1819 * aborted early. Those code paths can't handle restarting the
1820 * vm_fault() operation so don't allow it to return early without
1821 * creating the wanted mapping.
1822 */
1823 interruptible = (recover) ? THREAD_UNINT : THREAD_ABORTSAFE;
1824
1825 }
1826
1827 if (fault_addr >= gVirtBase && fault_addr < static_memory_end) {
1828 panic_with_thread_kernel_state(msg: "Unexpected fault in kernel static region\n", ss: state);
1829 }
1830
1831 /* check to see if it is just a pmap ref/modify fault */
1832 if (!is_translation_fault(status: fault_code)) {
1833 result = arm_fast_fault(map->pmap,
1834 fault_addr,
1835 fault_type, (fault_code == FSC_ACCESS_FLAG_FAULT_L3), FALSE);
1836 if (result == KERN_SUCCESS) {
1837 return;
1838 }
1839 }
1840
1841 if (result != KERN_PROTECTION_FAILURE) {
1842 /*
1843 * We have to "fault" the page in.
1844 */
1845 result = vm_fault(map, vaddr: fault_addr, fault_type,
1846 /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, interruptible,
1847 /* caller_pmap */ NULL, /* caller_pmap_addr */ pmap_addr: 0);
1848 }
1849
1850 if (result == KERN_SUCCESS) {
1851 return;
1852 }
1853
1854 /*
1855 * If we have a recover handler, invoke it now.
1856 */
1857 if (recover) {
1858 handle_kernel_abort_recover(state, esr, fault_addr, thread, recover);
1859 return;
1860 }
1861
1862 panic_fault_address = fault_addr;
1863 } else if (is_alignment_fault(status: fault_code)) {
1864 if (recover) {
1865 handle_kernel_abort_recover(state, esr, fault_addr, thread, recover);
1866 return;
1867 }
1868 panic_with_thread_kernel_state(msg: "Unaligned kernel data abort.", ss: state);
1869 } else if (is_parity_error(status: fault_code)) {
1870#if defined(APPLE_ARM64_ARCH_FAMILY)
1871 /*
1872 * Platform errors are handled in sleh_sync before interrupts are enabled.
1873 */
1874#else
1875 panic_with_thread_kernel_state("Kernel parity error.", state);
1876#endif
1877 } else {
1878 kprintf(fmt: "Unclassified kernel abort (fault_code=0x%x)\n", fault_code);
1879 }
1880
1881 panic_with_thread_kernel_state(msg: "Kernel data abort.", ss: state);
1882}
1883
1884extern void syscall_trace(struct arm_saved_state * regs);
1885
1886static void
1887handle_svc(arm_saved_state_t *state)
1888{
1889 int trap_no = get_saved_state_svc_number(iss: state);
1890 thread_t thread = current_thread();
1891 struct proc *p;
1892
1893#define handle_svc_kprintf(x...) /* kprintf("handle_svc: " x) */
1894
1895#define TRACE_SYSCALL 1
1896#if TRACE_SYSCALL
1897 syscall_trace(regs: state);
1898#endif
1899
1900 thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling SVC from userspace */
1901
1902 if (trap_no == (int)PLATFORM_SYSCALL_TRAP_NO) {
1903 platform_syscall(state);
1904 panic("Returned from platform_syscall()?");
1905 }
1906
1907 current_cached_proc_cred_update();
1908
1909 if (trap_no < 0) {
1910 switch (trap_no) {
1911 case MACH_ARM_TRAP_ABSTIME:
1912 handle_mach_absolute_time_trap(state);
1913 return;
1914 case MACH_ARM_TRAP_CONTTIME:
1915 handle_mach_continuous_time_trap(state);
1916 return;
1917 }
1918
1919 /* Counting perhaps better in the handler, but this is how it's been done */
1920 thread->syscalls_mach++;
1921 mach_syscall(state);
1922 } else {
1923 /* Counting perhaps better in the handler, but this is how it's been done */
1924 thread->syscalls_unix++;
1925 p = get_bsdthreadtask_info(thread);
1926
1927 assert(p);
1928
1929 unix_syscall(regs: state, thread_act: thread, proc: p);
1930 }
1931}
1932
1933static void
1934handle_mach_absolute_time_trap(arm_saved_state_t *state)
1935{
1936 uint64_t now = mach_absolute_time();
1937 saved_state64(iss: state)->x[0] = now;
1938}
1939
1940static void
1941handle_mach_continuous_time_trap(arm_saved_state_t *state)
1942{
1943 uint64_t now = mach_continuous_time();
1944 saved_state64(iss: state)->x[0] = now;
1945}
1946
1947
1948__attribute__((noreturn))
1949static void
1950handle_msr_trap(arm_saved_state_t *state, uint32_t esr)
1951{
1952 exception_type_t exception = EXC_BAD_INSTRUCTION;
1953 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1954 mach_msg_type_number_t numcodes = 2;
1955 uint32_t instr = 0;
1956
1957 if (!is_saved_state64(iss: state)) {
1958 panic("MSR/MRS trap (ESR 0x%x) from 32-bit state", esr);
1959 }
1960
1961 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1962 panic("MSR/MRS trap (ESR 0x%x) from kernel", esr);
1963 }
1964
1965 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1966 codes[1] = instr;
1967
1968 exception_triage(exception, code: codes, codeCnt: numcodes);
1969 __builtin_unreachable();
1970}
1971
1972#if __has_feature(ptrauth_calls)
1973static void
1974stringify_gpr(unsigned int r, char reg[4])
1975{
1976 switch (r) {
1977 case 29:
1978 strncpy(reg, "fp", 4);
1979 return;
1980
1981 case 30:
1982 strncpy(reg, "lr", 4);
1983 return;
1984
1985 case 31:
1986 strncpy(reg, "xzr", 4);
1987 return;
1988
1989 default:
1990 snprintf(reg, 4, "x%u", r);
1991 return;
1992 }
1993}
1994
1995static void
1996autxx_instruction_extract_reg(uint32_t instr, char reg[4])
1997{
1998 unsigned int rd = ARM64_INSTR_AUTxx_RD_GET(instr);
1999 stringify_gpr(rd, reg);
2000}
2001
2002static const char *
2003autix_system_instruction_extract_reg(uint32_t instr)
2004{
2005 unsigned int crm_op2 = ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_GET(instr);
2006 if (crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIA1716 ||
2007 crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIB1716) {
2008 return "x17";
2009 } else {
2010 return "lr";
2011 }
2012}
2013
2014
2015static void
2016handle_pac_fail(arm_saved_state_t *state, uint32_t esr)
2017{
2018 exception_type_t exception = EXC_BAD_ACCESS | EXC_PTRAUTH_BIT;
2019 mach_exception_data_type_t codes[2] = {EXC_ARM_PAC_FAIL};
2020 mach_msg_type_number_t numcodes = 2;
2021 uint32_t instr = 0;
2022
2023 if (!is_saved_state64(state)) {
2024 panic("PAC failure (ESR 0x%x) from 32-bit state", esr);
2025 }
2026
2027 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
2028
2029 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
2030#define GENERIC_PAC_FAILURE_MSG_FMT "PAC failure from kernel with %s key"
2031#define AUTXX_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT " while authing %s"
2032#define GENERIC_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT
2033#define MAX_PAC_MSG_FMT AUTXX_MSG_FMT
2034
2035 char msg[strlen(MAX_PAC_MSG_FMT)
2036 - strlen("%s") + strlen("IA")
2037 - strlen("%s") + strlen("xzr")
2038 + 1];
2039 ptrauth_key key = (ptrauth_key)(esr & 0x3);
2040 const char *key_str = ptrauth_key_to_string(key);
2041
2042 if (ARM64_INSTR_IS_AUTxx(instr)) {
2043 char reg[4];
2044 autxx_instruction_extract_reg(instr, reg);
2045 snprintf(msg, sizeof(msg), AUTXX_MSG_FMT, key_str, reg);
2046 } else if (ARM64_INSTR_IS_AUTIx_SYSTEM(instr)) {
2047 const char *reg = autix_system_instruction_extract_reg(instr);
2048 snprintf(msg, sizeof(msg), AUTXX_MSG_FMT, key_str, reg);
2049 } else {
2050 snprintf(msg, sizeof(msg), GENERIC_MSG_FMT, key_str);
2051 }
2052 panic_with_thread_kernel_state(msg, state);
2053 }
2054
2055 codes[1] = instr;
2056
2057 exception_triage(exception, codes, numcodes);
2058 __builtin_unreachable();
2059}
2060#endif /* __has_feature(ptrauth_calls) */
2061
2062static void
2063handle_user_trapped_instruction32(arm_saved_state_t *state, uint32_t esr)
2064{
2065 exception_type_t exception = EXC_BAD_INSTRUCTION;
2066 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
2067 mach_msg_type_number_t numcodes = 2;
2068 uint32_t instr;
2069
2070 if (is_saved_state64(iss: state)) {
2071 panic("ESR (0x%x) for instruction trapped from U32, but saved state is 64-bit.", esr);
2072 }
2073
2074 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
2075 panic("ESR (0x%x) for instruction trapped from U32, actually came from kernel?", esr);
2076 }
2077
2078 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
2079 codes[1] = instr;
2080
2081 exception_triage(exception, code: codes, codeCnt: numcodes);
2082 __builtin_unreachable();
2083}
2084
2085static void
2086handle_simd_trap(arm_saved_state_t *state, uint32_t esr)
2087{
2088 exception_type_t exception = EXC_BAD_INSTRUCTION;
2089 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
2090 mach_msg_type_number_t numcodes = 2;
2091 uint32_t instr = 0;
2092
2093 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
2094 panic("ESR (0x%x) for SIMD trap from userland, actually came from kernel?", esr);
2095 }
2096
2097 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
2098 codes[1] = instr;
2099
2100 exception_triage(exception, code: codes, codeCnt: numcodes);
2101 __builtin_unreachable();
2102}
2103
2104void
2105sleh_irq(arm_saved_state_t *state)
2106{
2107 cpu_data_t * cdp __unused = getCpuDatap();
2108#if MACH_ASSERT
2109 int preemption_level = sleh_get_preemption_level();
2110#endif
2111
2112
2113 sleh_interrupt_handler_prologue(state, DBG_INTR_TYPE_OTHER);
2114
2115#if USE_APPLEARMSMP
2116 PE_handle_ext_interrupt();
2117#else
2118 /* Run the registered interrupt handler. */
2119 cdp->interrupt_handler(cdp->interrupt_target,
2120 cdp->interrupt_refCon,
2121 cdp->interrupt_nub,
2122 cdp->interrupt_source);
2123#endif
2124
2125 entropy_collect();
2126
2127
2128 sleh_interrupt_handler_epilogue();
2129#if MACH_ASSERT
2130 if (preemption_level != sleh_get_preemption_level()) {
2131 panic("irq handler %p changed preemption level from %d to %d", cdp->interrupt_handler, preemption_level, sleh_get_preemption_level());
2132 }
2133#endif
2134}
2135
2136void
2137sleh_fiq(arm_saved_state_t *state)
2138{
2139 unsigned int type = DBG_INTR_TYPE_UNKNOWN;
2140#if MACH_ASSERT
2141 int preemption_level = sleh_get_preemption_level();
2142#endif
2143
2144#if MONOTONIC_FIQ
2145 uint64_t pmcr0 = 0, upmsr = 0;
2146#endif /* MONOTONIC_FIQ */
2147
2148#if defined(HAS_IPI)
2149 boolean_t is_ipi = FALSE;
2150 uint64_t ipi_sr = 0;
2151
2152 if (gFastIPI) {
2153 MRS(ipi_sr, "S3_5_C15_C1_1");
2154
2155 if (ipi_sr & ARM64_IPISR_IPI_PENDING) {
2156 is_ipi = TRUE;
2157 }
2158 }
2159
2160 if (is_ipi) {
2161 type = DBG_INTR_TYPE_IPI;
2162 } else
2163#endif /* defined(HAS_IPI) */
2164 if (ml_get_timer_pending()) {
2165 type = DBG_INTR_TYPE_TIMER;
2166 }
2167#if MONOTONIC_FIQ
2168 /* Consult the PMI sysregs last, after IPI/timer
2169 * classification.
2170 */
2171 else if (mt_pmi_pending(&pmcr0, &upmsr)) {
2172 type = DBG_INTR_TYPE_PMI;
2173 }
2174#endif /* MONOTONIC_FIQ */
2175
2176 sleh_interrupt_handler_prologue(state, type);
2177
2178#if APPLEVIRTUALPLATFORM
2179 uint64_t iar = __builtin_arm_rsr64("ICC_IAR0_EL1");
2180#endif
2181
2182#if defined(HAS_IPI)
2183 if (type == DBG_INTR_TYPE_IPI) {
2184 /*
2185 * Order is important here: we must ack the IPI by writing IPI_SR
2186 * before we call cpu_signal_handler(). Otherwise, there will be
2187 * a window between the completion of pending-signal processing in
2188 * cpu_signal_handler() and the ack during which a newly-issued
2189 * IPI to this CPU may be lost. ISB is required to ensure the msr
2190 * is retired before execution of cpu_signal_handler().
2191 */
2192 MSR("S3_5_C15_C1_1", ARM64_IPISR_IPI_PENDING);
2193 __builtin_arm_isb(ISB_SY);
2194 cpu_signal_handler();
2195 } else
2196#endif /* defined(HAS_IPI) */
2197#if MONOTONIC_FIQ
2198 if (type == DBG_INTR_TYPE_PMI) {
2199 INTERRUPT_MASKED_DEBUG_START(mt_fiq, DBG_INTR_TYPE_PMI);
2200 mt_fiq(getCpuDatap(), pmcr0, upmsr);
2201 INTERRUPT_MASKED_DEBUG_END();
2202 } else
2203#endif /* MONOTONIC_FIQ */
2204 {
2205 /*
2206 * We don't know that this is a timer, but we don't have insight into
2207 * the other interrupts that go down this path.
2208 */
2209
2210 cpu_data_t *cdp = getCpuDatap();
2211
2212 cdp->cpu_decrementer = -1; /* Large */
2213
2214 /*
2215 * ARM64_TODO: whether we're coming from userland is ignored right now.
2216 * We can easily thread it through, but not bothering for the
2217 * moment (AArch32 doesn't either).
2218 */
2219 INTERRUPT_MASKED_DEBUG_START(rtclock_intr, DBG_INTR_TYPE_TIMER);
2220 rtclock_intr(TRUE);
2221 INTERRUPT_MASKED_DEBUG_END();
2222 }
2223
2224#if APPLEVIRTUALPLATFORM
2225 if (iar != GIC_SPURIOUS_IRQ) {
2226 __builtin_arm_wsr64("ICC_EOIR0_EL1", iar);
2227 __builtin_arm_isb(ISB_SY);
2228 }
2229#endif
2230
2231 sleh_interrupt_handler_epilogue();
2232#if MACH_ASSERT
2233 if (preemption_level != sleh_get_preemption_level()) {
2234 panic("fiq type %u changed preemption level from %d to %d", type, preemption_level, sleh_get_preemption_level());
2235 }
2236#endif
2237}
2238
2239void
2240sleh_serror(arm_context_t *context, uint32_t esr, vm_offset_t far)
2241{
2242 task_vtimer_check(thread: current_thread());
2243
2244 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_START,
2245 esr, VM_KERNEL_ADDRHIDE(far));
2246 arm_saved_state_t *state = &context->ss;
2247#if MACH_ASSERT
2248 int preemption_level = sleh_get_preemption_level();
2249#endif
2250
2251 if (PSR64_IS_USER(get_saved_state_cpsr(state))) {
2252 /* Sanitize FAR (only if we came from userspace) */
2253 saved_state64(iss: state)->far = 0;
2254 }
2255
2256 ASSERT_CONTEXT_SANITY(context);
2257 arm64_platform_error(state, esr, far, source: PLAT_ERR_SRC_ASYNC);
2258#if MACH_ASSERT
2259 if (preemption_level != sleh_get_preemption_level()) {
2260 panic("serror changed preemption level from %d to %d", preemption_level, sleh_get_preemption_level());
2261 }
2262#endif
2263 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_END,
2264 esr, VM_KERNEL_ADDRHIDE(far));
2265}
2266
2267void
2268mach_syscall_trace_exit(unsigned int retval,
2269 unsigned int call_number)
2270{
2271 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2272 MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) |
2273 DBG_FUNC_END, retval, 0, 0, 0, 0);
2274}
2275
2276__attribute__((noreturn))
2277void
2278thread_syscall_return(kern_return_t error)
2279{
2280 thread_t thread;
2281 struct arm_saved_state *state;
2282
2283 thread = current_thread();
2284 state = get_user_regs(thread);
2285
2286 assert(is_saved_state64(state));
2287 saved_state64(iss: state)->x[0] = error;
2288
2289#if MACH_ASSERT
2290 kern_allocation_name_t
2291 prior __assert_only = thread_get_kernel_state(thread)->allocation_name;
2292 assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior));
2293#endif /* MACH_ASSERT */
2294
2295 if (kdebug_enable) {
2296 /* Invert syscall number (negative for a mach syscall) */
2297 mach_syscall_trace_exit(retval: error, call_number: (-1) * get_saved_state_svc_number(iss: state));
2298 }
2299
2300 thread_exception_return();
2301}
2302
2303void
2304syscall_trace(
2305 struct arm_saved_state * regs __unused)
2306{
2307 /* kprintf("syscall: %d\n", saved_state64(regs)->x[16]); */
2308}
2309
2310static void
2311sleh_interrupt_handler_prologue(arm_saved_state_t *state, unsigned int type)
2312{
2313 const bool is_user = PSR64_IS_USER(get_saved_state_cpsr(state));
2314
2315 if (is_user == true) {
2316 /* Sanitize FAR (only if the interrupt occurred while the CPU was in usermode) */
2317 saved_state64(iss: state)->far = 0;
2318 }
2319
2320 recount_enter_interrupt();
2321
2322 task_vtimer_check(thread: current_thread());
2323
2324 uint64_t pc = is_user ? get_saved_state_pc(iss: state) :
2325 VM_KERNEL_UNSLIDE(get_saved_state_pc(state));
2326
2327 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START,
2328 0, pc, is_user, type);
2329
2330#if CONFIG_TELEMETRY
2331 if (telemetry_needs_record) {
2332 telemetry_mark_curthread(interrupted_userspace: is_user, FALSE);
2333 }
2334#endif /* CONFIG_TELEMETRY */
2335}
2336
2337static void
2338sleh_interrupt_handler_epilogue(void)
2339{
2340#if KPERF
2341 kperf_interrupt();
2342#endif /* KPERF */
2343 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END);
2344 recount_leave_interrupt();
2345}
2346
2347void
2348sleh_invalid_stack(arm_context_t *context, uint32_t esr __unused, vm_offset_t far __unused)
2349{
2350 thread_t thread = current_thread();
2351 vm_offset_t kernel_stack_bottom, sp;
2352
2353 sp = get_saved_state_sp(iss: &context->ss);
2354 vm_offset_t kstackptr = (vm_offset_t)thread->machine.kstackptr;
2355 kernel_stack_bottom = round_page(x: kstackptr) - KERNEL_STACK_SIZE;
2356
2357 if ((sp < kernel_stack_bottom) && (sp >= (kernel_stack_bottom - PAGE_SIZE))) {
2358 panic_with_thread_kernel_state(msg: "Invalid kernel stack pointer (probable overflow).", ss: &context->ss);
2359 }
2360
2361 panic_with_thread_kernel_state(msg: "Invalid kernel stack pointer (probable corruption).", ss: &context->ss);
2362}
2363
2364
2365#if DEVELOPMENT || DEBUG
2366static int trap_handled;
2367static void
2368handle_recoverable_kernel_trap(
2369 __unused void *tstate,
2370 uint16_t comment)
2371{
2372 assert(comment == TEST_RECOVERABLE_SOFT_TRAP);
2373
2374 printf("Recoverable trap handled.\n");
2375 trap_handled = 1;
2376}
2377
2378KERNEL_BRK_DESCRIPTOR_DEFINE(test_desc,
2379 .type = KERNEL_BRK_TYPE_TEST,
2380 .base = TEST_RECOVERABLE_SOFT_TRAP,
2381 .max = TEST_RECOVERABLE_SOFT_TRAP,
2382 .options = KERNEL_BRK_RECOVERABLE,
2383 .handle_breakpoint = handle_recoverable_kernel_trap);
2384
2385static int
2386recoverable_kernel_trap_test(__unused int64_t in, int64_t *out)
2387{
2388 ml_recoverable_trap(TEST_RECOVERABLE_SOFT_TRAP);
2389
2390 *out = trap_handled;
2391 return 0;
2392}
2393
2394SYSCTL_TEST_REGISTER(recoverable_kernel_trap, recoverable_kernel_trap_test);
2395
2396#endif
2397