1/*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57#include <mach_assert.h>
58#include <mach_kdp.h>
59#include <kdp/kdp.h>
60#include <kdp/kdp_core.h>
61#include <kdp/kdp_internal.h>
62#include <kdp/kdp_callout.h>
63#include <kern/cpu_number.h>
64#include <kern/kalloc.h>
65#include <kern/percpu.h>
66#include <kern/spl.h>
67#include <kern/thread.h>
68#include <kern/assert.h>
69#include <kern/sched_prim.h>
70#include <kern/socd_client.h>
71#include <kern/misc_protos.h>
72#include <kern/clock.h>
73#include <kern/telemetry.h>
74#include <kern/ecc.h>
75#include <kern/kern_cdata.h>
76#include <kern/zalloc_internal.h>
77#include <kern/iotrace.h>
78#include <pexpert/device_tree.h>
79#include <vm/vm_kern.h>
80#include <vm/vm_map.h>
81#include <vm/pmap.h>
82#include <vm/vm_compressor.h>
83#include <stdarg.h>
84#include <stdatomic.h>
85#include <sys/pgo.h>
86#include <console/serial_protos.h>
87#include <IOKit/IOBSD.h>
88
89#if !(MACH_KDP && CONFIG_KDP_INTERACTIVE_DEBUGGING)
90#include <kdp/kdp_udp.h>
91#endif
92#include <kern/processor.h>
93
94#if defined(__i386__) || defined(__x86_64__)
95#include <IOKit/IOBSD.h>
96
97#include <i386/cpu_threads.h>
98#include <i386/pmCPU.h>
99#include <i386/lbr.h>
100#endif
101
102#include <IOKit/IOPlatformExpert.h>
103#include <machine/machine_cpu.h>
104#include <machine/pal_routines.h>
105
106#include <sys/kdebug.h>
107#include <libkern/OSKextLibPrivate.h>
108#include <libkern/OSAtomic.h>
109#include <libkern/kernel_mach_header.h>
110#include <libkern/section_keywords.h>
111#include <uuid/uuid.h>
112#include <mach_debug/zone_info.h>
113#include <mach/resource_monitors.h>
114#include <machine/machine_routines.h>
115#include <sys/proc_require.h>
116
117#include <os/log_private.h>
118
119#include <kern/ext_paniclog.h>
120
121#if defined(__arm64__)
122#include <pexpert/pexpert.h> /* For gPanicBase */
123#include <arm/caches_internal.h>
124#include <arm/misc_protos.h>
125extern volatile struct xnu_hw_shmem_dbg_command_info *hwsd_info;
126#endif
127
128#include <san/kcov.h>
129
130#if CONFIG_XNUPOST
131#include <tests/xnupost.h>
132extern int vsnprintf(char *, size_t, const char *, va_list);
133#endif
134
135#if CONFIG_CSR
136#include <sys/csr.h>
137#endif
138
139#if CONFIG_EXCLAVES
140#include <xnuproxy/panic.h>
141#include "exclaves_panic.h"
142#endif
143
144#if CONFIG_SPTM
145#include <arm64/sptm/sptm.h>
146#include <arm64/sptm/pmap/pmap_data.h>
147#endif /* CONFIG_SPTM */
148
149extern int IODTGetLoaderInfo( const char *key, void **infoAddr, int *infosize );
150extern void IODTFreeLoaderInfo( const char *key, void *infoAddr, int infoSize );
151
152unsigned int halt_in_debugger = 0;
153unsigned int current_debugger = 0;
154unsigned int active_debugger = 0;
155SECURITY_READ_ONLY_LATE(unsigned int) panicDebugging = FALSE;
156unsigned int kernel_debugger_entry_count = 0;
157
158#if DEVELOPMENT || DEBUG
159unsigned int panic_test_failure_mode = PANIC_TEST_FAILURE_MODE_BADPTR;
160unsigned int panic_test_action_count = 1;
161unsigned int panic_test_case = PANIC_TEST_CASE_DISABLED;
162#endif
163
164#if defined(__arm64__)
165struct additional_panic_data_buffer *panic_data_buffers = NULL;
166#endif
167
168#if defined(__arm64__)
169/*
170 * Magic number; this should be identical to the armv7 encoding for trap.
171 */
172#define TRAP_DEBUGGER __asm__ volatile(".long 0xe7ffdeff")
173#elif defined (__x86_64__)
174#define TRAP_DEBUGGER __asm__("int3")
175#else
176#error No TRAP_DEBUGGER for this architecture
177#endif
178
179#if defined(__i386__) || defined(__x86_64__)
180#define panic_stop() pmCPUHalt(PM_HALT_PANIC)
181#else
182#define panic_stop() panic_spin_forever()
183#endif
184
185#if defined(__arm64__) && (DEVELOPMENT || DEBUG)
186/*
187 * More than enough for any typical format string passed to panic();
188 * anything longer will be truncated but that's better than nothing.
189 */
190#define EARLY_PANIC_BUFLEN 256
191#endif
192
193struct debugger_state {
194 uint64_t db_panic_options;
195 debugger_op db_current_op;
196 boolean_t db_proceed_on_sync_failure;
197 const char *db_message;
198 const char *db_panic_str;
199 va_list *db_panic_args;
200 void *db_panic_data_ptr;
201 unsigned long db_panic_caller;
202 /* incremented whenever we panic or call Debugger (current CPU panic level) */
203 uint32_t db_entry_count;
204 kern_return_t db_op_return;
205};
206static struct debugger_state PERCPU_DATA(debugger_state);
207
208/* __pure2 is correct if this function is called with preemption disabled */
209static inline __pure2 struct debugger_state *
210current_debugger_state(void)
211{
212 return PERCPU_GET(debugger_state);
213}
214
215#define CPUDEBUGGEROP current_debugger_state()->db_current_op
216#define CPUDEBUGGERMSG current_debugger_state()->db_message
217#define CPUPANICSTR current_debugger_state()->db_panic_str
218#define CPUPANICARGS current_debugger_state()->db_panic_args
219#define CPUPANICOPTS current_debugger_state()->db_panic_options
220#define CPUPANICDATAPTR current_debugger_state()->db_panic_data_ptr
221#define CPUDEBUGGERSYNC current_debugger_state()->db_proceed_on_sync_failure
222#define CPUDEBUGGERCOUNT current_debugger_state()->db_entry_count
223#define CPUDEBUGGERRET current_debugger_state()->db_op_return
224#define CPUPANICCALLER current_debugger_state()->db_panic_caller
225
226
227/*
228 * Usage:
229 * panic_test_action_count is in the context of other flags, e.g. for IO errors it is "succeed this many times then fail" and for nesting it is "panic this many times then succeed"
230 * panic_test_failure_mode is a bit map of things to do
231 * panic_test_case is what sort of test we are injecting
232 *
233 * For more details see definitions in debugger.h
234 *
235 * Note that not all combinations are sensible, but some actions can be combined, e.g.
236 * - BADPTR+SPIN with action count = 3 will cause panic->panic->spin
237 * - BADPTR with action count = 2 will cause 2 nested panics (in addition to the initial panic)
238 * - IO_ERR with action 15 will cause 14 successful IOs, then fail on the next one
239 */
240#if DEVELOPMENT || DEBUG
241#define INJECT_NESTED_PANIC_IF_REQUESTED(requested) \
242MACRO_BEGIN \
243 if ((panic_test_case & requested) && panic_test_action_count) { \
244 panic_test_action_count--; \
245 volatile int *panic_test_badpointer = (int *)4; \
246 if ((panic_test_failure_mode & PANIC_TEST_FAILURE_MODE_SPIN) && (!panic_test_action_count)) { printf("inject spin...\n"); while(panic_test_badpointer); } \
247 if ((panic_test_failure_mode & PANIC_TEST_FAILURE_MODE_BADPTR) && (panic_test_action_count+1)) { printf("inject badptr...\n"); *panic_test_badpointer = 0; } \
248 if ((panic_test_failure_mode & PANIC_TEST_FAILURE_MODE_PANIC) && (panic_test_action_count+1)) { printf("inject panic...\n"); panic("nested panic level %d", panic_test_action_count); } \
249 } \
250MACRO_END
251
252#endif /* DEVELOPMENT || DEBUG */
253
254debugger_op debugger_current_op = DBOP_NONE;
255const char *debugger_panic_str = NULL;
256va_list *debugger_panic_args = NULL;
257void *debugger_panic_data = NULL;
258uint64_t debugger_panic_options = 0;
259const char *debugger_message = NULL;
260unsigned long debugger_panic_caller = 0;
261
262void panic_trap_to_debugger(const char *panic_format_str, va_list *panic_args,
263 unsigned int reason, void *ctx, uint64_t panic_options_mask, void *panic_data,
264 unsigned long panic_caller) __dead2 __printflike(1, 0);
265static void kdp_machine_reboot_type(unsigned int type, uint64_t debugger_flags);
266void panic_spin_forever(void) __dead2;
267extern kern_return_t do_panic_stackshot(void);
268extern kern_return_t do_stackshot(void);
269extern void PE_panic_hook(const char*);
270extern int sync(proc_t p, void *, void *);
271
272#define NESTEDDEBUGGERENTRYMAX 5
273static TUNABLE(unsigned int, max_debugger_entry_count, "nested_panic_max",
274 NESTEDDEBUGGERENTRYMAX);
275
276SECURITY_READ_ONLY_LATE(bool) awl_scratch_reg_supported = false;
277static bool PERCPU_DATA(hv_entry_detected); // = false
278static void awl_set_scratch_reg_hv_bit(void);
279void awl_mark_hv_entry(void);
280static bool awl_pm_state_change_cbk(void *param, enum cpu_event event, unsigned int cpu_or_cluster);
281
282#if defined(__arm64__)
283#define DEBUG_BUF_SIZE (4096)
284
285/* debug_buf is directly linked with iBoot panic region for arm targets */
286char *debug_buf_base = NULL;
287char *debug_buf_ptr = NULL;
288unsigned int debug_buf_size = 0;
289
290SECURITY_READ_ONLY_LATE(boolean_t) kdp_explicitly_requested = FALSE;
291#else /* defined(__arm64__) */
292#define DEBUG_BUF_SIZE ((3 * PAGE_SIZE) + offsetof(struct macos_panic_header, mph_data))
293/* EXTENDED_DEBUG_BUF_SIZE definition is now in debug.h */
294static_assert(((EXTENDED_DEBUG_BUF_SIZE % PANIC_FLUSH_BOUNDARY) == 0), "Extended debug buf size must match SMC alignment requirements");
295
296char debug_buf[DEBUG_BUF_SIZE];
297struct macos_panic_header *panic_info = (struct macos_panic_header *)debug_buf;
298char *debug_buf_base = (debug_buf + offsetof(struct macos_panic_header, mph_data));
299char *debug_buf_ptr = (debug_buf + offsetof(struct macos_panic_header, mph_data));
300
301/*
302 * We don't include the size of the panic header in the length of the data we actually write.
303 * On co-processor platforms, we lose sizeof(struct macos_panic_header) bytes from the end of
304 * the end of the log because we only support writing (3*PAGESIZE) bytes.
305 */
306unsigned int debug_buf_size = (DEBUG_BUF_SIZE - offsetof(struct macos_panic_header, mph_data));
307
308boolean_t extended_debug_log_enabled = FALSE;
309#endif /* defined(__arm64__) */
310
311#if defined(XNU_TARGET_OS_OSX)
312#define KDBG_TRACE_PANIC_FILENAME "/var/tmp/panic.trace"
313#else
314#define KDBG_TRACE_PANIC_FILENAME "/var/log/panic.trace"
315#endif
316
317/* Debugger state */
318atomic_int debugger_cpu = DEBUGGER_NO_CPU;
319boolean_t debugger_allcpus_halted = FALSE;
320boolean_t debugger_safe_to_return = TRUE;
321unsigned int debugger_context = 0;
322
323static char model_name[64];
324unsigned char *kernel_uuid;
325
326boolean_t kernelcache_uuid_valid = FALSE;
327uuid_t kernelcache_uuid;
328uuid_string_t kernelcache_uuid_string;
329
330boolean_t pageablekc_uuid_valid = FALSE;
331uuid_t pageablekc_uuid;
332uuid_string_t pageablekc_uuid_string;
333
334boolean_t auxkc_uuid_valid = FALSE;
335uuid_t auxkc_uuid;
336uuid_string_t auxkc_uuid_string;
337
338
339/*
340 * By default we treat Debugger() the same as calls to panic(), unless
341 * we have debug boot-args present and the DB_KERN_DUMP_ON_NMI *NOT* set.
342 * If DB_KERN_DUMP_ON_NMI is *NOT* set, return from Debugger() is supported.
343 *
344 * Return from Debugger() is currently only implemented on x86
345 */
346static boolean_t debugger_is_panic = TRUE;
347
348TUNABLE(unsigned int, debug_boot_arg, "debug", 0);
349
350TUNABLE(int, verbose_panic_flow_logging, "verbose_panic_flow_logging", 0);
351
352char kernel_uuid_string[37]; /* uuid_string_t */
353char kernelcache_uuid_string[37]; /* uuid_string_t */
354char panic_disk_error_description[512];
355size_t panic_disk_error_description_size = sizeof(panic_disk_error_description);
356
357extern unsigned int write_trace_on_panic;
358int kext_assertions_enable =
359#if DEBUG || DEVELOPMENT
360 TRUE;
361#else
362 FALSE;
363#endif
364
365#if (DEVELOPMENT || DEBUG)
366uint64_t xnu_platform_stall_value = PLATFORM_STALL_XNU_DISABLE;
367#endif
368
369/*
370 * Maintain the physically-contiguous carveouts for the carveout bootargs.
371 */
372TUNABLE_WRITEABLE(boolean_t, phys_carveout_core, "phys_carveout_core", 1);
373
374TUNABLE(uint32_t, phys_carveout_mb, "phys_carveout_mb", 0);
375SECURITY_READ_ONLY_LATE(vm_offset_t) phys_carveout = 0;
376SECURITY_READ_ONLY_LATE(uintptr_t) phys_carveout_pa = 0;
377SECURITY_READ_ONLY_LATE(size_t) phys_carveout_size = 0;
378
379
380/*
381 * Returns whether kernel debugging is expected to be restricted
382 * on the device currently based on CSR or other platform restrictions.
383 */
384boolean_t
385kernel_debugging_restricted(void)
386{
387#if XNU_TARGET_OS_OSX
388#if CONFIG_CSR
389 if (csr_check(CSR_ALLOW_KERNEL_DEBUGGER) != 0) {
390 return TRUE;
391 }
392#endif /* CONFIG_CSR */
393 return FALSE;
394#else /* XNU_TARGET_OS_OSX */
395 return FALSE;
396#endif /* XNU_TARGET_OS_OSX */
397}
398
399__startup_func
400static void
401panic_init(void)
402{
403 unsigned long uuidlen = 0;
404 void *uuid;
405
406 uuid = getuuidfromheader(&_mh_execute_header, &uuidlen);
407 if ((uuid != NULL) && (uuidlen == sizeof(uuid_t))) {
408 kernel_uuid = uuid;
409 uuid_unparse_upper(uu: *(uuid_t *)uuid, out: kernel_uuid_string);
410 }
411
412 /*
413 * Take the value of the debug boot-arg into account
414 */
415#if MACH_KDP
416 if (!kernel_debugging_restricted() && debug_boot_arg) {
417 if (debug_boot_arg & DB_HALT) {
418 halt_in_debugger = 1;
419 }
420
421#if defined(__arm64__)
422 if (debug_boot_arg & DB_NMI) {
423 panicDebugging = TRUE;
424 }
425#else
426 panicDebugging = TRUE;
427#endif /* defined(__arm64__) */
428 }
429
430#if defined(__arm64__)
431 char kdpname[80];
432
433 kdp_explicitly_requested = PE_parse_boot_argn(arg_string: "kdp_match_name", arg_ptr: kdpname, max_arg: sizeof(kdpname));
434#endif /* defined(__arm64__) */
435
436#endif /* MACH_KDP */
437
438#if defined (__x86_64__)
439 /*
440 * By default we treat Debugger() the same as calls to panic(), unless
441 * we have debug boot-args present and the DB_KERN_DUMP_ON_NMI *NOT* set.
442 * If DB_KERN_DUMP_ON_NMI is *NOT* set, return from Debugger() is supported.
443 * This is because writing an on-device corefile is a destructive operation.
444 *
445 * Return from Debugger() is currently only implemented on x86
446 */
447 if (PE_i_can_has_debugger(NULL) && !(debug_boot_arg & DB_KERN_DUMP_ON_NMI)) {
448 debugger_is_panic = FALSE;
449 }
450#endif
451}
452STARTUP(TUNABLES, STARTUP_RANK_MIDDLE, panic_init);
453
454#if defined (__x86_64__)
455void
456extended_debug_log_init(void)
457{
458 assert(coprocessor_paniclog_flush);
459 /*
460 * Allocate an extended panic log buffer that has space for the panic
461 * stackshot at the end. Update the debug buf pointers appropriately
462 * to point at this new buffer.
463 *
464 * iBoot pre-initializes the panic region with the NULL character. We set this here
465 * so we can accurately calculate the CRC for the region without needing to flush the
466 * full region over SMC.
467 */
468 char *new_debug_buf = kalloc_data(EXTENDED_DEBUG_BUF_SIZE, Z_WAITOK | Z_ZERO);
469
470 panic_info = (struct macos_panic_header *)new_debug_buf;
471 debug_buf_ptr = debug_buf_base = (new_debug_buf + offsetof(struct macos_panic_header, mph_data));
472 debug_buf_size = (EXTENDED_DEBUG_BUF_SIZE - offsetof(struct macos_panic_header, mph_data));
473
474 extended_debug_log_enabled = TRUE;
475
476 /*
477 * Insert a compiler barrier so we don't free the other panic stackshot buffer
478 * until after we've marked the new one as available
479 */
480 __compiler_barrier();
481 kmem_free(kernel_map, panic_stackshot_buf, panic_stackshot_buf_len);
482 panic_stackshot_buf = 0;
483 panic_stackshot_buf_len = 0;
484}
485#endif /* defined (__x86_64__) */
486
487void
488debug_log_init(void)
489{
490#if defined(__arm64__)
491 if (!gPanicBase) {
492 printf(format: "debug_log_init: Error!! gPanicBase is still not initialized\n");
493 return;
494 }
495 /* Shift debug buf start location and size by the length of the panic header */
496 debug_buf_base = (char *)gPanicBase + sizeof(struct embedded_panic_header);
497 debug_buf_ptr = debug_buf_base;
498 debug_buf_size = gPanicSize - sizeof(struct embedded_panic_header);
499
500#if CONFIG_EXT_PANICLOG
501 ext_paniclog_init();
502#endif
503#else
504 kern_return_t kr = KERN_SUCCESS;
505 bzero(panic_info, DEBUG_BUF_SIZE);
506
507 assert(debug_buf_base != NULL);
508 assert(debug_buf_ptr != NULL);
509 assert(debug_buf_size != 0);
510
511 /*
512 * We allocate a buffer to store a panic time stackshot. If we later discover that this is a
513 * system that supports flushing a stackshot via an extended debug log (see above), we'll free this memory
514 * as it's not necessary on this platform. This information won't be available until the IOPlatform has come
515 * up.
516 */
517 kr = kmem_alloc(kernel_map, &panic_stackshot_buf, PANIC_STACKSHOT_BUFSIZE,
518 KMA_DATA | KMA_ZERO, VM_KERN_MEMORY_DIAG);
519 assert(kr == KERN_SUCCESS);
520 if (kr == KERN_SUCCESS) {
521 panic_stackshot_buf_len = PANIC_STACKSHOT_BUFSIZE;
522 }
523#endif
524}
525
526void
527phys_carveout_init(void)
528{
529 if (!PE_i_can_has_debugger(NULL)) {
530 return;
531 }
532
533#if __arm__ || __arm64__
534#if DEVELOPMENT || DEBUG
535#endif /* DEVELOPMENT || DEBUG */
536#endif /* __arm__ || __arm64__ */
537
538 struct carveout {
539 const char *name;
540 vm_offset_t *va;
541 uint32_t requested_size;
542 uintptr_t *pa;
543 size_t *allocated_size;
544 uint64_t present;
545 } carveouts[] = {
546 {
547 "phys_carveout",
548 &phys_carveout,
549 phys_carveout_mb,
550 &phys_carveout_pa,
551 &phys_carveout_size,
552 phys_carveout_mb != 0,
553 }
554 };
555
556 for (int i = 0; i < (sizeof(carveouts) / sizeof(struct carveout)); i++) {
557 if (carveouts[i].present) {
558 size_t temp_carveout_size = 0;
559 if (os_mul_overflow(carveouts[i].requested_size, 1024 * 1024, &temp_carveout_size)) {
560 panic("%s_mb size overflowed (%uMB)",
561 carveouts[i].name, carveouts[i].requested_size);
562 return;
563 }
564
565 kmem_alloc_contig(map: kernel_map, addrp: carveouts[i].va,
566 size: temp_carveout_size, PAGE_MASK, max_pnum: 0, pnum_mask: 0,
567 flags: KMA_NOFAIL | KMA_PERMANENT | KMA_NOPAGEWAIT | KMA_DATA,
568 VM_KERN_MEMORY_DIAG);
569
570 *carveouts[i].pa = kvtophys(va: *carveouts[i].va);
571 *carveouts[i].allocated_size = temp_carveout_size;
572 }
573 }
574
575#if __arm64__ && (DEVELOPMENT || DEBUG)
576 /* likely panic_trace boot-arg is also set so check and enable tracing if necessary into new carveout */
577 PE_arm_debug_enable_trace(true);
578#endif /* __arm64__ && (DEVELOPMENT || DEBUG) */
579}
580
581boolean_t
582debug_is_in_phys_carveout(vm_map_offset_t va)
583{
584 return phys_carveout_size && va >= phys_carveout &&
585 va < (phys_carveout + phys_carveout_size);
586}
587
588boolean_t
589debug_can_coredump_phys_carveout(void)
590{
591 return phys_carveout_core;
592}
593
594static void
595DebuggerLock(void)
596{
597 int my_cpu = cpu_number();
598 int debugger_exp_cpu = DEBUGGER_NO_CPU;
599 assert(ml_get_interrupts_enabled() == FALSE);
600
601 if (atomic_load(&debugger_cpu) == my_cpu) {
602 return;
603 }
604
605 while (!atomic_compare_exchange_strong(&debugger_cpu, &debugger_exp_cpu, my_cpu)) {
606 debugger_exp_cpu = DEBUGGER_NO_CPU;
607 }
608
609 return;
610}
611
612static void
613DebuggerUnlock(void)
614{
615 assert(atomic_load_explicit(&debugger_cpu, memory_order_relaxed) == cpu_number());
616
617 /*
618 * We don't do an atomic exchange here in case
619 * there's another CPU spinning to acquire the debugger_lock
620 * and we never get a chance to update it. We already have the
621 * lock so we can simply store DEBUGGER_NO_CPU and follow with
622 * a barrier.
623 */
624 atomic_store(&debugger_cpu, DEBUGGER_NO_CPU);
625 OSMemoryBarrier();
626
627 return;
628}
629
630static kern_return_t
631DebuggerHaltOtherCores(boolean_t proceed_on_failure, bool is_stackshot)
632{
633#if defined(__arm64__)
634 return DebuggerXCallEnter(proceed_on_failure, is_stackshot);
635#else /* defined(__arm64__) */
636#pragma unused(proceed_on_failure)
637#pragma unused(is_stackshot)
638 mp_kdp_enter(proceed_on_failure);
639 return KERN_SUCCESS;
640#endif
641}
642
643static void
644DebuggerResumeOtherCores(void)
645{
646#if defined(__arm64__)
647 DebuggerXCallReturn();
648#else /* defined(__arm64__) */
649 mp_kdp_exit();
650#endif
651}
652
653__printflike(3, 0)
654static void
655DebuggerSaveState(debugger_op db_op, const char *db_message, const char *db_panic_str,
656 va_list *db_panic_args, uint64_t db_panic_options, void *db_panic_data_ptr,
657 boolean_t db_proceed_on_sync_failure, unsigned long db_panic_caller)
658{
659 CPUDEBUGGEROP = db_op;
660
661 /*
662 * Note:
663 * if CPUDEBUGGERCOUNT == 1 then we are in the normal case - record the panic data
664 * if CPUDEBUGGERCOUNT > 1 and CPUPANICSTR == NULL then we are in a nested panic that happened before DebuggerSaveState was called, so store the nested panic data
665 * if CPUDEBUGGERCOUNT > 1 and CPUPANICSTR != NULL then we are in a nested panic that happened after DebuggerSaveState was called, so leave the original panic data
666 *
667 * TODO: is it safe to flatten this to if (CPUPANICSTR == NULL)?
668 */
669 if (CPUDEBUGGERCOUNT == 1 || CPUPANICSTR == NULL) {
670 CPUDEBUGGERMSG = db_message;
671 CPUPANICSTR = db_panic_str;
672 CPUPANICARGS = db_panic_args;
673 CPUPANICDATAPTR = db_panic_data_ptr;
674 CPUPANICCALLER = db_panic_caller;
675
676#if CONFIG_EXCLAVES
677 char *panic_str;
678 if (exclaves_panic_get_string(&panic_str) == KERN_SUCCESS) {
679 CPUPANICSTR = panic_str;
680 }
681#endif
682 }
683
684 CPUDEBUGGERSYNC = db_proceed_on_sync_failure;
685 CPUDEBUGGERRET = KERN_SUCCESS;
686
687 /* Reset these on any nested panics */
688 // follow up in rdar://88497308 (nested panics should not clobber panic flags)
689 CPUPANICOPTS = db_panic_options;
690
691 return;
692}
693
694/*
695 * Save the requested debugger state/action into the current processor's
696 * percu state and trap to the debugger.
697 */
698kern_return_t
699DebuggerTrapWithState(debugger_op db_op, const char *db_message, const char *db_panic_str,
700 va_list *db_panic_args, uint64_t db_panic_options, void *db_panic_data_ptr,
701 boolean_t db_proceed_on_sync_failure, unsigned long db_panic_caller)
702{
703 kern_return_t ret;
704
705#if defined(__arm64__) && (DEVELOPMENT || DEBUG)
706 if (!PE_arm_debug_and_trace_initialized()) {
707 /*
708 * In practice this can only happen if we panicked very early,
709 * when only the boot CPU is online and before it has finished
710 * initializing the debug and trace infrastructure. We're going
711 * to hang soon, so let's at least make sure the message passed
712 * to panic() is actually logged.
713 */
714 char buf[EARLY_PANIC_BUFLEN];
715 vsnprintf(buf, EARLY_PANIC_BUFLEN, db_panic_str, *db_panic_args);
716 paniclog_append_noflush("%s\n", buf);
717 }
718#endif
719
720 assert(ml_get_interrupts_enabled() == FALSE);
721 DebuggerSaveState(db_op, db_message, db_panic_str, db_panic_args,
722 db_panic_options, db_panic_data_ptr,
723 db_proceed_on_sync_failure, db_panic_caller);
724
725 /*
726 * On ARM this generates an uncategorized exception -> sleh code ->
727 * DebuggerCall -> kdp_trap -> handle_debugger_trap
728 * So that is how XNU ensures that only one core can panic.
729 * The rest of the cores are halted by IPI if possible; if that
730 * fails it will fall back to dbgwrap.
731 */
732 TRAP_DEBUGGER;
733
734 ret = CPUDEBUGGERRET;
735
736 DebuggerSaveState(db_op: DBOP_NONE, NULL, NULL, NULL, db_panic_options: 0, NULL, FALSE, db_panic_caller: 0);
737
738 return ret;
739}
740
741void __attribute__((noinline))
742Assert(
743 const char *file,
744 int line,
745 const char *expression
746 )
747{
748#if CONFIG_NONFATAL_ASSERTS
749 static TUNABLE(bool, mach_assert, "assertions", true);
750
751 if (!mach_assert) {
752 kprintf("%s:%d non-fatal Assertion: %s", file, line, expression);
753 return;
754 }
755#endif
756
757 panic_plain("%s:%d Assertion failed: %s", file, line, expression);
758}
759
760boolean_t
761debug_is_current_cpu_in_panic_state(void)
762{
763 return current_debugger_state()->db_entry_count > 0;
764}
765
766/*
767 * check if we are in a nested panic, report findings, take evasive action where necessary
768 *
769 * see also PE_update_panicheader_nestedpanic
770 */
771static void
772check_and_handle_nested_panic(uint64_t panic_options_mask, unsigned long panic_caller, const char *db_panic_str, va_list *db_panic_args)
773{
774 if ((CPUDEBUGGERCOUNT > 1) && (CPUDEBUGGERCOUNT < max_debugger_entry_count)) {
775 // Note: this is the first indication in the panic log or serial that we are off the rails...
776 //
777 // if we panic *before* the paniclog is finalized then this will end up in the ips report with a panic_caller addr that gives us a clue
778 // if we panic *after* the log is finalized then we will only see it in the serial log
779 //
780 paniclog_append_noflush(format: "Nested panic detected - entry count: %d panic_caller: 0x%016lx\n", CPUDEBUGGERCOUNT, panic_caller);
781 paniclog_flush();
782
783 // print the *new* panic string to the console, we might not get it by other means...
784 // TODO: I tried to write this stuff to the paniclog, but the serial output gets corrupted and the panicstring in the ips file is <mysterious>
785 // rdar://87846117 (NestedPanic: output panic string to paniclog)
786 if (db_panic_str) {
787 printf(format: "Nested panic string:\n");
788#pragma clang diagnostic push
789#pragma clang diagnostic ignored "-Wformat-nonliteral"
790 _doprnt(fmt: db_panic_str, argp: db_panic_args, putc: PE_kputc, radix: 0);
791#pragma clang diagnostic pop
792 printf(format: "\n<end nested panic string>\n");
793 }
794 }
795
796 // Stage 1 bailout
797 //
798 // Try to complete the normal panic flow, i.e. try to make sure the callouts happen and we flush the paniclog. If this fails with another nested
799 // panic then we will land in Stage 2 below...
800 //
801 if (CPUDEBUGGERCOUNT == max_debugger_entry_count) {
802 uint32_t panic_details = 0;
803
804 // if this is a force-reset panic then capture a log and reboot immediately.
805 if (panic_options_mask & DEBUGGER_OPTION_PANICLOGANDREBOOT) {
806 panic_details |= kPanicDetailsForcePowerOff;
807 }
808
809 // normally the kPEPanicBegin is sent from debugger_collect_diagnostics(), but we might nested-panic before we get
810 // there. To be safe send another notification, the function called below will only send kPEPanicBegin if it has not yet been sent.
811 //
812 PEHaltRestartInternal(type: kPEPanicBegin, details: panic_details);
813
814 paniclog_append_noflush(format: "Nested panic count exceeds limit %d, machine will reset or spin\n", max_debugger_entry_count);
815 PE_update_panicheader_nestedpanic();
816 paniclog_flush();
817
818 if (!panicDebugging) {
819 // note that this will also send kPEPanicEnd
820 kdp_machine_reboot_type(type: kPEPanicRestartCPU, debugger_flags: panic_options_mask);
821 }
822
823 // prints to console
824 paniclog_append_noflush(format: "\nNested panic stall. Stage 1 bailout. Please go to https://panic.apple.com to report this panic\n");
825 panic_spin_forever();
826 }
827
828 // Stage 2 bailout
829 //
830 // Things are severely hosed, we have nested to the point of bailout and then nested again during the bailout path. Try to issue
831 // a chipreset as quickly as possible, hopefully something in the panic log is salvageable, since we flushed it during Stage 1.
832 //
833 if (CPUDEBUGGERCOUNT == max_debugger_entry_count + 1) {
834 if (!panicDebugging) {
835 // note that:
836 // - this code path should be audited for prints, as that is a common cause of nested panics
837 // - this code path should take the fastest route to the actual reset, and not call any un-necessary code
838 kdp_machine_reboot_type(type: kPEPanicRestartCPU, debugger_flags: panic_options_mask & DEBUGGER_OPTION_SKIP_PANICEND_CALLOUTS);
839 }
840
841 // prints to console, but another nested panic will land in Stage 3 where we simply spin, so that is sort of ok...
842 paniclog_append_noflush(format: "\nIn Nested panic stall. Stage 2 bailout. Please go to https://panic.apple.com to report this panic\n");
843 panic_spin_forever();
844 }
845
846 // Stage 3 bailout
847 //
848 // We are done here, we were unable to reset the platform without another nested panic. Spin until the watchdog kicks in.
849 //
850 if (CPUDEBUGGERCOUNT > max_debugger_entry_count + 1) {
851 kdp_machine_reboot_type(type: kPEHangCPU, debugger_flags: 0);
852 }
853}
854
855void
856Debugger(const char *message)
857{
858 DebuggerWithContext(reason: 0, NULL, message, DEBUGGER_OPTION_NONE, debugger_caller: (unsigned long)(char *)__builtin_return_address(0));
859}
860
861/*
862 * Enter the Debugger
863 *
864 * This is similar to, but not the same as a panic
865 *
866 * Key differences:
867 * - we get here from a debugger entry action (e.g. NMI)
868 * - the system is resumable on x86 (in theory, however it is not clear if this is tested)
869 * - rdar://57738811 (xnu: support resume from debugger via KDP on arm devices)
870 *
871 */
872void
873DebuggerWithContext(unsigned int reason, void *ctx, const char *message,
874 uint64_t debugger_options_mask, unsigned long debugger_caller)
875{
876 spl_t previous_interrupts_state;
877 boolean_t old_doprnt_hide_pointers = doprnt_hide_pointers;
878
879#if defined(__x86_64__) && (DEVELOPMENT || DEBUG)
880 read_lbr();
881#endif
882 previous_interrupts_state = ml_set_interrupts_enabled(FALSE);
883 disable_preemption();
884
885 /* track depth of debugger/panic entry */
886 CPUDEBUGGERCOUNT++;
887
888 /* emit a tracepoint as early as possible in case of hang */
889 SOCD_TRACE_XNU(PANIC, PACK_2X32(VALUE(cpu_number()), VALUE(CPUDEBUGGERCOUNT)), VALUE(debugger_options_mask), ADDR(message), ADDR(debugger_caller));
890
891 /* do max nested panic/debugger check, this will report nesting to the console and spin forever if we exceed a limit */
892 check_and_handle_nested_panic(panic_options_mask: debugger_options_mask, panic_caller: debugger_caller, db_panic_str: message, NULL);
893
894 /* Handle any necessary platform specific actions before we proceed */
895 PEInitiatePanic();
896
897#if DEVELOPMENT || DEBUG
898 INJECT_NESTED_PANIC_IF_REQUESTED(PANIC_TEST_CASE_RECURPANIC_ENTRY);
899#endif
900
901 PE_panic_hook(message);
902
903 doprnt_hide_pointers = FALSE;
904
905 if (ctx != NULL) {
906 DebuggerSaveState(db_op: DBOP_DEBUGGER, db_message: message,
907 NULL, NULL, db_panic_options: debugger_options_mask, NULL, TRUE, db_panic_caller: 0);
908 handle_debugger_trap(exception: reason, code: 0, subcode: 0, state: ctx);
909 DebuggerSaveState(db_op: DBOP_NONE, NULL, NULL,
910 NULL, db_panic_options: 0, NULL, FALSE, db_panic_caller: 0);
911 } else {
912 DebuggerTrapWithState(db_op: DBOP_DEBUGGER, db_message: message,
913 NULL, NULL, db_panic_options: debugger_options_mask, NULL, TRUE, db_panic_caller: 0);
914 }
915
916 /* resume from the debugger */
917
918 CPUDEBUGGERCOUNT--;
919 doprnt_hide_pointers = old_doprnt_hide_pointers;
920 enable_preemption();
921 ml_set_interrupts_enabled(enable: previous_interrupts_state);
922}
923
924static struct kdp_callout {
925 struct kdp_callout * callout_next;
926 kdp_callout_fn_t callout_fn;
927 boolean_t callout_in_progress;
928 void * callout_arg;
929} * kdp_callout_list = NULL;
930
931/*
932 * Called from kernel context to register a kdp event callout.
933 */
934void
935kdp_register_callout(kdp_callout_fn_t fn, void * arg)
936{
937 struct kdp_callout * kcp;
938 struct kdp_callout * list_head;
939
940 kcp = zalloc_permanent_type(struct kdp_callout);
941
942 kcp->callout_fn = fn;
943 kcp->callout_arg = arg;
944 kcp->callout_in_progress = FALSE;
945
946 /* Lock-less list insertion using compare and exchange. */
947 do {
948 list_head = kdp_callout_list;
949 kcp->callout_next = list_head;
950 } while (!OSCompareAndSwapPtr(list_head, kcp, &kdp_callout_list));
951}
952
953static void
954kdp_callouts(kdp_event_t event)
955{
956 struct kdp_callout *kcp = kdp_callout_list;
957
958 while (kcp) {
959 if (!kcp->callout_in_progress) {
960 kcp->callout_in_progress = TRUE;
961 kcp->callout_fn(kcp->callout_arg, event);
962 kcp->callout_in_progress = FALSE;
963 }
964 kcp = kcp->callout_next;
965 }
966}
967
968#if defined(__arm64__)
969/*
970 * Register an additional buffer with data to include in the panic log
971 *
972 * <rdar://problem/50137705> tracks supporting more than one buffer
973 *
974 * Note that producer_name and buf should never be de-allocated as we reference these during panic.
975 */
976void
977register_additional_panic_data_buffer(const char *producer_name, void *buf, int len)
978{
979 if (panic_data_buffers != NULL) {
980 panic("register_additional_panic_data_buffer called with buffer already registered");
981 }
982
983 if (producer_name == NULL || (strlen(s: producer_name) == 0)) {
984 panic("register_additional_panic_data_buffer called with invalid producer_name");
985 }
986
987 if (buf == NULL) {
988 panic("register_additional_panic_data_buffer called with invalid buffer pointer");
989 }
990
991 if ((len <= 0) || (len > ADDITIONAL_PANIC_DATA_BUFFER_MAX_LEN)) {
992 panic("register_additional_panic_data_buffer called with invalid length");
993 }
994
995 struct additional_panic_data_buffer *new_panic_data_buffer = zalloc_permanent_type(struct additional_panic_data_buffer);
996 new_panic_data_buffer->producer_name = producer_name;
997 new_panic_data_buffer->buf = buf;
998 new_panic_data_buffer->len = len;
999
1000 if (!OSCompareAndSwapPtr(NULL, new_panic_data_buffer, &panic_data_buffers)) {
1001 panic("register_additional_panic_data_buffer called with buffer already registered");
1002 }
1003
1004 return;
1005}
1006#endif /* defined(__arm64__) */
1007
1008/*
1009 * An overview of the xnu panic path:
1010 *
1011 * Several panic wrappers (panic(), panic_with_options(), etc.) all funnel into panic_trap_to_debugger().
1012 * panic_trap_to_debugger() sets the panic state in the current processor's debugger_state prior
1013 * to trapping into the debugger. Once we trap to the debugger, we end up in handle_debugger_trap()
1014 * which tries to acquire the panic lock by atomically swapping the current CPU number into debugger_cpu.
1015 * debugger_cpu acts as a synchronization point, from which the winning CPU can halt the other cores and
1016 * continue to debugger_collect_diagnostics() where we write the paniclog, corefile (if appropriate) and proceed
1017 * according to the device's boot-args.
1018 */
1019#undef panic
1020void
1021panic(const char *str, ...)
1022{
1023 va_list panic_str_args;
1024
1025 va_start(panic_str_args, str);
1026 panic_trap_to_debugger(panic_format_str: str, panic_args: &panic_str_args, reason: 0, NULL, panic_options_mask: 0, NULL, panic_caller: (unsigned long)(char *)__builtin_return_address(0));
1027 va_end(panic_str_args);
1028}
1029
1030void
1031panic_with_data(uuid_t uuid, void *addr, uint32_t len, const char *str, ...)
1032{
1033 va_list panic_str_args;
1034
1035 ext_paniclog_panic_with_data(uuid, addr, len);
1036
1037 va_start(panic_str_args, str);
1038 panic_trap_to_debugger(panic_format_str: str, panic_args: &panic_str_args, reason: 0, NULL, panic_options_mask: 0, NULL, panic_caller: (unsigned long)(char *)__builtin_return_address(0));
1039 va_end(panic_str_args);
1040}
1041
1042void
1043panic_with_options(unsigned int reason, void *ctx, uint64_t debugger_options_mask, const char *str, ...)
1044{
1045 va_list panic_str_args;
1046
1047 va_start(panic_str_args, str);
1048 panic_trap_to_debugger(panic_format_str: str, panic_args: &panic_str_args, reason, ctx, panic_options_mask: (debugger_options_mask & ~DEBUGGER_INTERNAL_OPTIONS_MASK),
1049 NULL, panic_caller: (unsigned long)(char *)__builtin_return_address(0));
1050 va_end(panic_str_args);
1051}
1052
1053boolean_t
1054panic_validate_ptr(void *ptr, vm_size_t size, const char *what)
1055{
1056 if (ptr == NULL) {
1057 paniclog_append_noflush(format: "NULL %s pointer\n", what);
1058 return false;
1059 }
1060
1061 if (!ml_validate_nofault(virtsrc: (vm_offset_t)ptr, size)) {
1062 paniclog_append_noflush(format: "Invalid %s pointer: %p (size %d)\n",
1063 what, ptr, (uint32_t)size);
1064 return false;
1065 }
1066
1067 return true;
1068}
1069
1070boolean_t
1071panic_get_thread_proc_task(struct thread *thread, struct task **task, struct proc **proc)
1072{
1073 if (!PANIC_VALIDATE_PTR(thread)) {
1074 return false;
1075 }
1076
1077 if (!PANIC_VALIDATE_PTR(thread->t_tro)) {
1078 return false;
1079 }
1080
1081 if (!PANIC_VALIDATE_PTR(thread->t_tro->tro_task)) {
1082 return false;
1083 }
1084
1085 if (task) {
1086 *task = thread->t_tro->tro_task;
1087 }
1088
1089 if (!panic_validate_ptr(ptr: thread->t_tro->tro_proc,
1090 size: sizeof(struct proc *), what: "bsd_info")) {
1091 *proc = NULL;
1092 } else {
1093 *proc = thread->t_tro->tro_proc;
1094 }
1095
1096 return true;
1097}
1098
1099#if defined (__x86_64__)
1100/*
1101 * panic_with_thread_context() is used on x86 platforms to specify a different thread that should be backtraced in the paniclog.
1102 * We don't generally need this functionality on embedded platforms because embedded platforms include a panic time stackshot
1103 * from customer devices. We plumb the thread pointer via the debugger trap mechanism and backtrace the kernel stack from the
1104 * thread when writing the panic log.
1105 *
1106 * NOTE: panic_with_thread_context() should be called with an explicit thread reference held on the passed thread.
1107 */
1108void
1109panic_with_thread_context(unsigned int reason, void *ctx, uint64_t debugger_options_mask, thread_t thread, const char *str, ...)
1110{
1111 va_list panic_str_args;
1112 __assert_only os_ref_count_t th_ref_count;
1113
1114 assert_thread_magic(thread);
1115 th_ref_count = os_ref_get_count_raw(&thread->ref_count);
1116 assertf(th_ref_count > 0, "panic_with_thread_context called with invalid thread %p with refcount %u", thread, th_ref_count);
1117
1118 /* Take a reference on the thread so it doesn't disappear by the time we try to backtrace it */
1119 thread_reference(thread);
1120
1121 va_start(panic_str_args, str);
1122 panic_trap_to_debugger(str, &panic_str_args, reason, ctx, ((debugger_options_mask & ~DEBUGGER_INTERNAL_OPTIONS_MASK) | DEBUGGER_INTERNAL_OPTION_THREAD_BACKTRACE),
1123 thread, (unsigned long)(char *)__builtin_return_address(0));
1124
1125 va_end(panic_str_args);
1126}
1127#endif /* defined (__x86_64__) */
1128
1129#pragma clang diagnostic push
1130#pragma clang diagnostic ignored "-Wmissing-noreturn"
1131void
1132panic_trap_to_debugger(const char *panic_format_str, va_list *panic_args, unsigned int reason, void *ctx,
1133 uint64_t panic_options_mask, void *panic_data_ptr, unsigned long panic_caller)
1134{
1135#pragma clang diagnostic pop
1136
1137#if defined(__x86_64__) && (DEVELOPMENT || DEBUG)
1138 read_lbr();
1139#endif
1140
1141#if CONFIG_SPTM
1142 /*
1143 * If SPTM has not itself already panicked, trigger a panic lockdown. This
1144 * check is necessary since attempting to re-enter the SPTM after it calls
1145 * panic will lead to a hang, which harms kernel field debugability.
1146 *
1147 * Whether or not this check can be subverted is murky. This doesn't really
1148 * matter, however, because any security critical panics events will have
1149 * already initiated lockdown before calling panic. Thus, lockdown from
1150 * panic itself is merely a "best effort".
1151 */
1152 libsptm_error_t sptm_error = LIBSPTM_SUCCESS;
1153 bool sptm_has_panicked = false;
1154 if (((sptm_error = sptm_triggered_panic(&sptm_has_panicked)) == LIBSPTM_SUCCESS) &&
1155 !sptm_has_panicked) {
1156 sptm_xnu_panic_begin();
1157 }
1158#endif /* CONFIG_SPTM */
1159
1160 /* optionally call sync, to reduce lost logs on restart, avoid on recursive panic. Unsafe due to unbounded sync() duration */
1161 if ((panic_options_mask & DEBUGGER_OPTION_SYNC_ON_PANIC_UNSAFE) && (CPUDEBUGGERCOUNT == 0)) {
1162 sync(NULL, NULL, NULL);
1163 }
1164
1165 /* Turn off I/O tracing once we've panicked */
1166 iotrace_disable();
1167
1168 /* call machine-layer panic handler */
1169 ml_panic_trap_to_debugger(panic_format_str, panic_args, reason, ctx, panic_options_mask, panic_caller);
1170
1171 /* track depth of debugger/panic entry */
1172 CPUDEBUGGERCOUNT++;
1173
1174 /* emit a tracepoint as early as possible in case of hang */
1175 SOCD_TRACE_XNU(PANIC, PACK_2X32(VALUE(cpu_number()), VALUE(CPUDEBUGGERCOUNT)), VALUE(panic_options_mask), ADDR(panic_format_str), ADDR(panic_caller));
1176
1177 /* do max nested panic/debugger check, this will report nesting to the console and spin forever if we exceed a limit */
1178 check_and_handle_nested_panic(panic_options_mask, panic_caller, db_panic_str: panic_format_str, db_panic_args: panic_args);
1179
1180 /* Handle any necessary platform specific actions before we proceed */
1181 PEInitiatePanic();
1182
1183#if DEVELOPMENT || DEBUG
1184 INJECT_NESTED_PANIC_IF_REQUESTED(PANIC_TEST_CASE_RECURPANIC_ENTRY);
1185#endif
1186
1187 PE_panic_hook(panic_format_str);
1188
1189#if defined (__x86_64__)
1190 plctrace_disable();
1191#endif
1192
1193 if (write_trace_on_panic && kdebug_enable) {
1194 if (get_preemption_level() == 0 && !ml_at_interrupt_context()) {
1195 ml_set_interrupts_enabled(TRUE);
1196 KDBG_RELEASE(TRACE_PANIC);
1197 kdbg_dump_trace_to_file(KDBG_TRACE_PANIC_FILENAME, false);
1198 }
1199 }
1200
1201 ml_set_interrupts_enabled(FALSE);
1202 disable_preemption();
1203
1204#if defined (__x86_64__)
1205 pmSafeMode(x86_lcpu(), PM_SAFE_FL_SAFE);
1206#endif /* defined (__x86_64__) */
1207
1208 /* Never hide pointers from panic logs. */
1209 doprnt_hide_pointers = FALSE;
1210
1211 if (ctx != NULL) {
1212 /*
1213 * We called into panic from a trap, no need to trap again. Set the
1214 * state on the current CPU and then jump to handle_debugger_trap.
1215 */
1216 DebuggerSaveState(db_op: DBOP_PANIC, db_message: "panic",
1217 db_panic_str: panic_format_str, db_panic_args: panic_args,
1218 db_panic_options: panic_options_mask, db_panic_data_ptr: panic_data_ptr, TRUE, db_panic_caller: panic_caller);
1219 handle_debugger_trap(exception: reason, code: 0, subcode: 0, state: ctx);
1220 }
1221
1222#if defined(__arm64__) && !APPLEVIRTUALPLATFORM
1223 /*
1224 * Signal to fastsim that it should open debug ports (nop on hardware)
1225 */
1226 __asm__ volatile ("hint #0x45");
1227#endif /* defined(__arm64__) && !APPLEVIRTUALPLATFORM */
1228
1229 DebuggerTrapWithState(db_op: DBOP_PANIC, db_message: "panic", db_panic_str: panic_format_str,
1230 db_panic_args: panic_args, db_panic_options: panic_options_mask, db_panic_data_ptr: panic_data_ptr, TRUE, db_panic_caller: panic_caller);
1231
1232 /*
1233 * Not reached.
1234 */
1235 panic_stop();
1236 __builtin_unreachable();
1237}
1238
1239void
1240panic_spin_forever(void)
1241{
1242 for (;;) {
1243#if defined(__arm__) || defined(__arm64__)
1244 /* On arm32, which doesn't have a WFE timeout, this may not return. But that should be OK on this path. */
1245 __builtin_arm_wfe();
1246#else
1247 cpu_pause();
1248#endif
1249 }
1250}
1251
1252static void
1253kdp_machine_reboot_type(unsigned int type, uint64_t debugger_flags)
1254{
1255 if ((type == kPEPanicRestartCPU) && (debugger_flags & DEBUGGER_OPTION_SKIP_PANICEND_CALLOUTS)) {
1256 PEHaltRestart(type: kPEPanicRestartCPUNoCallouts);
1257 } else {
1258 PEHaltRestart(type);
1259 }
1260 halt_all_cpus(TRUE);
1261}
1262
1263void
1264kdp_machine_reboot(void)
1265{
1266 kdp_machine_reboot_type(type: kPEPanicRestartCPU, debugger_flags: 0);
1267}
1268
1269static __attribute__((unused)) void
1270panic_debugger_log(const char *string, ...)
1271{
1272 va_list panic_debugger_log_args;
1273
1274 va_start(panic_debugger_log_args, string);
1275#pragma clang diagnostic push
1276#pragma clang diagnostic ignored "-Wformat-nonliteral"
1277 _doprnt(fmt: string, argp: &panic_debugger_log_args, putc: consdebug_putc, radix: 16);
1278#pragma clang diagnostic pop
1279 va_end(panic_debugger_log_args);
1280
1281#if defined(__arm64__)
1282 paniclog_flush();
1283#endif
1284}
1285
1286/*
1287 * Gather and save diagnostic information about a panic (or Debugger call).
1288 *
1289 * On embedded, Debugger and Panic are treated very similarly -- WDT uses Debugger so we can
1290 * theoretically return from it. On desktop, Debugger is treated as a conventional debugger -- i.e no
1291 * paniclog is written and no core is written unless we request a core on NMI.
1292 *
1293 * This routine handles kicking off local coredumps, paniclogs, calling into the Debugger/KDP (if it's configured),
1294 * and calling out to any other functions we have for collecting diagnostic info.
1295 */
1296static void
1297debugger_collect_diagnostics(unsigned int exception, unsigned int code, unsigned int subcode, void *state)
1298{
1299#if DEVELOPMENT || DEBUG
1300 INJECT_NESTED_PANIC_IF_REQUESTED(PANIC_TEST_CASE_RECURPANIC_PRELOG);
1301#endif
1302
1303#if defined(__x86_64__)
1304 kprintf("Debugger called: <%s>\n", debugger_message ? debugger_message : "");
1305#endif
1306 /*
1307 * DB_HALT (halt_in_debugger) can be requested on startup, we shouldn't generate
1308 * a coredump/paniclog for this type of debugger entry. If KDP isn't configured,
1309 * we'll just spin in kdp_raise_exception.
1310 */
1311 if (debugger_current_op == DBOP_DEBUGGER && halt_in_debugger) {
1312 kdp_raise_exception(exception, code, subcode, saved_state: state);
1313 if (debugger_safe_to_return && !debugger_is_panic) {
1314 return;
1315 }
1316 }
1317
1318#ifdef CONFIG_KCOV
1319 /* Try not to break core dump path by sanitizer. */
1320 kcov_panic_disable();
1321#endif
1322
1323 if ((debugger_current_op == DBOP_PANIC) ||
1324 ((debugger_current_op == DBOP_DEBUGGER) && debugger_is_panic)) {
1325 /*
1326 * Attempt to notify listeners once and only once that we've started
1327 * panicking. Only do this for Debugger() calls if we're treating
1328 * Debugger() calls like panic().
1329 */
1330 uint32_t panic_details = 0;
1331 /* if this is a force-reset panic then capture a log and reboot immediately. */
1332 if (debugger_panic_options & DEBUGGER_OPTION_PANICLOGANDREBOOT) {
1333 panic_details |= kPanicDetailsForcePowerOff;
1334 }
1335 PEHaltRestartInternal(type: kPEPanicBegin, details: panic_details);
1336
1337 /*
1338 * Set the begin pointer in the panic log structure. We key off of this
1339 * static variable rather than contents from the panic header itself in case someone
1340 * has stomped over the panic_info structure. Also initializes the header magic.
1341 */
1342 static boolean_t began_writing_paniclog = FALSE;
1343 if (!began_writing_paniclog) {
1344 PE_init_panicheader();
1345 began_writing_paniclog = TRUE;
1346 }
1347
1348 if (CPUDEBUGGERCOUNT > 1) {
1349 /*
1350 * we are in a nested panic. Record the nested bit in panic flags and do some housekeeping
1351 */
1352 PE_update_panicheader_nestedpanic();
1353 paniclog_flush();
1354 }
1355 }
1356
1357 /*
1358 * Write panic string if this was a panic.
1359 *
1360 * TODO: Consider moving to SavePanicInfo as this is part of the panic log.
1361 */
1362 if (debugger_current_op == DBOP_PANIC) {
1363 paniclog_append_noflush(format: "panic(cpu %u caller 0x%lx): ", (unsigned) cpu_number(), debugger_panic_caller);
1364 if (debugger_panic_str) {
1365#pragma clang diagnostic push
1366#pragma clang diagnostic ignored "-Wformat-nonliteral"
1367 _doprnt(fmt: debugger_panic_str, argp: debugger_panic_args, putc: consdebug_putc, radix: 0);
1368#pragma clang diagnostic pop
1369 }
1370 paniclog_append_noflush(format: "\n");
1371 }
1372#if defined(__x86_64__)
1373 else if (((debugger_current_op == DBOP_DEBUGGER) && debugger_is_panic)) {
1374 paniclog_append_noflush("Debugger called: <%s>\n", debugger_message ? debugger_message : "");
1375 }
1376
1377 /*
1378 * Debugger() is treated like panic() on embedded -- for example we use it for WDT
1379 * panics (so we need to write a paniclog). On desktop Debugger() is used in the
1380 * conventional sense.
1381 */
1382 if (debugger_current_op == DBOP_PANIC || ((debugger_current_op == DBOP_DEBUGGER) && debugger_is_panic))
1383#endif /* __x86_64__ */
1384 {
1385 kdp_callouts(event: KDP_EVENT_PANICLOG);
1386
1387 /*
1388 * Write paniclog and panic stackshot (if supported)
1389 * TODO: Need to clear panic log when return from debugger
1390 * hooked up for embedded
1391 */
1392 SavePanicInfo(message: debugger_message, panic_data: debugger_panic_data, panic_options: debugger_panic_options);
1393
1394#if DEVELOPMENT || DEBUG
1395 INJECT_NESTED_PANIC_IF_REQUESTED(PANIC_TEST_CASE_RECURPANIC_POSTLOG);
1396#endif
1397
1398 /* DEBUGGER_OPTION_PANICLOGANDREBOOT is used for two finger resets on embedded so we get a paniclog */
1399 if (debugger_panic_options & DEBUGGER_OPTION_PANICLOGANDREBOOT) {
1400 PEHaltRestart(type: kPEPanicDiagnosticsDone);
1401 PEHaltRestart(type: kPEPanicRestartCPUNoCallouts);
1402 }
1403 }
1404
1405#if CONFIG_KDP_INTERACTIVE_DEBUGGING
1406 /*
1407 * If reboot on panic is enabled and the caller of panic indicated that we should skip
1408 * local coredumps, don't try to write these and instead go straight to reboot. This
1409 * allows us to persist any data that's stored in the panic log.
1410 */
1411 if ((debugger_panic_options & DEBUGGER_OPTION_SKIP_LOCAL_COREDUMP) &&
1412 (debug_boot_arg & DB_REBOOT_POST_CORE)) {
1413 PEHaltRestart(type: kPEPanicDiagnosticsDone);
1414 kdp_machine_reboot_type(type: kPEPanicRestartCPU, debugger_flags: debugger_panic_options);
1415 }
1416
1417 /*
1418 * Consider generating a local corefile if the infrastructure is configured
1419 * and we haven't disabled on-device coredumps.
1420 */
1421 if (on_device_corefile_enabled()) {
1422#if CONFIG_SPTM
1423 /* We want to skip taking a local core dump if this is a panic from SPTM/TXM/cL4. */
1424 extern uint8_t sptm_supports_local_coredump;
1425 bool sptm_interrupted = false;
1426 pmap_sptm_percpu_data_t *sptm_pcpu = PERCPU_GET(pmap_sptm_percpu);
1427 sptm_get_cpu_state(sptm_pcpu->sptm_cpu_id, CPUSTATE_SPTM_INTERRUPTED, &sptm_interrupted);
1428#endif
1429 if (!kdp_has_polled_corefile()) {
1430 if (debug_boot_arg & (DB_KERN_DUMP_ON_PANIC | DB_KERN_DUMP_ON_NMI)) {
1431 paniclog_append_noflush(format: "skipping local kernel core because core file could not be opened prior to panic (mode : 0x%x, error : 0x%x)\n",
1432 kdp_polled_corefile_mode(), kdp_polled_corefile_error());
1433#if defined(__arm64__)
1434 if (kdp_polled_corefile_mode() == kIOPolledCoreFileModeUnlinked) {
1435 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COREFILE_UNLINKED;
1436 }
1437 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED;
1438 paniclog_flush();
1439#else /* defined(__arm64__) */
1440 if (panic_info->mph_panic_log_offset != 0) {
1441 if (kdp_polled_corefile_mode() == kIOPolledCoreFileModeUnlinked) {
1442 panic_info->mph_panic_flags |= MACOS_PANIC_HEADER_FLAG_COREFILE_UNLINKED;
1443 }
1444 panic_info->mph_panic_flags |= MACOS_PANIC_HEADER_FLAG_COREDUMP_FAILED;
1445 paniclog_flush();
1446 }
1447#endif /* defined(__arm64__) */
1448 }
1449 }
1450#if XNU_MONITOR
1451 else if (pmap_get_cpu_data()->ppl_state != PPL_STATE_KERNEL) {
1452 paniclog_append_noflush("skipping local kernel core because the PPL is not in KERNEL state\n");
1453 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED;
1454 paniclog_flush();
1455 }
1456#elif CONFIG_SPTM
1457 else if (!sptm_supports_local_coredump) {
1458 paniclog_append_noflush("skipping local kernel core because the SPTM is in PANIC state and can't support core dump generation\n");
1459 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED;
1460 paniclog_flush();
1461 } else if (sptm_interrupted) {
1462 paniclog_append_noflush("skipping local kernel core because the SPTM is in INTERRUPTED state and can't support core dump generation\n");
1463 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED;
1464 paniclog_flush();
1465 }
1466#endif /* XNU_MONITOR */
1467 else {
1468 int ret = -1;
1469
1470#if defined (__x86_64__)
1471 /* On x86 we don't do a coredump on Debugger unless the DB_KERN_DUMP_ON_NMI boot-arg is specified. */
1472 if (debugger_current_op != DBOP_DEBUGGER || (debug_boot_arg & DB_KERN_DUMP_ON_NMI))
1473#endif
1474 {
1475 /*
1476 * Doing an on-device coredump leaves the disk driver in a state
1477 * that can not be resumed.
1478 */
1479 debugger_safe_to_return = FALSE;
1480 begin_panic_transfer();
1481 ret = kern_dump(kd_variant: KERN_DUMP_DISK);
1482 abort_panic_transfer();
1483
1484#if DEVELOPMENT || DEBUG
1485 INJECT_NESTED_PANIC_IF_REQUESTED(PANIC_TEST_CASE_RECURPANIC_POSTCORE);
1486#endif
1487 }
1488
1489 /*
1490 * If DB_REBOOT_POST_CORE is set, then reboot if coredump is sucessfully saved
1491 * or if option to ignore failures is set.
1492 */
1493 if ((debug_boot_arg & DB_REBOOT_POST_CORE) &&
1494 ((ret == 0) || (debugger_panic_options & DEBUGGER_OPTION_ATTEMPTCOREDUMPANDREBOOT))) {
1495 PEHaltRestart(type: kPEPanicDiagnosticsDone);
1496 kdp_machine_reboot_type(type: kPEPanicRestartCPU, debugger_flags: debugger_panic_options);
1497 }
1498 }
1499 }
1500
1501 if (debugger_current_op == DBOP_PANIC ||
1502 ((debugger_current_op == DBOP_DEBUGGER) && debugger_is_panic)) {
1503 PEHaltRestart(type: kPEPanicDiagnosticsDone);
1504 }
1505
1506 if (debug_boot_arg & DB_REBOOT_ALWAYS) {
1507 kdp_machine_reboot_type(type: kPEPanicRestartCPU, debugger_flags: debugger_panic_options);
1508 }
1509
1510 /* If KDP is configured, try to trap to the debugger */
1511#if defined(__arm64__)
1512 if (kdp_explicitly_requested && (current_debugger != NO_CUR_DB)) {
1513#else
1514 if (current_debugger != NO_CUR_DB) {
1515#endif
1516 kdp_raise_exception(exception, code, subcode, saved_state: state);
1517 /*
1518 * Only return if we entered via Debugger and it's safe to return
1519 * (we halted the other cores successfully, this isn't a nested panic, etc)
1520 */
1521 if (debugger_current_op == DBOP_DEBUGGER &&
1522 debugger_safe_to_return &&
1523 kernel_debugger_entry_count == 1 &&
1524 !debugger_is_panic) {
1525 return;
1526 }
1527 }
1528
1529#if defined(__arm64__)
1530 if (PE_i_can_has_debugger(NULL) && panicDebugging) {
1531 /*
1532 * Print panic string at the end of serial output
1533 * to make panic more obvious when someone connects a debugger
1534 */
1535 if (debugger_panic_str) {
1536 panic_debugger_log(string: "Original panic string:\n");
1537 panic_debugger_log(string: "panic(cpu %u caller 0x%lx): ", (unsigned) cpu_number(), debugger_panic_caller);
1538#pragma clang diagnostic push
1539#pragma clang diagnostic ignored "-Wformat-nonliteral"
1540 _doprnt(fmt: debugger_panic_str, argp: debugger_panic_args, putc: consdebug_putc, radix: 0);
1541#pragma clang diagnostic pop
1542 panic_debugger_log(string: "\n");
1543 }
1544
1545 /* If panic debugging is configured and we're on a dev fused device, spin for astris to connect */
1546 panic_spin_shmcon();
1547 }
1548#endif /* defined(__arm64__) */
1549
1550#else /* CONFIG_KDP_INTERACTIVE_DEBUGGING */
1551
1552 PEHaltRestart(kPEPanicDiagnosticsDone);
1553
1554#endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */
1555
1556 if (!panicDebugging) {
1557 kdp_machine_reboot_type(type: kPEPanicRestartCPU, debugger_flags: debugger_panic_options);
1558 }
1559
1560 paniclog_append_noflush(format: "\nPlease go to https://panic.apple.com to report this panic\n");
1561 panic_spin_forever();
1562}
1563
1564#if SCHED_HYGIENE_DEBUG
1565uint64_t debugger_trap_timestamps[9];
1566# define DEBUGGER_TRAP_TIMESTAMP(i) debugger_trap_timestamps[i] = mach_absolute_time();
1567#else
1568# define DEBUGGER_TRAP_TIMESTAMP(i)
1569#endif /* SCHED_HYGIENE_DEBUG */
1570
1571void
1572handle_debugger_trap(unsigned int exception, unsigned int code, unsigned int subcode, void *state)
1573{
1574 unsigned int initial_not_in_kdp = not_in_kdp;
1575 kern_return_t ret;
1576 debugger_op db_prev_op = debugger_current_op;
1577
1578 DEBUGGER_TRAP_TIMESTAMP(0);
1579
1580 DebuggerLock();
1581 ret = DebuggerHaltOtherCores(CPUDEBUGGERSYNC, is_stackshot: (CPUDEBUGGEROP == DBOP_STACKSHOT));
1582
1583 DEBUGGER_TRAP_TIMESTAMP(1);
1584
1585#if SCHED_HYGIENE_DEBUG
1586 if (serialmode & SERIALMODE_OUTPUT) {
1587 ml_spin_debug_reset(current_thread());
1588 }
1589#endif /* SCHED_HYGIENE_DEBUG */
1590 if (ret != KERN_SUCCESS) {
1591 CPUDEBUGGERRET = ret;
1592 DebuggerUnlock();
1593 return;
1594 }
1595
1596 /* Update the global panic/debugger nested entry level */
1597 kernel_debugger_entry_count = CPUDEBUGGERCOUNT;
1598 if (kernel_debugger_entry_count > 0) {
1599 console_suspend();
1600 }
1601
1602 /*
1603 * TODO: Should we do anything special for nested panics here? i.e. if we've trapped more than twice
1604 * should we call into the debugger if it's configured and then reboot if the panic log has been written?
1605 */
1606
1607 if (CPUDEBUGGEROP == DBOP_NONE) {
1608 /* If there was no debugger context setup, we trapped due to a software breakpoint */
1609 debugger_current_op = DBOP_BREAKPOINT;
1610 } else {
1611 /* Not safe to return from a nested panic/debugger call */
1612 if (debugger_current_op == DBOP_PANIC ||
1613 debugger_current_op == DBOP_DEBUGGER) {
1614 debugger_safe_to_return = FALSE;
1615 }
1616
1617 debugger_current_op = CPUDEBUGGEROP;
1618
1619 /* Only overwrite the panic message if there is none already - save the data from the first call */
1620 if (debugger_panic_str == NULL) {
1621 debugger_panic_str = CPUPANICSTR;
1622 debugger_panic_args = CPUPANICARGS;
1623 debugger_panic_data = CPUPANICDATAPTR;
1624 debugger_message = CPUDEBUGGERMSG;
1625 debugger_panic_caller = CPUPANICCALLER;
1626 }
1627
1628 debugger_panic_options = CPUPANICOPTS;
1629 }
1630
1631 /*
1632 * Clear the op from the processor debugger context so we can handle
1633 * breakpoints in the debugger
1634 */
1635 CPUDEBUGGEROP = DBOP_NONE;
1636
1637 DEBUGGER_TRAP_TIMESTAMP(2);
1638
1639 kdp_callouts(event: KDP_EVENT_ENTER);
1640 not_in_kdp = 0;
1641
1642 DEBUGGER_TRAP_TIMESTAMP(3);
1643
1644#if defined(__arm64__) && CONFIG_KDP_INTERACTIVE_DEBUGGING
1645 shmem_mark_as_busy();
1646#endif
1647
1648 if (debugger_current_op == DBOP_BREAKPOINT) {
1649 kdp_raise_exception(exception, code, subcode, saved_state: state);
1650 } else if (debugger_current_op == DBOP_STACKSHOT) {
1651 CPUDEBUGGERRET = do_stackshot();
1652#if PGO
1653 } else if (debugger_current_op == DBOP_RESET_PGO_COUNTERS) {
1654 CPUDEBUGGERRET = do_pgo_reset_counters();
1655#endif
1656 } else {
1657 /* note: this is the panic path... */
1658#if CONFIG_SPTM
1659 /*
1660 * Debug trap panics do not go through the standard panic flows so we
1661 * have to notify the SPTM that we're going down now. This is not so
1662 * much for security (critical cases are handled elsewhere) but rather
1663 * to just keep the SPTM bit in sync with the actual XNU state.
1664 */
1665 bool sptm_has_panicked = false;
1666 if (sptm_triggered_panic(&sptm_has_panicked) == LIBSPTM_SUCCESS &&
1667 !sptm_has_panicked) {
1668 sptm_xnu_panic_begin();
1669 }
1670#endif /* CONFIG_SPTM */
1671#if defined(__arm64__) && (DEBUG || DEVELOPMENT)
1672 if (!PE_arm_debug_and_trace_initialized()) {
1673 paniclog_append_noflush("kernel panicked before debug and trace infrastructure initialized!\n"
1674 "spinning forever...\n");
1675 panic_spin_forever();
1676 }
1677#endif
1678 debugger_collect_diagnostics(exception, code, subcode, state);
1679 }
1680
1681#if defined(__arm64__) && CONFIG_KDP_INTERACTIVE_DEBUGGING
1682 shmem_unmark_as_busy();
1683#endif
1684
1685 DEBUGGER_TRAP_TIMESTAMP(4);
1686
1687 not_in_kdp = initial_not_in_kdp;
1688 kdp_callouts(event: KDP_EVENT_EXIT);
1689
1690 DEBUGGER_TRAP_TIMESTAMP(5);
1691
1692 if (debugger_current_op != DBOP_BREAKPOINT) {
1693 debugger_panic_str = NULL;
1694 debugger_panic_args = NULL;
1695 debugger_panic_data = NULL;
1696 debugger_panic_options = 0;
1697 debugger_message = NULL;
1698 }
1699
1700 /* Restore the previous debugger state */
1701 debugger_current_op = db_prev_op;
1702
1703 DEBUGGER_TRAP_TIMESTAMP(6);
1704
1705 DebuggerResumeOtherCores();
1706
1707 DEBUGGER_TRAP_TIMESTAMP(7);
1708
1709 DebuggerUnlock();
1710
1711 DEBUGGER_TRAP_TIMESTAMP(8);
1712
1713 return;
1714}
1715
1716__attribute__((noinline, not_tail_called))
1717void
1718log(__unused int level, char *fmt, ...)
1719{
1720 void *caller = __builtin_return_address(0);
1721 va_list listp;
1722 va_list listp2;
1723
1724
1725#ifdef lint
1726 level++;
1727#endif /* lint */
1728#ifdef MACH_BSD
1729 va_start(listp, fmt);
1730 va_copy(listp2, listp);
1731
1732 disable_preemption();
1733 _doprnt(fmt, argp: &listp, putc: cons_putc_locked, radix: 0);
1734 enable_preemption();
1735
1736 va_end(listp);
1737
1738#pragma clang diagnostic push
1739#pragma clang diagnostic ignored "-Wformat-nonliteral"
1740 os_log_with_args(OS_LOG_DEFAULT, type: OS_LOG_TYPE_DEFAULT, format: fmt, args: listp2, ret_addr: caller);
1741#pragma clang diagnostic pop
1742 va_end(listp2);
1743#endif
1744}
1745
1746/*
1747 * Per <rdar://problem/24974766>, skip appending log messages to
1748 * the new logging infrastructure in contexts where safety is
1749 * uncertain. These contexts include:
1750 * - When we're in the debugger
1751 * - We're in a panic
1752 * - Interrupts are disabled
1753 * - Or Pre-emption is disabled
1754 * In all the above cases, it is potentially unsafe to log messages.
1755 */
1756
1757boolean_t
1758oslog_is_safe(void)
1759{
1760 return kernel_debugger_entry_count == 0 &&
1761 not_in_kdp == 1 &&
1762 get_preemption_level() == 0 &&
1763 ml_get_interrupts_enabled() == TRUE;
1764}
1765
1766boolean_t
1767debug_mode_active(void)
1768{
1769 return (0 != kernel_debugger_entry_count != 0) || (0 == not_in_kdp);
1770}
1771
1772void
1773debug_putc(char c)
1774{
1775 if ((debug_buf_size != 0) &&
1776 ((debug_buf_ptr - debug_buf_base) < (int)debug_buf_size) &&
1777 (!is_debug_ptr_in_ext_paniclog())) {
1778 *debug_buf_ptr = c;
1779 debug_buf_ptr++;
1780 }
1781}
1782
1783#if defined (__x86_64__)
1784struct pasc {
1785 unsigned a: 7;
1786 unsigned b: 7;
1787 unsigned c: 7;
1788 unsigned d: 7;
1789 unsigned e: 7;
1790 unsigned f: 7;
1791 unsigned g: 7;
1792 unsigned h: 7;
1793} __attribute__((packed));
1794
1795typedef struct pasc pasc_t;
1796
1797/*
1798 * In-place packing routines -- inefficient, but they're called at most once.
1799 * Assumes "buflen" is a multiple of 8. Used for compressing paniclogs on x86.
1800 */
1801int
1802packA(char *inbuf, uint32_t length, uint32_t buflen)
1803{
1804 unsigned int i, j = 0;
1805 pasc_t pack;
1806
1807 length = MIN(((length + 7) & ~7), buflen);
1808
1809 for (i = 0; i < length; i += 8) {
1810 pack.a = inbuf[i];
1811 pack.b = inbuf[i + 1];
1812 pack.c = inbuf[i + 2];
1813 pack.d = inbuf[i + 3];
1814 pack.e = inbuf[i + 4];
1815 pack.f = inbuf[i + 5];
1816 pack.g = inbuf[i + 6];
1817 pack.h = inbuf[i + 7];
1818 bcopy((char *) &pack, inbuf + j, 7);
1819 j += 7;
1820 }
1821 return j;
1822}
1823
1824void
1825unpackA(char *inbuf, uint32_t length)
1826{
1827 pasc_t packs;
1828 unsigned i = 0;
1829 length = (length * 8) / 7;
1830
1831 while (i < length) {
1832 packs = *(pasc_t *)&inbuf[i];
1833 bcopy(&inbuf[i + 7], &inbuf[i + 8], MAX(0, (int) (length - i - 8)));
1834 inbuf[i++] = packs.a;
1835 inbuf[i++] = packs.b;
1836 inbuf[i++] = packs.c;
1837 inbuf[i++] = packs.d;
1838 inbuf[i++] = packs.e;
1839 inbuf[i++] = packs.f;
1840 inbuf[i++] = packs.g;
1841 inbuf[i++] = packs.h;
1842 }
1843}
1844#endif /* defined (__x86_64__) */
1845
1846extern char *proc_name_address(void *);
1847extern char *proc_longname_address(void *);
1848
1849__private_extern__ void
1850panic_display_process_name(void)
1851{
1852 proc_name_t proc_name = {};
1853 struct proc *cbsd_info = NULL;
1854 task_t ctask = NULL;
1855 vm_size_t size;
1856
1857 if (!panic_get_thread_proc_task(thread: current_thread(), task: &ctask, proc: &cbsd_info)) {
1858 goto out;
1859 }
1860
1861 if (cbsd_info == NULL) {
1862 goto out;
1863 }
1864
1865 size = ml_nofault_copy(virtsrc: (vm_offset_t)proc_longname_address(cbsd_info),
1866 virtdst: (vm_offset_t)&proc_name, size: sizeof(proc_name));
1867
1868 if (size == 0 || proc_name[0] == '\0') {
1869 size = ml_nofault_copy(virtsrc: (vm_offset_t)proc_name_address(cbsd_info),
1870 virtdst: (vm_offset_t)&proc_name,
1871 MIN(sizeof(command_t), sizeof(proc_name)));
1872 if (size > 0) {
1873 proc_name[size - 1] = '\0';
1874 }
1875 }
1876
1877out:
1878 proc_name[sizeof(proc_name) - 1] = '\0';
1879 paniclog_append_noflush(format: "\nProcess name corresponding to current thread (%p): %s\n",
1880 current_thread(), proc_name[0] != '\0' ? proc_name : "Unknown");
1881}
1882
1883unsigned
1884panic_active(void)
1885{
1886 return debugger_current_op == DBOP_PANIC ||
1887 (debugger_current_op == DBOP_DEBUGGER && debugger_is_panic);
1888}
1889
1890void
1891populate_model_name(char *model_string)
1892{
1893 strlcpy(dst: model_name, src: model_string, n: sizeof(model_name));
1894}
1895
1896void
1897panic_display_model_name(void)
1898{
1899 char tmp_model_name[sizeof(model_name)];
1900
1901 if (ml_nofault_copy(virtsrc: (vm_offset_t) &model_name, virtdst: (vm_offset_t) &tmp_model_name, size: sizeof(model_name)) != sizeof(model_name)) {
1902 return;
1903 }
1904
1905 tmp_model_name[sizeof(tmp_model_name) - 1] = '\0';
1906
1907 if (tmp_model_name[0] != 0) {
1908 paniclog_append_noflush(format: "System model name: %s\n", tmp_model_name);
1909 }
1910}
1911
1912void
1913panic_display_kernel_uuid(void)
1914{
1915 char tmp_kernel_uuid[sizeof(kernel_uuid_string)];
1916
1917 if (ml_nofault_copy(virtsrc: (vm_offset_t) &kernel_uuid_string, virtdst: (vm_offset_t) &tmp_kernel_uuid, size: sizeof(kernel_uuid_string)) != sizeof(kernel_uuid_string)) {
1918 return;
1919 }
1920
1921 if (tmp_kernel_uuid[0] != '\0') {
1922 paniclog_append_noflush(format: "Kernel UUID: %s\n", tmp_kernel_uuid);
1923 }
1924}
1925
1926#if CONFIG_SPTM
1927static void
1928panic_display_component_uuid(char const *component_name, void *component_address)
1929{
1930 uuid_t *component_uuid;
1931 unsigned long component_uuid_len = 0;
1932 uuid_string_t component_uuid_string;
1933
1934 component_uuid = getuuidfromheader((kernel_mach_header_t *)component_address, &component_uuid_len);
1935
1936 if (component_uuid != NULL && component_uuid_len == sizeof(uuid_t)) {
1937 uuid_unparse_upper(*component_uuid, component_uuid_string);
1938 paniclog_append_noflush("%s UUID: %s\n", component_name, component_uuid_string);
1939 }
1940}
1941#endif /* CONFIG_SPTM */
1942
1943void
1944panic_display_kernel_aslr(void)
1945{
1946#if CONFIG_SPTM
1947 {
1948 struct debug_header const *dh = SPTMArgs->debug_header;
1949
1950 paniclog_append_noflush("Debug Header address: %p\n", dh);
1951
1952 if (dh != NULL) {
1953 void *component_address;
1954
1955 paniclog_append_noflush("Debug Header entry count: %d\n", dh->count);
1956
1957 switch (dh->count) {
1958 default: // 3 or more
1959 component_address = dh->image[DEBUG_HEADER_ENTRY_TXM];
1960 paniclog_append_noflush("TXM load address: %p\n", component_address);
1961
1962 panic_display_component_uuid("TXM", component_address);
1963 OS_FALLTHROUGH;
1964 case 2:
1965 component_address = dh->image[DEBUG_HEADER_ENTRY_XNU];
1966 paniclog_append_noflush("Debug Header kernelcache load address: %p\n", component_address);
1967
1968 panic_display_component_uuid("Debug Header kernelcache", component_address);
1969 OS_FALLTHROUGH;
1970 case 1:
1971 component_address = dh->image[DEBUG_HEADER_ENTRY_SPTM];
1972 paniclog_append_noflush("SPTM load address: %p\n", component_address);
1973
1974 panic_display_component_uuid("SPTM", component_address);
1975 OS_FALLTHROUGH;
1976 case 0:
1977 ; // nothing to print
1978 }
1979 }
1980 }
1981#endif /* CONFIG_SPTM */
1982
1983 kc_format_t kc_format;
1984
1985 PE_get_primary_kc_format(type: &kc_format);
1986
1987 if (kc_format == KCFormatFileset) {
1988 void *kch = PE_get_kc_header(type: KCKindPrimary);
1989 paniclog_append_noflush(format: "KernelCache slide: 0x%016lx\n", (unsigned long) vm_kernel_slide);
1990 paniclog_append_noflush(format: "KernelCache base: %p\n", (void*) kch);
1991 paniclog_append_noflush(format: "Kernel slide: 0x%016lx\n", vm_kernel_stext - (unsigned long)kch + vm_kernel_slide);
1992 paniclog_append_noflush(format: "Kernel text base: %p\n", (void *) vm_kernel_stext);
1993#if defined(__arm64__)
1994 extern vm_offset_t segTEXTEXECB;
1995 paniclog_append_noflush(format: "Kernel text exec slide: 0x%016lx\n", (unsigned long)segTEXTEXECB - (unsigned long)kch + vm_kernel_slide);
1996 paniclog_append_noflush(format: "Kernel text exec base: 0x%016lx\n", (unsigned long)segTEXTEXECB);
1997#endif /* defined(__arm64__) */
1998 } else if (vm_kernel_slide) {
1999 paniclog_append_noflush(format: "Kernel slide: 0x%016lx\n", (unsigned long) vm_kernel_slide);
2000 paniclog_append_noflush(format: "Kernel text base: %p\n", (void *)vm_kernel_stext);
2001 } else {
2002 paniclog_append_noflush(format: "Kernel text base: %p\n", (void *)vm_kernel_stext);
2003 }
2004}
2005
2006void
2007panic_display_hibb(void)
2008{
2009#if defined(__i386__) || defined (__x86_64__)
2010 paniclog_append_noflush("__HIB text base: %p\n", (void *) vm_hib_base);
2011#endif
2012}
2013
2014#if CONFIG_ECC_LOGGING
2015__private_extern__ void
2016panic_display_ecc_errors(void)
2017{
2018 uint32_t count = ecc_log_get_correction_count();
2019
2020 if (count > 0) {
2021 paniclog_append_noflush(format: "ECC Corrections:%u\n", count);
2022 }
2023}
2024#endif /* CONFIG_ECC_LOGGING */
2025
2026#if CONFIG_FREEZE
2027extern bool freezer_incore_cseg_acct;
2028extern int32_t c_segment_pages_compressed_incore;
2029#endif
2030
2031extern uint32_t c_segment_pages_compressed;
2032extern uint32_t c_segment_count;
2033extern uint32_t c_segments_limit;
2034extern uint32_t c_segment_pages_compressed_limit;
2035extern uint32_t c_segment_pages_compressed_nearing_limit;
2036extern uint32_t c_segments_nearing_limit;
2037extern int vm_num_swap_files;
2038
2039void
2040panic_display_compressor_stats(void)
2041{
2042 int isswaplow = vm_swap_low_on_space();
2043#if CONFIG_FREEZE
2044 uint32_t incore_seg_count;
2045 uint32_t incore_compressed_pages;
2046 if (freezer_incore_cseg_acct) {
2047 incore_seg_count = c_segment_count - c_swappedout_count - c_swappedout_sparse_count;
2048 incore_compressed_pages = c_segment_pages_compressed_incore;
2049 } else {
2050 incore_seg_count = c_segment_count;
2051 incore_compressed_pages = c_segment_pages_compressed;
2052 }
2053
2054 paniclog_append_noflush("Compressor Info: %u%% of compressed pages limit (%s) and %u%% of segments limit (%s) with %d swapfiles and %s swap space\n",
2055 (incore_compressed_pages * 100) / c_segment_pages_compressed_limit,
2056 (incore_compressed_pages > c_segment_pages_compressed_nearing_limit) ? "BAD":"OK",
2057 (incore_seg_count * 100) / c_segments_limit,
2058 (incore_seg_count > c_segments_nearing_limit) ? "BAD":"OK",
2059 vm_num_swap_files,
2060 isswaplow ? "LOW":"OK");
2061#else /* CONFIG_FREEZE */
2062 paniclog_append_noflush(format: "Compressor Info: %u%% of compressed pages limit (%s) and %u%% of segments limit (%s) with %d swapfiles and %s swap space\n",
2063 (c_segment_pages_compressed * 100) / c_segment_pages_compressed_limit,
2064 (c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) ? "BAD":"OK",
2065 (c_segment_count * 100) / c_segments_limit,
2066 (c_segment_count > c_segments_nearing_limit) ? "BAD":"OK",
2067 vm_num_swap_files,
2068 isswaplow ? "LOW":"OK");
2069#endif /* CONFIG_FREEZE */
2070}
2071
2072#if !CONFIG_TELEMETRY
2073int
2074telemetry_gather(user_addr_t buffer __unused, uint32_t *length __unused, bool mark __unused)
2075{
2076 return KERN_NOT_SUPPORTED;
2077}
2078#endif
2079
2080#include <machine/machine_cpu.h>
2081
2082TUNABLE(uint32_t, kern_feature_overrides, "validation_disables", 0);
2083
2084boolean_t
2085kern_feature_override(uint32_t fmask)
2086{
2087 return (kern_feature_overrides & fmask) == fmask;
2088}
2089
2090boolean_t
2091on_device_corefile_enabled(void)
2092{
2093 assert(startup_phase >= STARTUP_SUB_TUNABLES);
2094#if CONFIG_KDP_INTERACTIVE_DEBUGGING
2095 if (debug_boot_arg == 0) {
2096 return FALSE;
2097 }
2098 if (debug_boot_arg & DB_DISABLE_LOCAL_CORE) {
2099 return FALSE;
2100 }
2101#if !XNU_TARGET_OS_OSX
2102 /*
2103 * outside of macOS, if there's a debug boot-arg set and local
2104 * cores aren't explicitly disabled, we always write a corefile.
2105 */
2106 return TRUE;
2107#else /* !XNU_TARGET_OS_OSX */
2108 /*
2109 * on macOS, if corefiles on panic are requested and local cores
2110 * aren't disabled we write a local core.
2111 */
2112 if (debug_boot_arg & (DB_KERN_DUMP_ON_NMI | DB_KERN_DUMP_ON_PANIC)) {
2113 return TRUE;
2114 }
2115#endif /* !XNU_TARGET_OS_OSX */
2116#endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */
2117 return FALSE;
2118}
2119
2120boolean_t
2121panic_stackshot_to_disk_enabled(void)
2122{
2123 assert(startup_phase >= STARTUP_SUB_TUNABLES);
2124#if defined(__x86_64__)
2125 if (PEGetCoprocessorVersion() < kCoprocessorVersion2) {
2126 /* Only enabled on pre-Gibraltar machines where it hasn't been disabled explicitly */
2127 if ((debug_boot_arg != 0) && (debug_boot_arg & DB_DISABLE_STACKSHOT_TO_DISK)) {
2128 return FALSE;
2129 }
2130
2131 return TRUE;
2132 }
2133#endif
2134 return FALSE;
2135}
2136
2137const char *
2138sysctl_debug_get_preoslog(size_t *size)
2139{
2140 int result = 0;
2141 void *preoslog_pa = NULL;
2142 int preoslog_size = 0;
2143
2144 result = IODTGetLoaderInfo(key: "preoslog", infoAddr: &preoslog_pa, infosize: &preoslog_size);
2145 if (result || preoslog_pa == NULL || preoslog_size == 0) {
2146 kprintf(fmt: "Couldn't obtain preoslog region: result = %d, preoslog_pa = %p, preoslog_size = %d\n", result, preoslog_pa, preoslog_size);
2147 *size = 0;
2148 return NULL;
2149 }
2150
2151 /*
2152 * Beware:
2153 * On release builds, we would need to call IODTFreeLoaderInfo("preoslog", preoslog_pa, preoslog_size) to free the preoslog buffer.
2154 * On Development & Debug builds, we retain the buffer so it can be extracted from coredumps.
2155 */
2156 *size = preoslog_size;
2157 return (char *)(ml_static_ptovirt((vm_offset_t)(preoslog_pa)));
2158}
2159
2160void
2161sysctl_debug_free_preoslog(void)
2162{
2163#if RELEASE
2164 int result = 0;
2165 void *preoslog_pa = NULL;
2166 int preoslog_size = 0;
2167
2168 result = IODTGetLoaderInfo("preoslog", &preoslog_pa, &preoslog_size);
2169 if (result || preoslog_pa == NULL || preoslog_size == 0) {
2170 kprintf("Couldn't obtain preoslog region: result = %d, preoslog_pa = %p, preoslog_size = %d\n", result, preoslog_pa, preoslog_size);
2171 return;
2172 }
2173
2174 IODTFreeLoaderInfo("preoslog", preoslog_pa, preoslog_size);
2175#else
2176 /* On Development & Debug builds, we retain the buffer so it can be extracted from coredumps. */
2177#endif // RELEASE
2178}
2179
2180
2181#if (DEVELOPMENT || DEBUG)
2182
2183void
2184platform_stall_panic_or_spin(uint32_t req)
2185{
2186 if (xnu_platform_stall_value & req) {
2187 if (xnu_platform_stall_value & PLATFORM_STALL_XNU_ACTION_PANIC) {
2188 panic("Platform stall: User requested panic");
2189 } else {
2190 paniclog_append_noflush("\nUser requested platform stall. Stall Code: 0x%x", req);
2191 panic_spin_forever();
2192 }
2193 }
2194}
2195#endif
2196
2197
2198#define AWL_HV_ENTRY_FLAG (0x1)
2199
2200static inline void
2201awl_set_scratch_reg_hv_bit(void)
2202{
2203#if defined(__arm64__)
2204#define WATCHDOG_DIAG0 "S3_5_c15_c2_6"
2205 uint64_t awl_diag0 = __builtin_arm_rsr64(WATCHDOG_DIAG0);
2206 awl_diag0 |= AWL_HV_ENTRY_FLAG;
2207 __builtin_arm_wsr64(WATCHDOG_DIAG0, awl_diag0);
2208#endif // defined(__arm64__)
2209}
2210
2211void
2212awl_mark_hv_entry(void)
2213{
2214 if (__probable(*PERCPU_GET(hv_entry_detected) || !awl_scratch_reg_supported)) {
2215 return;
2216 }
2217 *PERCPU_GET(hv_entry_detected) = true;
2218
2219 awl_set_scratch_reg_hv_bit();
2220}
2221
2222/*
2223 * Awl WatchdogDiag0 is not restored by hardware when coming out of reset,
2224 * so restore it manually.
2225 */
2226static bool
2227awl_pm_state_change_cbk(void *param __unused, enum cpu_event event, unsigned int cpu_or_cluster __unused)
2228{
2229 if (event == CPU_BOOTED) {
2230 if (*PERCPU_GET(hv_entry_detected)) {
2231 awl_set_scratch_reg_hv_bit();
2232 }
2233 }
2234
2235 return true;
2236}
2237
2238/*
2239 * Identifies and sets a flag if AWL Scratch0/1 exists in the system, subscribes
2240 * for a callback to restore register after hibernation
2241 */
2242__startup_func
2243static void
2244set_awl_scratch_exists_flag_and_subscribe_for_pm(void)
2245{
2246 DTEntry base = NULL;
2247
2248 if (SecureDTLookupEntry(NULL, pathName: "/arm-io/wdt", foundEntry: &base) != kSuccess) {
2249 return;
2250 }
2251 const uint8_t *data = NULL;
2252 unsigned int data_size = sizeof(uint8_t);
2253
2254 if (base != NULL && SecureDTGetProperty(entry: base, propertyName: "awl-scratch-supported", propertyValue: (const void **)&data, propertySize: &data_size) == kSuccess) {
2255 for (unsigned int i = 0; i < data_size; i++) {
2256 if (data[i] != 0) {
2257 awl_scratch_reg_supported = true;
2258 cpu_event_register_callback(fn: awl_pm_state_change_cbk, NULL);
2259 break;
2260 }
2261 }
2262 }
2263}
2264STARTUP(EARLY_BOOT, STARTUP_RANK_MIDDLE, set_awl_scratch_exists_flag_and_subscribe_for_pm);
2265