1/*
2 * Copyright (c) 2011-2022 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29/* Collect kernel callstacks */
30
31#include <mach/mach_types.h>
32#include <kern/thread.h>
33#include <kern/backtrace.h>
34#include <kern/cambria_layout.h>
35#include <vm/vm_map.h>
36#include <kperf/buffer.h>
37#include <kperf/context.h>
38#include <kperf/callstack.h>
39#include <kperf/ast.h>
40#include <sys/errno.h>
41#include <mach/exclaves.h>
42
43#if defined(__arm64__)
44#include <arm/cpu_data.h>
45#include <arm/cpu_data_internal.h>
46#endif
47
48static void
49callstack_fixup_user(struct kp_ucallstack *cs, thread_t thread)
50{
51 uint64_t fixup_val = 0;
52 assert(cs->kpuc_nframes < MAX_UCALLSTACK_FRAMES);
53
54#if defined(__x86_64__)
55 user_addr_t sp_user;
56 bool user_64;
57 x86_saved_state_t *state;
58
59 state = get_user_regs(thread);
60 if (!state) {
61 goto out;
62 }
63
64 user_64 = is_saved_state64(state);
65 if (user_64) {
66 sp_user = saved_state64(state)->isf.rsp;
67 } else {
68 sp_user = saved_state32(state)->uesp;
69 }
70
71 if (thread == current_thread()) {
72 (void)copyin(sp_user, (char *)&fixup_val,
73 user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
74 } else {
75 (void)vm_map_read_user(get_task_map(get_threadtask(thread)), sp_user,
76 &fixup_val, user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
77 }
78
79#elif defined(__arm64__)
80
81 struct arm_saved_state *state = get_user_regs(thread);
82 if (!state) {
83 goto out;
84 }
85
86 /* encode thumb mode into low bit of PC */
87 if (is_saved_state32(iss: state) && (get_saved_state_cpsr(iss: state) & PSR_TF)) {
88 cs->kpuc_frames[0] |= 1ULL;
89 }
90
91
92 fixup_val = get_saved_state_lr(iss: state);
93
94#else
95#error "callstack_fixup_user: unsupported architecture"
96#endif
97
98out:
99 cs->kpuc_frames[cs->kpuc_nframes++] = fixup_val;
100}
101
102#if defined(__x86_64__)
103
104__attribute__((used))
105static kern_return_t
106interrupted_kernel_sp_value(uintptr_t *sp_val)
107{
108 x86_saved_state_t *state;
109 uintptr_t sp;
110 bool state_64;
111 uint64_t cs;
112 uintptr_t top, bottom;
113
114 state = current_cpu_datap()->cpu_int_state;
115 if (!state) {
116 return KERN_FAILURE;
117 }
118
119 state_64 = is_saved_state64(state);
120
121 if (state_64) {
122 cs = saved_state64(state)->isf.cs;
123 } else {
124 cs = saved_state32(state)->cs;
125 }
126 /* return early if interrupted a thread in user space */
127 if ((cs & SEL_PL) == SEL_PL_U) {
128 return KERN_FAILURE;
129 }
130
131 if (state_64) {
132 sp = saved_state64(state)->isf.rsp;
133 } else {
134 sp = saved_state32(state)->uesp;
135 }
136
137 /* make sure the stack pointer is pointing somewhere in this stack */
138 bottom = current_thread()->kernel_stack;
139 top = bottom + kernel_stack_size;
140 if (sp >= bottom && sp < top) {
141 return KERN_FAILURE;
142 }
143
144 *sp_val = *(uintptr_t *)sp;
145 return KERN_SUCCESS;
146}
147
148#elif defined(__arm64__)
149
150__attribute__((used))
151static kern_return_t
152interrupted_kernel_lr(uintptr_t *lr)
153{
154 struct arm_saved_state *state;
155
156 state = getCpuDatap()->cpu_int_state;
157
158 /* return early if interrupted a thread in user space */
159 if (PSR64_IS_USER(get_saved_state_cpsr(state))) {
160 return KERN_FAILURE;
161 }
162
163 *lr = get_saved_state_lr(iss: state);
164 return KERN_SUCCESS;
165}
166#else /* defined(__arm64__) */
167#error "interrupted_kernel_{sp,lr}: unsupported architecture"
168#endif /* !defined(__arm64__) */
169
170
171static void
172callstack_fixup_interrupted(struct kp_kcallstack *cs)
173{
174 uintptr_t fixup_val = 0;
175 assert(cs->kpkc_nframes < MAX_KCALLSTACK_FRAMES);
176
177 /*
178 * Only provide arbitrary data on development or debug kernels.
179 */
180#if DEVELOPMENT || DEBUG
181#if defined(__x86_64__)
182 (void)interrupted_kernel_sp_value(&fixup_val);
183#elif defined(__arm64__)
184 (void)interrupted_kernel_lr(&fixup_val);
185#endif /* defined(__x86_64__) */
186#endif /* DEVELOPMENT || DEBUG */
187
188 assert(cs->kpkc_flags & CALLSTACK_KERNEL);
189 cs->kpkc_frames[cs->kpkc_nframes++] = fixup_val;
190}
191
192void
193kperf_continuation_sample(struct kp_kcallstack *cs, struct kperf_context *context)
194{
195 thread_t thread;
196
197 assert(cs != NULL);
198 assert(context != NULL);
199
200 thread = context->cur_thread;
201 assert(thread != NULL);
202 assert(thread->continuation != NULL);
203
204 cs->kpkc_flags = CALLSTACK_CONTINUATION | CALLSTACK_VALID | CALLSTACK_KERNEL;
205#ifdef __LP64__
206 cs->kpkc_flags |= CALLSTACK_64BIT;
207#endif
208
209 cs->kpkc_nframes = 1;
210 cs->kpkc_frames[0] = VM_KERNEL_UNSLIDE(thread->continuation);
211}
212
213void
214kperf_backtrace_sample(struct kp_kcallstack *cs, struct kperf_context *context)
215{
216 assert(cs != NULL);
217 assert(context != NULL);
218 assert(context->cur_thread == current_thread());
219
220 cs->kpkc_flags = CALLSTACK_KERNEL | CALLSTACK_KERNEL_WORDS;
221#ifdef __LP64__
222 cs->kpkc_flags |= CALLSTACK_64BIT;
223#endif
224
225 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, 1);
226
227 backtrace_info_t btinfo = BTI_NONE;
228 struct backtrace_control ctl = {
229 .btc_frame_addr = (uintptr_t)context->starting_fp,
230 };
231 cs->kpkc_nframes = backtrace(bt: cs->kpkc_word_frames, btlen: cs->kpkc_nframes - 1,
232 ctl: &ctl, info_out: &btinfo);
233 if (cs->kpkc_nframes > 0) {
234 cs->kpkc_flags |= CALLSTACK_VALID;
235
236 cs->kpkc_exclaves_offset = 0;
237#if CONFIG_EXCLAVES
238 if ((context->cur_thread->th_exclaves_state & TH_EXCLAVES_RPC) != 0) {
239 cs->kpkc_exclaves_offset = exclaves_stack_offset(cs->kpkc_word_frames, cs->kpkc_nframes, true);
240 }
241#endif /* CONFIG_EXCLAVES */
242
243 /*
244 * Fake the value pointed to by the stack pointer or the link
245 * register for symbolicators.
246 */
247 cs->kpkc_word_frames[cs->kpkc_nframes + 1] = 0;
248 cs->kpkc_nframes += 1;
249 }
250 if ((btinfo & BTI_TRUNCATED)) {
251 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
252 }
253
254 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, cs->kpkc_nframes);
255}
256
257kern_return_t chudxnu_thread_get_callstack64_kperf(thread_t thread,
258 uint64_t *callStack, mach_msg_type_number_t *count,
259 boolean_t user_only);
260
261void
262kperf_kcallstack_sample(struct kp_kcallstack *cs, struct kperf_context *context)
263{
264 thread_t thread;
265
266 assert(cs != NULL);
267 assert(context != NULL);
268 assert(cs->kpkc_nframes <= MAX_KCALLSTACK_FRAMES);
269
270 thread = context->cur_thread;
271 assert(thread != NULL);
272
273 BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread),
274 cs->kpkc_nframes);
275
276 cs->kpkc_flags = CALLSTACK_KERNEL;
277#ifdef __LP64__
278 cs->kpkc_flags |= CALLSTACK_64BIT;
279#endif
280
281 if (ml_at_interrupt_context()) {
282 assert(thread == current_thread());
283 cs->kpkc_flags |= CALLSTACK_KERNEL_WORDS;
284 backtrace_info_t btinfo = BTI_NONE;
285 struct backtrace_control ctl = { .btc_flags = BTF_KERN_INTERRUPTED, };
286 cs->kpkc_nframes = backtrace(bt: cs->kpkc_word_frames, btlen: cs->kpkc_nframes - 1,
287 ctl: &ctl, info_out: &btinfo);
288 if (cs->kpkc_nframes != 0) {
289 callstack_fixup_interrupted(cs);
290 }
291 if ((btinfo & BTI_TRUNCATED)) {
292 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
293 }
294
295 cs->kpkc_exclaves_offset = 0;
296#if CONFIG_EXCLAVES
297 if ((thread->th_exclaves_state & TH_EXCLAVES_RPC) != 0) {
298 cs->kpkc_exclaves_offset = exclaves_stack_offset(cs->kpkc_word_frames, cs->kpkc_nframes, true);
299 }
300#endif /* CONFIG_EXCLAVES */
301 } else {
302 /*
303 * Rely on legacy CHUD backtracer to backtrace kernel stacks on
304 * other threads.
305 */
306 kern_return_t kr;
307 kr = chudxnu_thread_get_callstack64_kperf(thread,
308 callStack: cs->kpkc_frames, count: &cs->kpkc_nframes, FALSE);
309 if (kr == KERN_SUCCESS) {
310 cs->kpkc_flags |= CALLSTACK_VALID;
311 } else if (kr == KERN_RESOURCE_SHORTAGE) {
312 cs->kpkc_flags |= CALLSTACK_VALID;
313 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
314 } else {
315 cs->kpkc_nframes = 0;
316 }
317 }
318
319 if (!(cs->kpkc_flags & CALLSTACK_VALID)) {
320 BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK);
321 }
322
323 BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread),
324 cs->kpkc_flags, cs->kpkc_nframes);
325}
326
327void
328kperf_ucallstack_sample(struct kp_ucallstack *cs, struct kperf_context *context)
329{
330 assert(ml_get_interrupts_enabled() == TRUE);
331
332 thread_t thread = context->cur_thread;
333 assert(thread != NULL);
334
335 BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_START,
336 (uintptr_t)thread_tid(thread), cs->kpuc_nframes);
337
338 struct backtrace_user_info btinfo = BTUINFO_INIT;
339 /*
340 * Leave space for the fixup information.
341 */
342 unsigned int maxnframes = cs->kpuc_nframes - 1;
343 struct backtrace_control ctl = { .btc_user_thread = thread, };
344 unsigned int nframes = backtrace_user(bt: cs->kpuc_frames, btlen: maxnframes, ctl: &ctl,
345 info_out: &btinfo);
346 cs->kpuc_nframes = MIN(maxnframes, nframes);
347
348 cs->kpuc_flags |= CALLSTACK_KERNEL_WORDS |
349 ((btinfo.btui_info & BTI_TRUNCATED) ? CALLSTACK_TRUNCATED : 0) |
350 ((btinfo.btui_info & BTI_64_BIT) ? CALLSTACK_64BIT : 0);
351
352 /*
353 * Ignore EFAULT to get as much of the stack as possible.
354 */
355 if (btinfo.btui_error == 0 || btinfo.btui_error == EFAULT) {
356 callstack_fixup_user(cs, thread);
357 cs->kpuc_flags |= CALLSTACK_VALID;
358
359 if (cs->kpuc_nframes < maxnframes &&
360 btinfo.btui_async_frame_addr != 0) {
361 cs->kpuc_async_index = btinfo.btui_async_start_index;
362 ctl.btc_frame_addr = btinfo.btui_async_frame_addr;
363 ctl.btc_addr_offset = BTCTL_ASYNC_ADDR_OFFSET;
364 maxnframes -= cs->kpuc_nframes;
365 btinfo = BTUINFO_INIT;
366 unsigned int nasync_frames = backtrace_user(
367 bt: &cs->kpuc_frames[cs->kpuc_nframes], btlen: maxnframes, ctl: &ctl, info_out: &btinfo);
368 if (btinfo.btui_info & BTI_TRUNCATED) {
369 cs->kpuc_flags |= CALLSTACK_TRUNCATED;
370 }
371 if (btinfo.btui_error == 0 || btinfo.btui_error == EFAULT) {
372 cs->kpuc_flags |= CALLSTACK_HAS_ASYNC;
373 cs->kpuc_async_nframes = nasync_frames;
374 }
375 }
376 } else {
377 cs->kpuc_nframes = 0;
378 BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK, btinfo.btui_error);
379 }
380
381 BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread),
382 cs->kpuc_flags, cs->kpuc_nframes);
383}
384
385static inline uintptr_t
386scrub_word(uintptr_t *bt, int n_frames, int frame, bool kern)
387{
388 if (frame < n_frames) {
389 if (kern) {
390 return VM_KERNEL_UNSLIDE(bt[frame]);
391 } else {
392 return bt[frame];
393 }
394 } else {
395 return 0;
396 }
397}
398
399static inline uintptr_t
400scrub_frame(uint64_t *bt, int n_frames, int frame)
401{
402 if (frame < n_frames) {
403 return (uintptr_t)(bt[frame]);
404 } else {
405 return 0;
406 }
407}
408
409static void
410callstack_log(uint32_t hdrid, uint32_t dataid, void *vframes,
411 unsigned int nframes, unsigned int flags, unsigned int async_index,
412 unsigned int async_nframes)
413{
414 BUF_VERB(PERF_CS_LOG | DBG_FUNC_START, flags, nframes);
415 BUF_DATA(hdrid, flags, nframes - async_nframes, async_index, async_nframes);
416
417 unsigned int nevts = nframes / 4;
418 unsigned int ovf = nframes % 4;
419 if (ovf != 0) {
420 nevts++;
421 }
422
423 bool kern = flags & CALLSTACK_KERNEL;
424
425 if (flags & CALLSTACK_KERNEL_WORDS) {
426 uintptr_t *frames = vframes;
427 for (unsigned int i = 0; i < nevts; i++) {
428 unsigned int j = i * 4;
429 BUF_DATA(dataid,
430 scrub_word(frames, nframes, j + 0, kern),
431 scrub_word(frames, nframes, j + 1, kern),
432 scrub_word(frames, nframes, j + 2, kern),
433 scrub_word(frames, nframes, j + 3, kern));
434 }
435 } else {
436 for (unsigned int i = 0; i < nevts; i++) {
437 uint64_t *frames = vframes;
438 unsigned int j = i * 4;
439 BUF_DATA(dataid,
440 scrub_frame(frames, nframes, j + 0),
441 scrub_frame(frames, nframes, j + 1),
442 scrub_frame(frames, nframes, j + 2),
443 scrub_frame(frames, nframes, j + 3));
444 }
445 }
446
447 BUF_VERB(PERF_CS_LOG | DBG_FUNC_END, flags, nframes);
448}
449
450void
451kperf_kcallstack_log(struct kp_kcallstack *cs)
452{
453 callstack_log(PERF_CS_KHDR, PERF_CS_KDATA, vframes: cs->kpkc_frames,
454 nframes: cs->kpkc_nframes, flags: cs->kpkc_flags, async_index: 0, async_nframes: 0);
455
456 if (cs->kpkc_exclaves_offset != 0) {
457 BUF_DATA(PERF_CS_KEXOFFSET, cs->kpkc_exclaves_offset);
458 }
459}
460
461void
462kperf_ucallstack_log(struct kp_ucallstack *cs)
463{
464 callstack_log(PERF_CS_UHDR, PERF_CS_UDATA, vframes: cs->kpuc_frames,
465 nframes: cs->kpuc_nframes + cs->kpuc_async_nframes, flags: cs->kpuc_flags,
466 async_index: cs->kpuc_async_index, async_nframes: cs->kpuc_async_nframes);
467}
468
469#if CONFIG_EXCLAVES
470void
471kperf_excallstack_log(const stackshot_ipcstackentry_s *ipcstack)
472{
473 __block unsigned int nframes = 0;
474 __block unsigned int flags = CALLSTACK_VALID;
475 uint64_t frames[MAX_EXCALLSTACK_FRAMES] = {};
476 uint64_t *frames_block = frames;
477
478 BUF_DATA(PERF_CS_EXSTACK, ipcstack->asid);
479
480 if (ipcstack->stacktrace.has_value) {
481 address__v_visit(&ipcstack->stacktrace.value, ^(size_t i, const stackshot_address_s item) {
482 if (i >= MAX_EXCALLSTACK_FRAMES) {
483 flags |= CALLSTACK_TRUNCATED;
484 return;
485 }
486 frames_block[i] = item;
487 nframes += 1;
488 });
489 callstack_log(PERF_CS_EXHDR, PERF_CS_EXDATA, frames, nframes, flags, 0, 0);
490 }
491}
492
493bool
494kperf_exclave_callstack_pend(struct kperf_context *context, unsigned int actionid)
495{
496 if ((context->cur_thread->th_exclaves_state & TH_EXCLAVES_RPC)
497 && (os_atomic_load(&context->cur_thread->th_exclaves_inspection_state, relaxed) & TH_EXCLAVES_INSPECTION_NOINSPECT) == 0) {
498 os_atomic_or(&context->cur_thread->th_exclaves_inspection_state, TH_EXCLAVES_INSPECTION_KPERF, relaxed);
499 context->cur_thread->kperf_exclaves_ast |= T_KPERF_SET_ACTIONID(actionid);
500 return true;
501 }
502 return false;
503}
504#endif /* CONFIG_EXCLAVES */
505
506int
507kperf_ucallstack_pend(struct kperf_context * context, uint32_t depth,
508 unsigned int actionid)
509{
510 if (depth < 2) {
511 panic("HUH");
512 }
513 kperf_ast_set_callstack_depth(thread: context->cur_thread, depth);
514 return kperf_ast_pend(thread: context->cur_thread, T_KPERF_AST_CALLSTACK,
515 actionid);
516}
517
518static kern_return_t
519chudxnu_kern_read(void *dstaddr, vm_offset_t srcaddr, vm_size_t size)
520{
521 return (ml_nofault_copy(virtsrc: srcaddr, virtdst: (vm_offset_t)dstaddr, size) == size) ?
522 KERN_SUCCESS : KERN_FAILURE;
523}
524
525static kern_return_t
526chudxnu_task_read(
527 task_t task,
528 void *kernaddr,
529 uint64_t usraddr,
530 vm_size_t size)
531{
532 //ppc version ported to arm
533 kern_return_t ret = KERN_SUCCESS;
534
535 if (ml_at_interrupt_context()) {
536 return KERN_FAILURE; // can't look at tasks on interrupt stack
537 }
538
539 if (current_task() == task) {
540 if (copyin(usraddr, kernaddr, size)) {
541 ret = KERN_FAILURE;
542 }
543 } else {
544 vm_map_t map = get_task_map(task);
545 ret = vm_map_read_user(map, src_addr: usraddr, dst_p: kernaddr, size);
546 }
547
548 return ret;
549}
550
551static inline uint64_t
552chudxnu_vm_unslide( uint64_t ptr, int kaddr )
553{
554 if (!kaddr) {
555 return ptr;
556 }
557
558 return VM_KERNEL_UNSLIDE(ptr);
559}
560
561#if __arm64__
562
563#if defined(HAS_APPLE_PAC)
564#include <ptrauth.h>
565#endif
566
567// chudxnu_thread_get_callstack gathers a raw callstack along with any information needed to
568// fix it up later (in case we stopped program as it was saving values into prev stack frame, etc.)
569// after sampling has finished.
570//
571// For an N-entry callstack:
572//
573// [0] current pc
574// [1..N-3] stack frames (including current one)
575// [N-2] current LR (return value if we're in a leaf function)
576// [N-1] current r0 (in case we've saved LR in r0) (optional)
577//
578//
579#define CS_FLAG_EXTRASP 1 // capture extra sp register
580
581static kern_return_t
582chudxnu_thread_get_callstack64_internal(
583 thread_t thread,
584 uint64_t *callStack,
585 mach_msg_type_number_t *count,
586 boolean_t user_only,
587 int flags)
588{
589 kern_return_t kr = KERN_SUCCESS;
590 task_t task;
591 uint64_t currPC = 0ULL, currLR = 0ULL, currSP = 0ULL;
592 uint64_t prevPC = 0ULL;
593 uint64_t kernStackMin = thread->kernel_stack;
594 uint64_t kernStackMax = kernStackMin + kernel_stack_size;
595 uint64_t *buffer = callStack;
596 int bufferIndex = 0;
597 int bufferMaxIndex = 0;
598 boolean_t kernel = FALSE;
599 struct arm_saved_state *sstate = NULL;
600 uint64_t pc = 0ULL;
601
602 task = get_threadtask(thread);
603 bufferMaxIndex = *count;
604 //get thread state
605 if (user_only) {
606 sstate = find_user_regs(thread);
607 } else {
608 sstate = find_kern_regs(thread);
609 }
610
611 if (!sstate) {
612 *count = 0;
613 return KERN_FAILURE;
614 }
615
616 if (is_saved_state64(iss: sstate)) {
617 struct arm_saved_state64 *state = NULL;
618 uint64_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
619 uint64_t frame[2];
620
621 state = saved_state64(iss: sstate);
622
623 /* make sure it is safe to dereference before you do it */
624 kernel = PSR64_IS_KERNEL(state->cpsr);
625
626 /* can't take a kernel callstack if we've got a user frame */
627 if (!user_only && !kernel) {
628 return KERN_FAILURE;
629 }
630
631 /*
632 * Reserve space for saving LR (and sometimes SP) at the end of the
633 * backtrace.
634 */
635 if (flags & CS_FLAG_EXTRASP) {
636 bufferMaxIndex -= 2;
637 } else {
638 bufferMaxIndex -= 1;
639 }
640
641 if (bufferMaxIndex < 2) {
642 *count = 0;
643 return KERN_RESOURCE_SHORTAGE;
644 }
645
646 currPC = state->pc;
647 currLR = state->lr;
648 currSP = state->sp;
649
650 fp = (uint64_t *)state->fp; /* frame pointer */
651#if defined(HAS_APPLE_PAC)
652 /* frame pointers on stack will be signed by arm64e ABI */
653 fp = ptrauth_strip(fp, ptrauth_key_frame_pointer);
654#endif
655 topfp = fp;
656
657 bufferIndex = 0; // start with a stack of size zero
658 buffer[bufferIndex++] = chudxnu_vm_unslide(ptr: currPC, kaddr: kernel); // save PC in position 0.
659
660 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 0);
661
662 // Now, fill buffer with stack backtraces.
663 while (bufferIndex < bufferMaxIndex) {
664 pc = 0ULL;
665 /*
666 * Below the frame pointer, the following values are saved:
667 * -> FP
668 */
669
670 /*
671 * Note that we read the pc even for the first stack frame
672 * (which, in theory, is always empty because the callee fills
673 * it in just before it lowers the stack. However, if we
674 * catch the program in between filling in the return address
675 * and lowering the stack, we want to still have a valid
676 * backtrace. FixupStack correctly disregards this value if
677 * necessary.
678 */
679
680 if ((uint64_t)fp == 0 || ((uint64_t)fp & 0x3) != 0) {
681 /* frame pointer is invalid - stop backtracing */
682 pc = 0ULL;
683 break;
684 }
685
686 if (kernel) {
687 if (((uint64_t)fp > kernStackMax) ||
688 ((uint64_t)fp < kernStackMin)) {
689 kr = KERN_FAILURE;
690 } else {
691 kr = chudxnu_kern_read(dstaddr: &frame,
692 srcaddr: (vm_offset_t)fp,
693 size: (vm_size_t)sizeof(frame));
694 if (kr == KERN_SUCCESS) {
695#if defined(HAS_APPLE_PAC)
696 /* return addresses on stack will be signed by arm64e ABI */
697 pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address);
698#else
699 pc = frame[1];
700#endif
701 nextFramePointer = (uint64_t *)frame[0];
702#if defined(HAS_APPLE_PAC)
703 /* frame pointers on stack will be signed by arm64e ABI */
704 nextFramePointer = ptrauth_strip(nextFramePointer, ptrauth_key_frame_pointer);
705#endif
706 } else {
707 pc = 0ULL;
708 nextFramePointer = 0ULL;
709 kr = KERN_FAILURE;
710 }
711 }
712 } else {
713 kr = chudxnu_task_read(task,
714 kernaddr: &frame,
715 usraddr: (vm_offset_t)fp,
716 size: (vm_size_t)sizeof(frame));
717 if (kr == KERN_SUCCESS) {
718#if defined(HAS_APPLE_PAC)
719 /* return addresses on stack will be signed by arm64e ABI */
720 pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address);
721#else
722 pc = frame[1];
723#endif
724 nextFramePointer = (uint64_t *)(frame[0]);
725#if defined(HAS_APPLE_PAC)
726 /* frame pointers on stack will be signed by arm64e ABI */
727 nextFramePointer = ptrauth_strip(nextFramePointer, ptrauth_key_frame_pointer);
728#endif
729 } else {
730 pc = 0ULL;
731 nextFramePointer = 0ULL;
732 kr = KERN_FAILURE;
733 }
734 }
735
736 if (kr != KERN_SUCCESS) {
737 pc = 0ULL;
738 break;
739 }
740
741 if (nextFramePointer) {
742 buffer[bufferIndex++] = chudxnu_vm_unslide(ptr: pc, kaddr: kernel);
743 prevPC = pc;
744 }
745
746 if (nextFramePointer < fp) {
747 break;
748 } else {
749 fp = nextFramePointer;
750 }
751 }
752
753 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex);
754
755 if (bufferIndex >= bufferMaxIndex) {
756 bufferIndex = bufferMaxIndex;
757 kr = KERN_RESOURCE_SHORTAGE;
758 } else {
759 kr = KERN_SUCCESS;
760 }
761
762 // Save link register and SP at bottom of stack (used for later fixup).
763 buffer[bufferIndex++] = chudxnu_vm_unslide(ptr: currLR, kaddr: kernel);
764 if (flags & CS_FLAG_EXTRASP) {
765 buffer[bufferIndex++] = chudxnu_vm_unslide(ptr: currSP, kaddr: kernel);
766 }
767 } else {
768 struct arm_saved_state32 *state = NULL;
769 uint32_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
770
771 /* 64-bit kernel stacks, 32-bit user stacks */
772 uint64_t frame[2];
773 uint32_t frame32[2];
774
775 state = saved_state32(iss: sstate);
776
777 /* make sure it is safe to dereference before you do it */
778 kernel = PSR_IS_KERNEL(state->cpsr);
779
780 /* can't take a kernel callstack if we've got a user frame */
781 if (!user_only && !kernel) {
782 return KERN_FAILURE;
783 }
784
785 /*
786 * Reserve space for saving LR (and sometimes SP) at the end of the
787 * backtrace.
788 */
789 if (flags & CS_FLAG_EXTRASP) {
790 bufferMaxIndex -= 2;
791 } else {
792 bufferMaxIndex -= 1;
793 }
794
795 if (bufferMaxIndex < 2) {
796 *count = 0;
797 return KERN_RESOURCE_SHORTAGE;
798 }
799
800 currPC = (uint64_t)state->pc; /* r15 */
801 if (state->cpsr & PSR_TF) {
802 currPC |= 1ULL; /* encode thumb mode into low bit of PC */
803 }
804 currLR = (uint64_t)state->lr; /* r14 */
805 currSP = (uint64_t)state->sp; /* r13 */
806
807 fp = (uint32_t *)(uintptr_t)state->r[7]; /* frame pointer */
808 topfp = fp;
809
810 bufferIndex = 0; // start with a stack of size zero
811 buffer[bufferIndex++] = chudxnu_vm_unslide(ptr: currPC, kaddr: kernel); // save PC in position 0.
812
813 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 1);
814
815 // Now, fill buffer with stack backtraces.
816 while (bufferIndex < bufferMaxIndex) {
817 pc = 0ULL;
818 /*
819 * Below the frame pointer, the following values are saved:
820 * -> FP
821 */
822
823 /*
824 * Note that we read the pc even for the first stack frame
825 * (which, in theory, is always empty because the callee fills
826 * it in just before it lowers the stack. However, if we
827 * catch the program in between filling in the return address
828 * and lowering the stack, we want to still have a valid
829 * backtrace. FixupStack correctly disregards this value if
830 * necessary.
831 */
832
833 if ((uint32_t)fp == 0 || ((uint32_t)fp & 0x3) != 0) {
834 /* frame pointer is invalid - stop backtracing */
835 pc = 0ULL;
836 break;
837 }
838
839 if (kernel) {
840 if (((uint32_t)fp > kernStackMax) ||
841 ((uint32_t)fp < kernStackMin)) {
842 kr = KERN_FAILURE;
843 } else {
844 kr = chudxnu_kern_read(dstaddr: &frame,
845 srcaddr: (vm_offset_t)fp,
846 size: (vm_size_t)sizeof(frame));
847 if (kr == KERN_SUCCESS) {
848 pc = (uint64_t)frame[1];
849 nextFramePointer = (uint32_t *) (frame[0]);
850 } else {
851 pc = 0ULL;
852 nextFramePointer = 0ULL;
853 kr = KERN_FAILURE;
854 }
855 }
856 } else {
857 kr = chudxnu_task_read(task,
858 kernaddr: &frame32,
859 usraddr: (((uint64_t)(uint32_t)fp) & 0x00000000FFFFFFFFULL),
860 size: sizeof(frame32));
861 if (kr == KERN_SUCCESS) {
862 pc = (uint64_t)frame32[1];
863 nextFramePointer = (uint32_t *)(uintptr_t)(frame32[0]);
864 } else {
865 pc = 0ULL;
866 nextFramePointer = 0ULL;
867 kr = KERN_FAILURE;
868 }
869 }
870
871 if (kr != KERN_SUCCESS) {
872 pc = 0ULL;
873 break;
874 }
875
876 if (nextFramePointer) {
877 buffer[bufferIndex++] = chudxnu_vm_unslide(ptr: pc, kaddr: kernel);
878 prevPC = pc;
879 }
880
881 if (nextFramePointer < fp) {
882 break;
883 } else {
884 fp = nextFramePointer;
885 }
886 }
887
888 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex);
889
890 /* clamp callstack size to max */
891 if (bufferIndex >= bufferMaxIndex) {
892 bufferIndex = bufferMaxIndex;
893 kr = KERN_RESOURCE_SHORTAGE;
894 } else {
895 /* ignore all other failures */
896 kr = KERN_SUCCESS;
897 }
898
899 // Save link register and R13 (sp) at bottom of stack (used for later fixup).
900 buffer[bufferIndex++] = chudxnu_vm_unslide(ptr: currLR, kaddr: kernel);
901 if (flags & CS_FLAG_EXTRASP) {
902 buffer[bufferIndex++] = chudxnu_vm_unslide(ptr: currSP, kaddr: kernel);
903 }
904 }
905
906 *count = bufferIndex;
907 return kr;
908}
909
910kern_return_t
911chudxnu_thread_get_callstack64_kperf(
912 thread_t thread,
913 uint64_t *callStack,
914 mach_msg_type_number_t *count,
915 boolean_t user_only)
916{
917 return chudxnu_thread_get_callstack64_internal( thread, callStack, count, user_only, flags: 0 );
918}
919#elif __x86_64__
920
921#define VALID_STACK_ADDRESS(supervisor, addr, minKernAddr, maxKernAddr) (supervisor ? (addr>=minKernAddr && addr<=maxKernAddr) : TRUE)
922// don't try to read in the hole
923#define VALID_STACK_ADDRESS64(supervisor, addr, minKernAddr, maxKernAddr) \
924(supervisor ? ((uint64_t)addr >= minKernAddr && (uint64_t)addr <= maxKernAddr) : \
925((uint64_t)addr != 0ULL && ((uint64_t)addr <= 0x00007FFFFFFFFFFFULL || (uint64_t)addr >= 0xFFFF800000000000ULL)))
926
927typedef struct _cframe64_t {
928 uint64_t prevFP; // can't use a real pointer here until we're a 64 bit kernel
929 uint64_t caller;
930 uint64_t args[0];
931}cframe64_t;
932
933
934typedef struct _cframe_t {
935 uint32_t prev; // this is really a user32-space pointer to the previous frame
936 uint32_t caller;
937 uint32_t args[0];
938} cframe_t;
939
940extern void * find_user_regs(thread_t);
941extern x86_saved_state32_t *find_kern_regs(thread_t);
942
943static kern_return_t
944do_kernel_backtrace(
945 thread_t thread,
946 struct x86_kernel_state *regs,
947 uint64_t *frames,
948 mach_msg_type_number_t *start_idx,
949 mach_msg_type_number_t max_idx)
950{
951 uint64_t kernStackMin = (uint64_t)thread->kernel_stack;
952 uint64_t kernStackMax = (uint64_t)kernStackMin + kernel_stack_size;
953 mach_msg_type_number_t ct = *start_idx;
954 kern_return_t kr = KERN_FAILURE;
955
956#if __LP64__
957 uint64_t currPC = 0ULL;
958 uint64_t currFP = 0ULL;
959 uint64_t prevPC = 0ULL;
960 uint64_t prevFP = 0ULL;
961 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(regs->k_rip), sizeof(uint64_t))) {
962 return KERN_FAILURE;
963 }
964 if (KERN_SUCCESS != chudxnu_kern_read(&currFP, (vm_offset_t)&(regs->k_rbp), sizeof(uint64_t))) {
965 return KERN_FAILURE;
966 }
967#else
968 uint32_t currPC = 0U;
969 uint32_t currFP = 0U;
970 uint32_t prevPC = 0U;
971 uint32_t prevFP = 0U;
972 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(regs->k_eip), sizeof(uint32_t))) {
973 return KERN_FAILURE;
974 }
975 if (KERN_SUCCESS != chudxnu_kern_read(&currFP, (vm_offset_t)&(regs->k_ebp), sizeof(uint32_t))) {
976 return KERN_FAILURE;
977 }
978#endif
979
980 if (*start_idx >= max_idx) {
981 return KERN_RESOURCE_SHORTAGE; // no frames traced
982 }
983 if (!currPC) {
984 return KERN_FAILURE;
985 }
986
987 frames[ct++] = chudxnu_vm_unslide((uint64_t)currPC, 1);
988
989 // build a backtrace of this kernel state
990#if __LP64__
991 while (VALID_STACK_ADDRESS64(TRUE, currFP, kernStackMin, kernStackMax)) {
992 // this is the address where caller lives in the user thread
993 uint64_t caller = currFP + sizeof(uint64_t);
994#else
995 while (VALID_STACK_ADDRESS(TRUE, currFP, kernStackMin, kernStackMax)) {
996 uint32_t caller = (uint32_t)currFP + sizeof(uint32_t);
997#endif
998
999 if (!currFP || !currPC) {
1000 currPC = 0;
1001 break;
1002 }
1003
1004 if (ct >= max_idx) {
1005 *start_idx = ct;
1006 return KERN_RESOURCE_SHORTAGE;
1007 }
1008
1009 /* read our caller */
1010 kr = chudxnu_kern_read(&currPC, (vm_offset_t)caller, sizeof(currPC));
1011
1012 if (kr != KERN_SUCCESS || !currPC) {
1013 currPC = 0UL;
1014 break;
1015 }
1016
1017 /*
1018 * retrive contents of the frame pointer and advance to the next stack
1019 * frame if it's valid
1020 */
1021 prevFP = 0;
1022 kr = chudxnu_kern_read(&prevFP, (vm_offset_t)currFP, sizeof(currPC));
1023
1024#if __LP64__
1025 if (VALID_STACK_ADDRESS64(TRUE, prevFP, kernStackMin, kernStackMax)) {
1026#else
1027 if (VALID_STACK_ADDRESS(TRUE, prevFP, kernStackMin, kernStackMax)) {
1028#endif
1029 frames[ct++] = chudxnu_vm_unslide((uint64_t)currPC, 1);
1030 prevPC = currPC;
1031 }
1032 if (prevFP <= currFP) {
1033 break;
1034 } else {
1035 currFP = prevFP;
1036 }
1037 }
1038
1039 *start_idx = ct;
1040 return KERN_SUCCESS;
1041}
1042
1043
1044
1045static kern_return_t
1046do_backtrace32(
1047 task_t task,
1048 thread_t thread,
1049 x86_saved_state32_t *regs,
1050 uint64_t *frames,
1051 mach_msg_type_number_t *start_idx,
1052 mach_msg_type_number_t max_idx,
1053 boolean_t supervisor)
1054{
1055 uint32_t tmpWord = 0UL;
1056 uint64_t currPC = (uint64_t) regs->eip;
1057 uint64_t currFP = (uint64_t) regs->ebp;
1058 uint64_t prevPC = 0ULL;
1059 uint64_t prevFP = 0ULL;
1060 uint64_t kernStackMin = thread->kernel_stack;
1061 uint64_t kernStackMax = kernStackMin + kernel_stack_size;
1062 mach_msg_type_number_t ct = *start_idx;
1063 kern_return_t kr = KERN_FAILURE;
1064
1065 if (ct >= max_idx) {
1066 return KERN_RESOURCE_SHORTAGE; // no frames traced
1067 }
1068 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1069
1070 // build a backtrace of this 32 bit state.
1071 while (VALID_STACK_ADDRESS(supervisor, currFP, kernStackMin, kernStackMax)) {
1072 cframe_t *fp = (cframe_t *) (uintptr_t) currFP;
1073
1074 if (!currFP) {
1075 currPC = 0;
1076 break;
1077 }
1078
1079 if (ct >= max_idx) {
1080 *start_idx = ct;
1081 return KERN_RESOURCE_SHORTAGE;
1082 }
1083
1084 /* read our caller */
1085 if (supervisor) {
1086 kr = chudxnu_kern_read(&tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));
1087 } else {
1088 kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));
1089 }
1090
1091 if (kr != KERN_SUCCESS) {
1092 currPC = 0ULL;
1093 break;
1094 }
1095
1096 currPC = (uint64_t) tmpWord; // promote 32 bit address
1097
1098 /*
1099 * retrive contents of the frame pointer and advance to the next stack
1100 * frame if it's valid
1101 */
1102 prevFP = 0;
1103 if (supervisor) {
1104 kr = chudxnu_kern_read(&tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
1105 } else {
1106 kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
1107 }
1108 prevFP = (uint64_t) tmpWord; // promote 32 bit address
1109
1110 if (prevFP) {
1111 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1112 prevPC = currPC;
1113 }
1114 if (prevFP < currFP) {
1115 break;
1116 } else {
1117 currFP = prevFP;
1118 }
1119 }
1120
1121 *start_idx = ct;
1122 return KERN_SUCCESS;
1123}
1124
1125static kern_return_t
1126do_backtrace64(
1127 task_t task,
1128 thread_t thread,
1129 x86_saved_state64_t *regs,
1130 uint64_t *frames,
1131 mach_msg_type_number_t *start_idx,
1132 mach_msg_type_number_t max_idx,
1133 boolean_t supervisor)
1134{
1135 uint64_t currPC = regs->isf.rip;
1136 uint64_t currFP = regs->rbp;
1137 uint64_t prevPC = 0ULL;
1138 uint64_t prevFP = 0ULL;
1139 uint64_t kernStackMin = (uint64_t)thread->kernel_stack;
1140 uint64_t kernStackMax = (uint64_t)kernStackMin + kernel_stack_size;
1141 mach_msg_type_number_t ct = *start_idx;
1142 kern_return_t kr = KERN_FAILURE;
1143
1144 if (*start_idx >= max_idx) {
1145 return KERN_RESOURCE_SHORTAGE; // no frames traced
1146 }
1147 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1148
1149 // build a backtrace of this 32 bit state.
1150 while (VALID_STACK_ADDRESS64(supervisor, currFP, kernStackMin, kernStackMax)) {
1151 // this is the address where caller lives in the user thread
1152 uint64_t caller = currFP + sizeof(uint64_t);
1153
1154 if (!currFP) {
1155 currPC = 0;
1156 break;
1157 }
1158
1159 if (ct >= max_idx) {
1160 *start_idx = ct;
1161 return KERN_RESOURCE_SHORTAGE;
1162 }
1163
1164 /* read our caller */
1165 if (supervisor) {
1166 kr = chudxnu_kern_read(&currPC, (vm_offset_t)caller, sizeof(uint64_t));
1167 } else {
1168 kr = chudxnu_task_read(task, &currPC, caller, sizeof(uint64_t));
1169 }
1170
1171 if (kr != KERN_SUCCESS) {
1172 currPC = 0ULL;
1173 break;
1174 }
1175
1176 /*
1177 * retrive contents of the frame pointer and advance to the next stack
1178 * frame if it's valid
1179 */
1180 prevFP = 0;
1181 if (supervisor) {
1182 kr = chudxnu_kern_read(&prevFP, (vm_offset_t)currFP, sizeof(uint64_t));
1183 } else {
1184 kr = chudxnu_task_read(task, &prevFP, currFP, sizeof(uint64_t));
1185 }
1186
1187 if (VALID_STACK_ADDRESS64(supervisor, prevFP, kernStackMin, kernStackMax)) {
1188 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1189 prevPC = currPC;
1190 }
1191 if (prevFP < currFP) {
1192 break;
1193 } else {
1194 currFP = prevFP;
1195 }
1196 }
1197
1198 *start_idx = ct;
1199 return KERN_SUCCESS;
1200}
1201
1202static kern_return_t
1203chudxnu_thread_get_callstack64_internal(
1204 thread_t thread,
1205 uint64_t *callstack,
1206 mach_msg_type_number_t *count,
1207 boolean_t user_only,
1208 boolean_t kern_only)
1209{
1210 kern_return_t kr = KERN_FAILURE;
1211 task_t task = get_threadtask(thread);
1212 uint64_t currPC = 0ULL;
1213 boolean_t supervisor = FALSE;
1214 mach_msg_type_number_t bufferIndex = 0;
1215 mach_msg_type_number_t bufferMaxIndex = *count;
1216 x86_saved_state_t *tagged_regs = NULL; // kernel register state
1217 x86_saved_state64_t *regs64 = NULL;
1218 x86_saved_state32_t *regs32 = NULL;
1219 x86_saved_state32_t *u_regs32 = NULL;
1220 x86_saved_state64_t *u_regs64 = NULL;
1221 struct x86_kernel_state *kregs = NULL;
1222
1223 if (ml_at_interrupt_context()) {
1224 if (user_only) {
1225 /* can't backtrace user state on interrupt stack. */
1226 return KERN_FAILURE;
1227 }
1228
1229 /* backtracing at interrupt context? */
1230 if (thread == current_thread() && current_cpu_datap()->cpu_int_state) {
1231 /*
1232 * Locate the registers for the interrupted thread, assuming it is
1233 * current_thread().
1234 */
1235 tagged_regs = current_cpu_datap()->cpu_int_state;
1236
1237 if (is_saved_state64(tagged_regs)) {
1238 /* 64 bit registers */
1239 regs64 = saved_state64(tagged_regs);
1240 supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U);
1241 } else {
1242 /* 32 bit registers */
1243 regs32 = saved_state32(tagged_regs);
1244 supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U);
1245 }
1246 }
1247 }
1248
1249 if (!ml_at_interrupt_context() && kernel_task == task) {
1250 if (!thread->kernel_stack) {
1251 return KERN_FAILURE;
1252 }
1253
1254 // Kernel thread not at interrupt context
1255 kregs = (struct x86_kernel_state *)NULL;
1256
1257 // nofault read of the thread->kernel_stack pointer
1258 if (KERN_SUCCESS != chudxnu_kern_read(&kregs, (vm_offset_t)&(thread->kernel_stack), sizeof(void *))) {
1259 return KERN_FAILURE;
1260 }
1261
1262 // Adjust to find the saved kernel state
1263 kregs = STACK_IKS((vm_offset_t)(uintptr_t)kregs);
1264
1265 supervisor = TRUE;
1266 } else if (!tagged_regs) {
1267 /*
1268 * not at interrupt context, or tracing a different thread than
1269 * current_thread() at interrupt context
1270 */
1271 tagged_regs = USER_STATE(thread);
1272 if (is_saved_state64(tagged_regs)) {
1273 /* 64 bit registers */
1274 regs64 = saved_state64(tagged_regs);
1275 supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U);
1276 } else {
1277 /* 32 bit registers */
1278 regs32 = saved_state32(tagged_regs);
1279 supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U);
1280 }
1281 }
1282
1283 *count = 0;
1284
1285 if (supervisor) {
1286 // the caller only wants a user callstack.
1287 if (user_only) {
1288 // bail - we've only got kernel state
1289 return KERN_FAILURE;
1290 }
1291 } else {
1292 // regs32(64) is not in supervisor mode.
1293 u_regs32 = regs32;
1294 u_regs64 = regs64;
1295 regs32 = NULL;
1296 regs64 = NULL;
1297 }
1298
1299 if (user_only) {
1300 /* we only want to backtrace the user mode */
1301 if (!(u_regs32 || u_regs64)) {
1302 /* no user state to look at */
1303 return KERN_FAILURE;
1304 }
1305 }
1306
1307 /*
1308 * Order of preference for top of stack:
1309 * 64 bit kernel state (not likely)
1310 * 32 bit kernel state
1311 * 64 bit user land state
1312 * 32 bit user land state
1313 */
1314
1315 if (kregs) {
1316 /*
1317 * nofault read of the registers from the kernel stack (as they can
1318 * disappear on the fly).
1319 */
1320
1321 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(kregs->k_rip), sizeof(uint64_t))) {
1322 return KERN_FAILURE;
1323 }
1324 } else if (regs64) {
1325 currPC = regs64->isf.rip;
1326 } else if (regs32) {
1327 currPC = (uint64_t) regs32->eip;
1328 } else if (u_regs64) {
1329 currPC = u_regs64->isf.rip;
1330 } else if (u_regs32) {
1331 currPC = (uint64_t) u_regs32->eip;
1332 }
1333
1334 if (!currPC) {
1335 /* no top of the stack, bail out */
1336 return KERN_FAILURE;
1337 }
1338
1339 bufferIndex = 0;
1340
1341 if (bufferMaxIndex < 1) {
1342 *count = 0;
1343 return KERN_RESOURCE_SHORTAGE;
1344 }
1345
1346 /* backtrace kernel */
1347 if (kregs) {
1348 addr64_t address = 0ULL;
1349 size_t size = 0UL;
1350
1351 // do the backtrace
1352 kr = do_kernel_backtrace(thread, kregs, callstack, &bufferIndex, bufferMaxIndex);
1353
1354 // and do a nofault read of (r|e)sp
1355 uint64_t rsp = 0ULL;
1356 size = sizeof(uint64_t);
1357
1358 if (KERN_SUCCESS != chudxnu_kern_read(&address, (vm_offset_t)&(kregs->k_rsp), size)) {
1359 address = 0ULL;
1360 }
1361
1362 if (address && KERN_SUCCESS == chudxnu_kern_read(&rsp, (vm_offset_t)address, size) && bufferIndex < bufferMaxIndex) {
1363 callstack[bufferIndex++] = (uint64_t)rsp;
1364 }
1365 } else if (regs64) {
1366 uint64_t rsp = 0ULL;
1367
1368 // backtrace the 64bit side.
1369 kr = do_backtrace64(task, thread, regs64, callstack, &bufferIndex,
1370 bufferMaxIndex - 1, TRUE);
1371
1372 if (KERN_SUCCESS == chudxnu_kern_read(&rsp, (vm_offset_t) regs64->isf.rsp, sizeof(uint64_t)) &&
1373 bufferIndex < bufferMaxIndex) {
1374 callstack[bufferIndex++] = rsp;
1375 }
1376 } else if (regs32) {
1377 uint32_t esp = 0UL;
1378
1379 // backtrace the 32bit side.
1380 kr = do_backtrace32(task, thread, regs32, callstack, &bufferIndex,
1381 bufferMaxIndex - 1, TRUE);
1382
1383 if (KERN_SUCCESS == chudxnu_kern_read(&esp, (vm_offset_t) regs32->uesp, sizeof(uint32_t)) &&
1384 bufferIndex < bufferMaxIndex) {
1385 callstack[bufferIndex++] = (uint64_t) esp;
1386 }
1387 } else if (u_regs64 && !kern_only) {
1388 /* backtrace user land */
1389 uint64_t rsp = 0ULL;
1390
1391 kr = do_backtrace64(task, thread, u_regs64, callstack, &bufferIndex,
1392 bufferMaxIndex - 1, FALSE);
1393
1394 if (KERN_SUCCESS == chudxnu_task_read(task, &rsp, (addr64_t) u_regs64->isf.rsp, sizeof(uint64_t)) &&
1395 bufferIndex < bufferMaxIndex) {
1396 callstack[bufferIndex++] = rsp;
1397 }
1398 } else if (u_regs32 && !kern_only) {
1399 uint32_t esp = 0UL;
1400
1401 kr = do_backtrace32(task, thread, u_regs32, callstack, &bufferIndex,
1402 bufferMaxIndex - 1, FALSE);
1403
1404 if (KERN_SUCCESS == chudxnu_task_read(task, &esp, (addr64_t) u_regs32->uesp, sizeof(uint32_t)) &&
1405 bufferIndex < bufferMaxIndex) {
1406 callstack[bufferIndex++] = (uint64_t) esp;
1407 }
1408 }
1409
1410 *count = bufferIndex;
1411 return kr;
1412}
1413
1414__private_extern__
1415kern_return_t
1416chudxnu_thread_get_callstack64_kperf(
1417 thread_t thread,
1418 uint64_t *callstack,
1419 mach_msg_type_number_t *count,
1420 boolean_t is_user)
1421{
1422 return chudxnu_thread_get_callstack64_internal(thread, callstack, count, is_user, !is_user);
1423}
1424#else /* !__arm64__ && !__x86_64__ */
1425#error kperf: unsupported architecture
1426#endif /* !__arm64__ && !__x86_64__ */
1427