1 | /* |
2 | * Copyright (c) 2011-2022 Apple Computer, Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | /* Collect kernel callstacks */ |
30 | |
31 | #include <mach/mach_types.h> |
32 | #include <kern/thread.h> |
33 | #include <kern/backtrace.h> |
34 | #include <kern/cambria_layout.h> |
35 | #include <vm/vm_map.h> |
36 | #include <kperf/buffer.h> |
37 | #include <kperf/context.h> |
38 | #include <kperf/callstack.h> |
39 | #include <kperf/ast.h> |
40 | #include <sys/errno.h> |
41 | #include <mach/exclaves.h> |
42 | |
43 | #if defined(__arm64__) |
44 | #include <arm/cpu_data.h> |
45 | #include <arm/cpu_data_internal.h> |
46 | #endif |
47 | |
48 | static void |
49 | callstack_fixup_user(struct kp_ucallstack *cs, thread_t thread) |
50 | { |
51 | uint64_t fixup_val = 0; |
52 | assert(cs->kpuc_nframes < MAX_UCALLSTACK_FRAMES); |
53 | |
54 | #if defined(__x86_64__) |
55 | user_addr_t sp_user; |
56 | bool user_64; |
57 | x86_saved_state_t *state; |
58 | |
59 | state = get_user_regs(thread); |
60 | if (!state) { |
61 | goto out; |
62 | } |
63 | |
64 | user_64 = is_saved_state64(state); |
65 | if (user_64) { |
66 | sp_user = saved_state64(state)->isf.rsp; |
67 | } else { |
68 | sp_user = saved_state32(state)->uesp; |
69 | } |
70 | |
71 | if (thread == current_thread()) { |
72 | (void)copyin(sp_user, (char *)&fixup_val, |
73 | user_64 ? sizeof(uint64_t) : sizeof(uint32_t)); |
74 | } else { |
75 | (void)vm_map_read_user(get_task_map(get_threadtask(thread)), sp_user, |
76 | &fixup_val, user_64 ? sizeof(uint64_t) : sizeof(uint32_t)); |
77 | } |
78 | |
79 | #elif defined(__arm64__) |
80 | |
81 | struct arm_saved_state *state = get_user_regs(thread); |
82 | if (!state) { |
83 | goto out; |
84 | } |
85 | |
86 | /* encode thumb mode into low bit of PC */ |
87 | if (is_saved_state32(iss: state) && (get_saved_state_cpsr(iss: state) & PSR_TF)) { |
88 | cs->kpuc_frames[0] |= 1ULL; |
89 | } |
90 | |
91 | |
92 | fixup_val = get_saved_state_lr(iss: state); |
93 | |
94 | #else |
95 | #error "callstack_fixup_user: unsupported architecture" |
96 | #endif |
97 | |
98 | out: |
99 | cs->kpuc_frames[cs->kpuc_nframes++] = fixup_val; |
100 | } |
101 | |
102 | #if defined(__x86_64__) |
103 | |
104 | __attribute__((used)) |
105 | static kern_return_t |
106 | interrupted_kernel_sp_value(uintptr_t *sp_val) |
107 | { |
108 | x86_saved_state_t *state; |
109 | uintptr_t sp; |
110 | bool state_64; |
111 | uint64_t cs; |
112 | uintptr_t top, bottom; |
113 | |
114 | state = current_cpu_datap()->cpu_int_state; |
115 | if (!state) { |
116 | return KERN_FAILURE; |
117 | } |
118 | |
119 | state_64 = is_saved_state64(state); |
120 | |
121 | if (state_64) { |
122 | cs = saved_state64(state)->isf.cs; |
123 | } else { |
124 | cs = saved_state32(state)->cs; |
125 | } |
126 | /* return early if interrupted a thread in user space */ |
127 | if ((cs & SEL_PL) == SEL_PL_U) { |
128 | return KERN_FAILURE; |
129 | } |
130 | |
131 | if (state_64) { |
132 | sp = saved_state64(state)->isf.rsp; |
133 | } else { |
134 | sp = saved_state32(state)->uesp; |
135 | } |
136 | |
137 | /* make sure the stack pointer is pointing somewhere in this stack */ |
138 | bottom = current_thread()->kernel_stack; |
139 | top = bottom + kernel_stack_size; |
140 | if (sp >= bottom && sp < top) { |
141 | return KERN_FAILURE; |
142 | } |
143 | |
144 | *sp_val = *(uintptr_t *)sp; |
145 | return KERN_SUCCESS; |
146 | } |
147 | |
148 | #elif defined(__arm64__) |
149 | |
150 | __attribute__((used)) |
151 | static kern_return_t |
152 | interrupted_kernel_lr(uintptr_t *lr) |
153 | { |
154 | struct arm_saved_state *state; |
155 | |
156 | state = getCpuDatap()->cpu_int_state; |
157 | |
158 | /* return early if interrupted a thread in user space */ |
159 | if (PSR64_IS_USER(get_saved_state_cpsr(state))) { |
160 | return KERN_FAILURE; |
161 | } |
162 | |
163 | *lr = get_saved_state_lr(iss: state); |
164 | return KERN_SUCCESS; |
165 | } |
166 | #else /* defined(__arm64__) */ |
167 | #error "interrupted_kernel_{sp,lr}: unsupported architecture" |
168 | #endif /* !defined(__arm64__) */ |
169 | |
170 | |
171 | static void |
172 | callstack_fixup_interrupted(struct kp_kcallstack *cs) |
173 | { |
174 | uintptr_t fixup_val = 0; |
175 | assert(cs->kpkc_nframes < MAX_KCALLSTACK_FRAMES); |
176 | |
177 | /* |
178 | * Only provide arbitrary data on development or debug kernels. |
179 | */ |
180 | #if DEVELOPMENT || DEBUG |
181 | #if defined(__x86_64__) |
182 | (void)interrupted_kernel_sp_value(&fixup_val); |
183 | #elif defined(__arm64__) |
184 | (void)interrupted_kernel_lr(&fixup_val); |
185 | #endif /* defined(__x86_64__) */ |
186 | #endif /* DEVELOPMENT || DEBUG */ |
187 | |
188 | assert(cs->kpkc_flags & CALLSTACK_KERNEL); |
189 | cs->kpkc_frames[cs->kpkc_nframes++] = fixup_val; |
190 | } |
191 | |
192 | void |
193 | kperf_continuation_sample(struct kp_kcallstack *cs, struct kperf_context *context) |
194 | { |
195 | thread_t thread; |
196 | |
197 | assert(cs != NULL); |
198 | assert(context != NULL); |
199 | |
200 | thread = context->cur_thread; |
201 | assert(thread != NULL); |
202 | assert(thread->continuation != NULL); |
203 | |
204 | cs->kpkc_flags = CALLSTACK_CONTINUATION | CALLSTACK_VALID | CALLSTACK_KERNEL; |
205 | #ifdef __LP64__ |
206 | cs->kpkc_flags |= CALLSTACK_64BIT; |
207 | #endif |
208 | |
209 | cs->kpkc_nframes = 1; |
210 | cs->kpkc_frames[0] = VM_KERNEL_UNSLIDE(thread->continuation); |
211 | } |
212 | |
213 | void |
214 | kperf_backtrace_sample(struct kp_kcallstack *cs, struct kperf_context *context) |
215 | { |
216 | assert(cs != NULL); |
217 | assert(context != NULL); |
218 | assert(context->cur_thread == current_thread()); |
219 | |
220 | cs->kpkc_flags = CALLSTACK_KERNEL | CALLSTACK_KERNEL_WORDS; |
221 | #ifdef __LP64__ |
222 | cs->kpkc_flags |= CALLSTACK_64BIT; |
223 | #endif |
224 | |
225 | BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, 1); |
226 | |
227 | backtrace_info_t btinfo = BTI_NONE; |
228 | struct backtrace_control ctl = { |
229 | .btc_frame_addr = (uintptr_t)context->starting_fp, |
230 | }; |
231 | cs->kpkc_nframes = backtrace(bt: cs->kpkc_word_frames, btlen: cs->kpkc_nframes - 1, |
232 | ctl: &ctl, info_out: &btinfo); |
233 | if (cs->kpkc_nframes > 0) { |
234 | cs->kpkc_flags |= CALLSTACK_VALID; |
235 | |
236 | cs->kpkc_exclaves_offset = 0; |
237 | #if CONFIG_EXCLAVES |
238 | if ((context->cur_thread->th_exclaves_state & TH_EXCLAVES_RPC) != 0) { |
239 | cs->kpkc_exclaves_offset = exclaves_stack_offset(cs->kpkc_word_frames, cs->kpkc_nframes, true); |
240 | } |
241 | #endif /* CONFIG_EXCLAVES */ |
242 | |
243 | /* |
244 | * Fake the value pointed to by the stack pointer or the link |
245 | * register for symbolicators. |
246 | */ |
247 | cs->kpkc_word_frames[cs->kpkc_nframes + 1] = 0; |
248 | cs->kpkc_nframes += 1; |
249 | } |
250 | if ((btinfo & BTI_TRUNCATED)) { |
251 | cs->kpkc_flags |= CALLSTACK_TRUNCATED; |
252 | } |
253 | |
254 | BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, cs->kpkc_nframes); |
255 | } |
256 | |
257 | kern_return_t chudxnu_thread_get_callstack64_kperf(thread_t thread, |
258 | uint64_t *callStack, mach_msg_type_number_t *count, |
259 | boolean_t user_only); |
260 | |
261 | void |
262 | kperf_kcallstack_sample(struct kp_kcallstack *cs, struct kperf_context *context) |
263 | { |
264 | thread_t thread; |
265 | |
266 | assert(cs != NULL); |
267 | assert(context != NULL); |
268 | assert(cs->kpkc_nframes <= MAX_KCALLSTACK_FRAMES); |
269 | |
270 | thread = context->cur_thread; |
271 | assert(thread != NULL); |
272 | |
273 | BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread), |
274 | cs->kpkc_nframes); |
275 | |
276 | cs->kpkc_flags = CALLSTACK_KERNEL; |
277 | #ifdef __LP64__ |
278 | cs->kpkc_flags |= CALLSTACK_64BIT; |
279 | #endif |
280 | |
281 | if (ml_at_interrupt_context()) { |
282 | assert(thread == current_thread()); |
283 | cs->kpkc_flags |= CALLSTACK_KERNEL_WORDS; |
284 | backtrace_info_t btinfo = BTI_NONE; |
285 | struct backtrace_control ctl = { .btc_flags = BTF_KERN_INTERRUPTED, }; |
286 | cs->kpkc_nframes = backtrace(bt: cs->kpkc_word_frames, btlen: cs->kpkc_nframes - 1, |
287 | ctl: &ctl, info_out: &btinfo); |
288 | if (cs->kpkc_nframes != 0) { |
289 | callstack_fixup_interrupted(cs); |
290 | } |
291 | if ((btinfo & BTI_TRUNCATED)) { |
292 | cs->kpkc_flags |= CALLSTACK_TRUNCATED; |
293 | } |
294 | |
295 | cs->kpkc_exclaves_offset = 0; |
296 | #if CONFIG_EXCLAVES |
297 | if ((thread->th_exclaves_state & TH_EXCLAVES_RPC) != 0) { |
298 | cs->kpkc_exclaves_offset = exclaves_stack_offset(cs->kpkc_word_frames, cs->kpkc_nframes, true); |
299 | } |
300 | #endif /* CONFIG_EXCLAVES */ |
301 | } else { |
302 | /* |
303 | * Rely on legacy CHUD backtracer to backtrace kernel stacks on |
304 | * other threads. |
305 | */ |
306 | kern_return_t kr; |
307 | kr = chudxnu_thread_get_callstack64_kperf(thread, |
308 | callStack: cs->kpkc_frames, count: &cs->kpkc_nframes, FALSE); |
309 | if (kr == KERN_SUCCESS) { |
310 | cs->kpkc_flags |= CALLSTACK_VALID; |
311 | } else if (kr == KERN_RESOURCE_SHORTAGE) { |
312 | cs->kpkc_flags |= CALLSTACK_VALID; |
313 | cs->kpkc_flags |= CALLSTACK_TRUNCATED; |
314 | } else { |
315 | cs->kpkc_nframes = 0; |
316 | } |
317 | } |
318 | |
319 | if (!(cs->kpkc_flags & CALLSTACK_VALID)) { |
320 | BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK); |
321 | } |
322 | |
323 | BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread), |
324 | cs->kpkc_flags, cs->kpkc_nframes); |
325 | } |
326 | |
327 | void |
328 | kperf_ucallstack_sample(struct kp_ucallstack *cs, struct kperf_context *context) |
329 | { |
330 | assert(ml_get_interrupts_enabled() == TRUE); |
331 | |
332 | thread_t thread = context->cur_thread; |
333 | assert(thread != NULL); |
334 | |
335 | BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_START, |
336 | (uintptr_t)thread_tid(thread), cs->kpuc_nframes); |
337 | |
338 | struct backtrace_user_info btinfo = BTUINFO_INIT; |
339 | /* |
340 | * Leave space for the fixup information. |
341 | */ |
342 | unsigned int maxnframes = cs->kpuc_nframes - 1; |
343 | struct backtrace_control ctl = { .btc_user_thread = thread, }; |
344 | unsigned int nframes = backtrace_user(bt: cs->kpuc_frames, btlen: maxnframes, ctl: &ctl, |
345 | info_out: &btinfo); |
346 | cs->kpuc_nframes = MIN(maxnframes, nframes); |
347 | |
348 | cs->kpuc_flags |= CALLSTACK_KERNEL_WORDS | |
349 | ((btinfo.btui_info & BTI_TRUNCATED) ? CALLSTACK_TRUNCATED : 0) | |
350 | ((btinfo.btui_info & BTI_64_BIT) ? CALLSTACK_64BIT : 0); |
351 | |
352 | /* |
353 | * Ignore EFAULT to get as much of the stack as possible. |
354 | */ |
355 | if (btinfo.btui_error == 0 || btinfo.btui_error == EFAULT) { |
356 | callstack_fixup_user(cs, thread); |
357 | cs->kpuc_flags |= CALLSTACK_VALID; |
358 | |
359 | if (cs->kpuc_nframes < maxnframes && |
360 | btinfo.btui_async_frame_addr != 0) { |
361 | cs->kpuc_async_index = btinfo.btui_async_start_index; |
362 | ctl.btc_frame_addr = btinfo.btui_async_frame_addr; |
363 | ctl.btc_addr_offset = BTCTL_ASYNC_ADDR_OFFSET; |
364 | maxnframes -= cs->kpuc_nframes; |
365 | btinfo = BTUINFO_INIT; |
366 | unsigned int nasync_frames = backtrace_user( |
367 | bt: &cs->kpuc_frames[cs->kpuc_nframes], btlen: maxnframes, ctl: &ctl, info_out: &btinfo); |
368 | if (btinfo.btui_info & BTI_TRUNCATED) { |
369 | cs->kpuc_flags |= CALLSTACK_TRUNCATED; |
370 | } |
371 | if (btinfo.btui_error == 0 || btinfo.btui_error == EFAULT) { |
372 | cs->kpuc_flags |= CALLSTACK_HAS_ASYNC; |
373 | cs->kpuc_async_nframes = nasync_frames; |
374 | } |
375 | } |
376 | } else { |
377 | cs->kpuc_nframes = 0; |
378 | BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK, btinfo.btui_error); |
379 | } |
380 | |
381 | BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread), |
382 | cs->kpuc_flags, cs->kpuc_nframes); |
383 | } |
384 | |
385 | static inline uintptr_t |
386 | scrub_word(uintptr_t *bt, int n_frames, int frame, bool kern) |
387 | { |
388 | if (frame < n_frames) { |
389 | if (kern) { |
390 | return VM_KERNEL_UNSLIDE(bt[frame]); |
391 | } else { |
392 | return bt[frame]; |
393 | } |
394 | } else { |
395 | return 0; |
396 | } |
397 | } |
398 | |
399 | static inline uintptr_t |
400 | scrub_frame(uint64_t *bt, int n_frames, int frame) |
401 | { |
402 | if (frame < n_frames) { |
403 | return (uintptr_t)(bt[frame]); |
404 | } else { |
405 | return 0; |
406 | } |
407 | } |
408 | |
409 | static void |
410 | callstack_log(uint32_t hdrid, uint32_t dataid, void *vframes, |
411 | unsigned int nframes, unsigned int flags, unsigned int async_index, |
412 | unsigned int async_nframes) |
413 | { |
414 | BUF_VERB(PERF_CS_LOG | DBG_FUNC_START, flags, nframes); |
415 | BUF_DATA(hdrid, flags, nframes - async_nframes, async_index, async_nframes); |
416 | |
417 | unsigned int nevts = nframes / 4; |
418 | unsigned int ovf = nframes % 4; |
419 | if (ovf != 0) { |
420 | nevts++; |
421 | } |
422 | |
423 | bool kern = flags & CALLSTACK_KERNEL; |
424 | |
425 | if (flags & CALLSTACK_KERNEL_WORDS) { |
426 | uintptr_t *frames = vframes; |
427 | for (unsigned int i = 0; i < nevts; i++) { |
428 | unsigned int j = i * 4; |
429 | BUF_DATA(dataid, |
430 | scrub_word(frames, nframes, j + 0, kern), |
431 | scrub_word(frames, nframes, j + 1, kern), |
432 | scrub_word(frames, nframes, j + 2, kern), |
433 | scrub_word(frames, nframes, j + 3, kern)); |
434 | } |
435 | } else { |
436 | for (unsigned int i = 0; i < nevts; i++) { |
437 | uint64_t *frames = vframes; |
438 | unsigned int j = i * 4; |
439 | BUF_DATA(dataid, |
440 | scrub_frame(frames, nframes, j + 0), |
441 | scrub_frame(frames, nframes, j + 1), |
442 | scrub_frame(frames, nframes, j + 2), |
443 | scrub_frame(frames, nframes, j + 3)); |
444 | } |
445 | } |
446 | |
447 | BUF_VERB(PERF_CS_LOG | DBG_FUNC_END, flags, nframes); |
448 | } |
449 | |
450 | void |
451 | kperf_kcallstack_log(struct kp_kcallstack *cs) |
452 | { |
453 | callstack_log(PERF_CS_KHDR, PERF_CS_KDATA, vframes: cs->kpkc_frames, |
454 | nframes: cs->kpkc_nframes, flags: cs->kpkc_flags, async_index: 0, async_nframes: 0); |
455 | |
456 | if (cs->kpkc_exclaves_offset != 0) { |
457 | BUF_DATA(PERF_CS_KEXOFFSET, cs->kpkc_exclaves_offset); |
458 | } |
459 | } |
460 | |
461 | void |
462 | kperf_ucallstack_log(struct kp_ucallstack *cs) |
463 | { |
464 | callstack_log(PERF_CS_UHDR, PERF_CS_UDATA, vframes: cs->kpuc_frames, |
465 | nframes: cs->kpuc_nframes + cs->kpuc_async_nframes, flags: cs->kpuc_flags, |
466 | async_index: cs->kpuc_async_index, async_nframes: cs->kpuc_async_nframes); |
467 | } |
468 | |
469 | #if CONFIG_EXCLAVES |
470 | void |
471 | kperf_excallstack_log(const stackshot_ipcstackentry_s *ipcstack) |
472 | { |
473 | __block unsigned int nframes = 0; |
474 | __block unsigned int flags = CALLSTACK_VALID; |
475 | uint64_t frames[MAX_EXCALLSTACK_FRAMES] = {}; |
476 | uint64_t *frames_block = frames; |
477 | |
478 | BUF_DATA(PERF_CS_EXSTACK, ipcstack->asid); |
479 | |
480 | if (ipcstack->stacktrace.has_value) { |
481 | address__v_visit(&ipcstack->stacktrace.value, ^(size_t i, const stackshot_address_s item) { |
482 | if (i >= MAX_EXCALLSTACK_FRAMES) { |
483 | flags |= CALLSTACK_TRUNCATED; |
484 | return; |
485 | } |
486 | frames_block[i] = item; |
487 | nframes += 1; |
488 | }); |
489 | callstack_log(PERF_CS_EXHDR, PERF_CS_EXDATA, frames, nframes, flags, 0, 0); |
490 | } |
491 | } |
492 | |
493 | bool |
494 | kperf_exclave_callstack_pend(struct kperf_context *context, unsigned int actionid) |
495 | { |
496 | if ((context->cur_thread->th_exclaves_state & TH_EXCLAVES_RPC) |
497 | && (os_atomic_load(&context->cur_thread->th_exclaves_inspection_state, relaxed) & TH_EXCLAVES_INSPECTION_NOINSPECT) == 0) { |
498 | os_atomic_or(&context->cur_thread->th_exclaves_inspection_state, TH_EXCLAVES_INSPECTION_KPERF, relaxed); |
499 | context->cur_thread->kperf_exclaves_ast |= T_KPERF_SET_ACTIONID(actionid); |
500 | return true; |
501 | } |
502 | return false; |
503 | } |
504 | #endif /* CONFIG_EXCLAVES */ |
505 | |
506 | int |
507 | kperf_ucallstack_pend(struct kperf_context * context, uint32_t depth, |
508 | unsigned int actionid) |
509 | { |
510 | if (depth < 2) { |
511 | panic("HUH" ); |
512 | } |
513 | kperf_ast_set_callstack_depth(thread: context->cur_thread, depth); |
514 | return kperf_ast_pend(thread: context->cur_thread, T_KPERF_AST_CALLSTACK, |
515 | actionid); |
516 | } |
517 | |
518 | static kern_return_t |
519 | chudxnu_kern_read(void *dstaddr, vm_offset_t srcaddr, vm_size_t size) |
520 | { |
521 | return (ml_nofault_copy(virtsrc: srcaddr, virtdst: (vm_offset_t)dstaddr, size) == size) ? |
522 | KERN_SUCCESS : KERN_FAILURE; |
523 | } |
524 | |
525 | static kern_return_t |
526 | chudxnu_task_read( |
527 | task_t task, |
528 | void *kernaddr, |
529 | uint64_t usraddr, |
530 | vm_size_t size) |
531 | { |
532 | //ppc version ported to arm |
533 | kern_return_t ret = KERN_SUCCESS; |
534 | |
535 | if (ml_at_interrupt_context()) { |
536 | return KERN_FAILURE; // can't look at tasks on interrupt stack |
537 | } |
538 | |
539 | if (current_task() == task) { |
540 | if (copyin(usraddr, kernaddr, size)) { |
541 | ret = KERN_FAILURE; |
542 | } |
543 | } else { |
544 | vm_map_t map = get_task_map(task); |
545 | ret = vm_map_read_user(map, src_addr: usraddr, dst_p: kernaddr, size); |
546 | } |
547 | |
548 | return ret; |
549 | } |
550 | |
551 | static inline uint64_t |
552 | chudxnu_vm_unslide( uint64_t ptr, int kaddr ) |
553 | { |
554 | if (!kaddr) { |
555 | return ptr; |
556 | } |
557 | |
558 | return VM_KERNEL_UNSLIDE(ptr); |
559 | } |
560 | |
561 | #if __arm64__ |
562 | |
563 | #if defined(HAS_APPLE_PAC) |
564 | #include <ptrauth.h> |
565 | #endif |
566 | |
567 | // chudxnu_thread_get_callstack gathers a raw callstack along with any information needed to |
568 | // fix it up later (in case we stopped program as it was saving values into prev stack frame, etc.) |
569 | // after sampling has finished. |
570 | // |
571 | // For an N-entry callstack: |
572 | // |
573 | // [0] current pc |
574 | // [1..N-3] stack frames (including current one) |
575 | // [N-2] current LR (return value if we're in a leaf function) |
576 | // [N-1] current r0 (in case we've saved LR in r0) (optional) |
577 | // |
578 | // |
579 | #define 1 // capture extra sp register |
580 | |
581 | static kern_return_t |
582 | chudxnu_thread_get_callstack64_internal( |
583 | thread_t thread, |
584 | uint64_t *callStack, |
585 | mach_msg_type_number_t *count, |
586 | boolean_t user_only, |
587 | int flags) |
588 | { |
589 | kern_return_t kr = KERN_SUCCESS; |
590 | task_t task; |
591 | uint64_t currPC = 0ULL, currLR = 0ULL, currSP = 0ULL; |
592 | uint64_t prevPC = 0ULL; |
593 | uint64_t kernStackMin = thread->kernel_stack; |
594 | uint64_t kernStackMax = kernStackMin + kernel_stack_size; |
595 | uint64_t *buffer = callStack; |
596 | int bufferIndex = 0; |
597 | int bufferMaxIndex = 0; |
598 | boolean_t kernel = FALSE; |
599 | struct arm_saved_state *sstate = NULL; |
600 | uint64_t pc = 0ULL; |
601 | |
602 | task = get_threadtask(thread); |
603 | bufferMaxIndex = *count; |
604 | //get thread state |
605 | if (user_only) { |
606 | sstate = find_user_regs(thread); |
607 | } else { |
608 | sstate = find_kern_regs(thread); |
609 | } |
610 | |
611 | if (!sstate) { |
612 | *count = 0; |
613 | return KERN_FAILURE; |
614 | } |
615 | |
616 | if (is_saved_state64(iss: sstate)) { |
617 | struct arm_saved_state64 *state = NULL; |
618 | uint64_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL; |
619 | uint64_t frame[2]; |
620 | |
621 | state = saved_state64(iss: sstate); |
622 | |
623 | /* make sure it is safe to dereference before you do it */ |
624 | kernel = PSR64_IS_KERNEL(state->cpsr); |
625 | |
626 | /* can't take a kernel callstack if we've got a user frame */ |
627 | if (!user_only && !kernel) { |
628 | return KERN_FAILURE; |
629 | } |
630 | |
631 | /* |
632 | * Reserve space for saving LR (and sometimes SP) at the end of the |
633 | * backtrace. |
634 | */ |
635 | if (flags & CS_FLAG_EXTRASP) { |
636 | bufferMaxIndex -= 2; |
637 | } else { |
638 | bufferMaxIndex -= 1; |
639 | } |
640 | |
641 | if (bufferMaxIndex < 2) { |
642 | *count = 0; |
643 | return KERN_RESOURCE_SHORTAGE; |
644 | } |
645 | |
646 | currPC = state->pc; |
647 | currLR = state->lr; |
648 | currSP = state->sp; |
649 | |
650 | fp = (uint64_t *)state->fp; /* frame pointer */ |
651 | #if defined(HAS_APPLE_PAC) |
652 | /* frame pointers on stack will be signed by arm64e ABI */ |
653 | fp = ptrauth_strip(fp, ptrauth_key_frame_pointer); |
654 | #endif |
655 | topfp = fp; |
656 | |
657 | bufferIndex = 0; // start with a stack of size zero |
658 | buffer[bufferIndex++] = chudxnu_vm_unslide(ptr: currPC, kaddr: kernel); // save PC in position 0. |
659 | |
660 | BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 0); |
661 | |
662 | // Now, fill buffer with stack backtraces. |
663 | while (bufferIndex < bufferMaxIndex) { |
664 | pc = 0ULL; |
665 | /* |
666 | * Below the frame pointer, the following values are saved: |
667 | * -> FP |
668 | */ |
669 | |
670 | /* |
671 | * Note that we read the pc even for the first stack frame |
672 | * (which, in theory, is always empty because the callee fills |
673 | * it in just before it lowers the stack. However, if we |
674 | * catch the program in between filling in the return address |
675 | * and lowering the stack, we want to still have a valid |
676 | * backtrace. FixupStack correctly disregards this value if |
677 | * necessary. |
678 | */ |
679 | |
680 | if ((uint64_t)fp == 0 || ((uint64_t)fp & 0x3) != 0) { |
681 | /* frame pointer is invalid - stop backtracing */ |
682 | pc = 0ULL; |
683 | break; |
684 | } |
685 | |
686 | if (kernel) { |
687 | if (((uint64_t)fp > kernStackMax) || |
688 | ((uint64_t)fp < kernStackMin)) { |
689 | kr = KERN_FAILURE; |
690 | } else { |
691 | kr = chudxnu_kern_read(dstaddr: &frame, |
692 | srcaddr: (vm_offset_t)fp, |
693 | size: (vm_size_t)sizeof(frame)); |
694 | if (kr == KERN_SUCCESS) { |
695 | #if defined(HAS_APPLE_PAC) |
696 | /* return addresses on stack will be signed by arm64e ABI */ |
697 | pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address); |
698 | #else |
699 | pc = frame[1]; |
700 | #endif |
701 | nextFramePointer = (uint64_t *)frame[0]; |
702 | #if defined(HAS_APPLE_PAC) |
703 | /* frame pointers on stack will be signed by arm64e ABI */ |
704 | nextFramePointer = ptrauth_strip(nextFramePointer, ptrauth_key_frame_pointer); |
705 | #endif |
706 | } else { |
707 | pc = 0ULL; |
708 | nextFramePointer = 0ULL; |
709 | kr = KERN_FAILURE; |
710 | } |
711 | } |
712 | } else { |
713 | kr = chudxnu_task_read(task, |
714 | kernaddr: &frame, |
715 | usraddr: (vm_offset_t)fp, |
716 | size: (vm_size_t)sizeof(frame)); |
717 | if (kr == KERN_SUCCESS) { |
718 | #if defined(HAS_APPLE_PAC) |
719 | /* return addresses on stack will be signed by arm64e ABI */ |
720 | pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address); |
721 | #else |
722 | pc = frame[1]; |
723 | #endif |
724 | nextFramePointer = (uint64_t *)(frame[0]); |
725 | #if defined(HAS_APPLE_PAC) |
726 | /* frame pointers on stack will be signed by arm64e ABI */ |
727 | nextFramePointer = ptrauth_strip(nextFramePointer, ptrauth_key_frame_pointer); |
728 | #endif |
729 | } else { |
730 | pc = 0ULL; |
731 | nextFramePointer = 0ULL; |
732 | kr = KERN_FAILURE; |
733 | } |
734 | } |
735 | |
736 | if (kr != KERN_SUCCESS) { |
737 | pc = 0ULL; |
738 | break; |
739 | } |
740 | |
741 | if (nextFramePointer) { |
742 | buffer[bufferIndex++] = chudxnu_vm_unslide(ptr: pc, kaddr: kernel); |
743 | prevPC = pc; |
744 | } |
745 | |
746 | if (nextFramePointer < fp) { |
747 | break; |
748 | } else { |
749 | fp = nextFramePointer; |
750 | } |
751 | } |
752 | |
753 | BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex); |
754 | |
755 | if (bufferIndex >= bufferMaxIndex) { |
756 | bufferIndex = bufferMaxIndex; |
757 | kr = KERN_RESOURCE_SHORTAGE; |
758 | } else { |
759 | kr = KERN_SUCCESS; |
760 | } |
761 | |
762 | // Save link register and SP at bottom of stack (used for later fixup). |
763 | buffer[bufferIndex++] = chudxnu_vm_unslide(ptr: currLR, kaddr: kernel); |
764 | if (flags & CS_FLAG_EXTRASP) { |
765 | buffer[bufferIndex++] = chudxnu_vm_unslide(ptr: currSP, kaddr: kernel); |
766 | } |
767 | } else { |
768 | struct arm_saved_state32 *state = NULL; |
769 | uint32_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL; |
770 | |
771 | /* 64-bit kernel stacks, 32-bit user stacks */ |
772 | uint64_t frame[2]; |
773 | uint32_t frame32[2]; |
774 | |
775 | state = saved_state32(iss: sstate); |
776 | |
777 | /* make sure it is safe to dereference before you do it */ |
778 | kernel = PSR_IS_KERNEL(state->cpsr); |
779 | |
780 | /* can't take a kernel callstack if we've got a user frame */ |
781 | if (!user_only && !kernel) { |
782 | return KERN_FAILURE; |
783 | } |
784 | |
785 | /* |
786 | * Reserve space for saving LR (and sometimes SP) at the end of the |
787 | * backtrace. |
788 | */ |
789 | if (flags & CS_FLAG_EXTRASP) { |
790 | bufferMaxIndex -= 2; |
791 | } else { |
792 | bufferMaxIndex -= 1; |
793 | } |
794 | |
795 | if (bufferMaxIndex < 2) { |
796 | *count = 0; |
797 | return KERN_RESOURCE_SHORTAGE; |
798 | } |
799 | |
800 | currPC = (uint64_t)state->pc; /* r15 */ |
801 | if (state->cpsr & PSR_TF) { |
802 | currPC |= 1ULL; /* encode thumb mode into low bit of PC */ |
803 | } |
804 | currLR = (uint64_t)state->lr; /* r14 */ |
805 | currSP = (uint64_t)state->sp; /* r13 */ |
806 | |
807 | fp = (uint32_t *)(uintptr_t)state->r[7]; /* frame pointer */ |
808 | topfp = fp; |
809 | |
810 | bufferIndex = 0; // start with a stack of size zero |
811 | buffer[bufferIndex++] = chudxnu_vm_unslide(ptr: currPC, kaddr: kernel); // save PC in position 0. |
812 | |
813 | BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 1); |
814 | |
815 | // Now, fill buffer with stack backtraces. |
816 | while (bufferIndex < bufferMaxIndex) { |
817 | pc = 0ULL; |
818 | /* |
819 | * Below the frame pointer, the following values are saved: |
820 | * -> FP |
821 | */ |
822 | |
823 | /* |
824 | * Note that we read the pc even for the first stack frame |
825 | * (which, in theory, is always empty because the callee fills |
826 | * it in just before it lowers the stack. However, if we |
827 | * catch the program in between filling in the return address |
828 | * and lowering the stack, we want to still have a valid |
829 | * backtrace. FixupStack correctly disregards this value if |
830 | * necessary. |
831 | */ |
832 | |
833 | if ((uint32_t)fp == 0 || ((uint32_t)fp & 0x3) != 0) { |
834 | /* frame pointer is invalid - stop backtracing */ |
835 | pc = 0ULL; |
836 | break; |
837 | } |
838 | |
839 | if (kernel) { |
840 | if (((uint32_t)fp > kernStackMax) || |
841 | ((uint32_t)fp < kernStackMin)) { |
842 | kr = KERN_FAILURE; |
843 | } else { |
844 | kr = chudxnu_kern_read(dstaddr: &frame, |
845 | srcaddr: (vm_offset_t)fp, |
846 | size: (vm_size_t)sizeof(frame)); |
847 | if (kr == KERN_SUCCESS) { |
848 | pc = (uint64_t)frame[1]; |
849 | nextFramePointer = (uint32_t *) (frame[0]); |
850 | } else { |
851 | pc = 0ULL; |
852 | nextFramePointer = 0ULL; |
853 | kr = KERN_FAILURE; |
854 | } |
855 | } |
856 | } else { |
857 | kr = chudxnu_task_read(task, |
858 | kernaddr: &frame32, |
859 | usraddr: (((uint64_t)(uint32_t)fp) & 0x00000000FFFFFFFFULL), |
860 | size: sizeof(frame32)); |
861 | if (kr == KERN_SUCCESS) { |
862 | pc = (uint64_t)frame32[1]; |
863 | nextFramePointer = (uint32_t *)(uintptr_t)(frame32[0]); |
864 | } else { |
865 | pc = 0ULL; |
866 | nextFramePointer = 0ULL; |
867 | kr = KERN_FAILURE; |
868 | } |
869 | } |
870 | |
871 | if (kr != KERN_SUCCESS) { |
872 | pc = 0ULL; |
873 | break; |
874 | } |
875 | |
876 | if (nextFramePointer) { |
877 | buffer[bufferIndex++] = chudxnu_vm_unslide(ptr: pc, kaddr: kernel); |
878 | prevPC = pc; |
879 | } |
880 | |
881 | if (nextFramePointer < fp) { |
882 | break; |
883 | } else { |
884 | fp = nextFramePointer; |
885 | } |
886 | } |
887 | |
888 | BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex); |
889 | |
890 | /* clamp callstack size to max */ |
891 | if (bufferIndex >= bufferMaxIndex) { |
892 | bufferIndex = bufferMaxIndex; |
893 | kr = KERN_RESOURCE_SHORTAGE; |
894 | } else { |
895 | /* ignore all other failures */ |
896 | kr = KERN_SUCCESS; |
897 | } |
898 | |
899 | // Save link register and R13 (sp) at bottom of stack (used for later fixup). |
900 | buffer[bufferIndex++] = chudxnu_vm_unslide(ptr: currLR, kaddr: kernel); |
901 | if (flags & CS_FLAG_EXTRASP) { |
902 | buffer[bufferIndex++] = chudxnu_vm_unslide(ptr: currSP, kaddr: kernel); |
903 | } |
904 | } |
905 | |
906 | *count = bufferIndex; |
907 | return kr; |
908 | } |
909 | |
910 | kern_return_t |
911 | chudxnu_thread_get_callstack64_kperf( |
912 | thread_t thread, |
913 | uint64_t *callStack, |
914 | mach_msg_type_number_t *count, |
915 | boolean_t user_only) |
916 | { |
917 | return chudxnu_thread_get_callstack64_internal( thread, callStack, count, user_only, flags: 0 ); |
918 | } |
919 | #elif __x86_64__ |
920 | |
921 | #define VALID_STACK_ADDRESS(supervisor, addr, minKernAddr, maxKernAddr) (supervisor ? (addr>=minKernAddr && addr<=maxKernAddr) : TRUE) |
922 | // don't try to read in the hole |
923 | #define VALID_STACK_ADDRESS64(supervisor, addr, minKernAddr, maxKernAddr) \ |
924 | (supervisor ? ((uint64_t)addr >= minKernAddr && (uint64_t)addr <= maxKernAddr) : \ |
925 | ((uint64_t)addr != 0ULL && ((uint64_t)addr <= 0x00007FFFFFFFFFFFULL || (uint64_t)addr >= 0xFFFF800000000000ULL))) |
926 | |
927 | typedef struct _cframe64_t { |
928 | uint64_t prevFP; // can't use a real pointer here until we're a 64 bit kernel |
929 | uint64_t caller; |
930 | uint64_t args[0]; |
931 | }cframe64_t; |
932 | |
933 | |
934 | typedef struct _cframe_t { |
935 | uint32_t prev; // this is really a user32-space pointer to the previous frame |
936 | uint32_t caller; |
937 | uint32_t args[0]; |
938 | } cframe_t; |
939 | |
940 | extern void * find_user_regs(thread_t); |
941 | extern x86_saved_state32_t *find_kern_regs(thread_t); |
942 | |
943 | static kern_return_t |
944 | do_kernel_backtrace( |
945 | thread_t thread, |
946 | struct x86_kernel_state *regs, |
947 | uint64_t *frames, |
948 | mach_msg_type_number_t *start_idx, |
949 | mach_msg_type_number_t max_idx) |
950 | { |
951 | uint64_t kernStackMin = (uint64_t)thread->kernel_stack; |
952 | uint64_t kernStackMax = (uint64_t)kernStackMin + kernel_stack_size; |
953 | mach_msg_type_number_t ct = *start_idx; |
954 | kern_return_t kr = KERN_FAILURE; |
955 | |
956 | #if __LP64__ |
957 | uint64_t currPC = 0ULL; |
958 | uint64_t currFP = 0ULL; |
959 | uint64_t prevPC = 0ULL; |
960 | uint64_t prevFP = 0ULL; |
961 | if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(regs->k_rip), sizeof(uint64_t))) { |
962 | return KERN_FAILURE; |
963 | } |
964 | if (KERN_SUCCESS != chudxnu_kern_read(&currFP, (vm_offset_t)&(regs->k_rbp), sizeof(uint64_t))) { |
965 | return KERN_FAILURE; |
966 | } |
967 | #else |
968 | uint32_t currPC = 0U; |
969 | uint32_t currFP = 0U; |
970 | uint32_t prevPC = 0U; |
971 | uint32_t prevFP = 0U; |
972 | if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(regs->k_eip), sizeof(uint32_t))) { |
973 | return KERN_FAILURE; |
974 | } |
975 | if (KERN_SUCCESS != chudxnu_kern_read(&currFP, (vm_offset_t)&(regs->k_ebp), sizeof(uint32_t))) { |
976 | return KERN_FAILURE; |
977 | } |
978 | #endif |
979 | |
980 | if (*start_idx >= max_idx) { |
981 | return KERN_RESOURCE_SHORTAGE; // no frames traced |
982 | } |
983 | if (!currPC) { |
984 | return KERN_FAILURE; |
985 | } |
986 | |
987 | frames[ct++] = chudxnu_vm_unslide((uint64_t)currPC, 1); |
988 | |
989 | // build a backtrace of this kernel state |
990 | #if __LP64__ |
991 | while (VALID_STACK_ADDRESS64(TRUE, currFP, kernStackMin, kernStackMax)) { |
992 | // this is the address where caller lives in the user thread |
993 | uint64_t caller = currFP + sizeof(uint64_t); |
994 | #else |
995 | while (VALID_STACK_ADDRESS(TRUE, currFP, kernStackMin, kernStackMax)) { |
996 | uint32_t caller = (uint32_t)currFP + sizeof(uint32_t); |
997 | #endif |
998 | |
999 | if (!currFP || !currPC) { |
1000 | currPC = 0; |
1001 | break; |
1002 | } |
1003 | |
1004 | if (ct >= max_idx) { |
1005 | *start_idx = ct; |
1006 | return KERN_RESOURCE_SHORTAGE; |
1007 | } |
1008 | |
1009 | /* read our caller */ |
1010 | kr = chudxnu_kern_read(&currPC, (vm_offset_t)caller, sizeof(currPC)); |
1011 | |
1012 | if (kr != KERN_SUCCESS || !currPC) { |
1013 | currPC = 0UL; |
1014 | break; |
1015 | } |
1016 | |
1017 | /* |
1018 | * retrive contents of the frame pointer and advance to the next stack |
1019 | * frame if it's valid |
1020 | */ |
1021 | prevFP = 0; |
1022 | kr = chudxnu_kern_read(&prevFP, (vm_offset_t)currFP, sizeof(currPC)); |
1023 | |
1024 | #if __LP64__ |
1025 | if (VALID_STACK_ADDRESS64(TRUE, prevFP, kernStackMin, kernStackMax)) { |
1026 | #else |
1027 | if (VALID_STACK_ADDRESS(TRUE, prevFP, kernStackMin, kernStackMax)) { |
1028 | #endif |
1029 | frames[ct++] = chudxnu_vm_unslide((uint64_t)currPC, 1); |
1030 | prevPC = currPC; |
1031 | } |
1032 | if (prevFP <= currFP) { |
1033 | break; |
1034 | } else { |
1035 | currFP = prevFP; |
1036 | } |
1037 | } |
1038 | |
1039 | *start_idx = ct; |
1040 | return KERN_SUCCESS; |
1041 | } |
1042 | |
1043 | |
1044 | |
1045 | static kern_return_t |
1046 | do_backtrace32( |
1047 | task_t task, |
1048 | thread_t thread, |
1049 | x86_saved_state32_t *regs, |
1050 | uint64_t *frames, |
1051 | mach_msg_type_number_t *start_idx, |
1052 | mach_msg_type_number_t max_idx, |
1053 | boolean_t supervisor) |
1054 | { |
1055 | uint32_t tmpWord = 0UL; |
1056 | uint64_t currPC = (uint64_t) regs->eip; |
1057 | uint64_t currFP = (uint64_t) regs->ebp; |
1058 | uint64_t prevPC = 0ULL; |
1059 | uint64_t prevFP = 0ULL; |
1060 | uint64_t kernStackMin = thread->kernel_stack; |
1061 | uint64_t kernStackMax = kernStackMin + kernel_stack_size; |
1062 | mach_msg_type_number_t ct = *start_idx; |
1063 | kern_return_t kr = KERN_FAILURE; |
1064 | |
1065 | if (ct >= max_idx) { |
1066 | return KERN_RESOURCE_SHORTAGE; // no frames traced |
1067 | } |
1068 | frames[ct++] = chudxnu_vm_unslide(currPC, supervisor); |
1069 | |
1070 | // build a backtrace of this 32 bit state. |
1071 | while (VALID_STACK_ADDRESS(supervisor, currFP, kernStackMin, kernStackMax)) { |
1072 | cframe_t *fp = (cframe_t *) (uintptr_t) currFP; |
1073 | |
1074 | if (!currFP) { |
1075 | currPC = 0; |
1076 | break; |
1077 | } |
1078 | |
1079 | if (ct >= max_idx) { |
1080 | *start_idx = ct; |
1081 | return KERN_RESOURCE_SHORTAGE; |
1082 | } |
1083 | |
1084 | /* read our caller */ |
1085 | if (supervisor) { |
1086 | kr = chudxnu_kern_read(&tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t)); |
1087 | } else { |
1088 | kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t)); |
1089 | } |
1090 | |
1091 | if (kr != KERN_SUCCESS) { |
1092 | currPC = 0ULL; |
1093 | break; |
1094 | } |
1095 | |
1096 | currPC = (uint64_t) tmpWord; // promote 32 bit address |
1097 | |
1098 | /* |
1099 | * retrive contents of the frame pointer and advance to the next stack |
1100 | * frame if it's valid |
1101 | */ |
1102 | prevFP = 0; |
1103 | if (supervisor) { |
1104 | kr = chudxnu_kern_read(&tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t)); |
1105 | } else { |
1106 | kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t)); |
1107 | } |
1108 | prevFP = (uint64_t) tmpWord; // promote 32 bit address |
1109 | |
1110 | if (prevFP) { |
1111 | frames[ct++] = chudxnu_vm_unslide(currPC, supervisor); |
1112 | prevPC = currPC; |
1113 | } |
1114 | if (prevFP < currFP) { |
1115 | break; |
1116 | } else { |
1117 | currFP = prevFP; |
1118 | } |
1119 | } |
1120 | |
1121 | *start_idx = ct; |
1122 | return KERN_SUCCESS; |
1123 | } |
1124 | |
1125 | static kern_return_t |
1126 | do_backtrace64( |
1127 | task_t task, |
1128 | thread_t thread, |
1129 | x86_saved_state64_t *regs, |
1130 | uint64_t *frames, |
1131 | mach_msg_type_number_t *start_idx, |
1132 | mach_msg_type_number_t max_idx, |
1133 | boolean_t supervisor) |
1134 | { |
1135 | uint64_t currPC = regs->isf.rip; |
1136 | uint64_t currFP = regs->rbp; |
1137 | uint64_t prevPC = 0ULL; |
1138 | uint64_t prevFP = 0ULL; |
1139 | uint64_t kernStackMin = (uint64_t)thread->kernel_stack; |
1140 | uint64_t kernStackMax = (uint64_t)kernStackMin + kernel_stack_size; |
1141 | mach_msg_type_number_t ct = *start_idx; |
1142 | kern_return_t kr = KERN_FAILURE; |
1143 | |
1144 | if (*start_idx >= max_idx) { |
1145 | return KERN_RESOURCE_SHORTAGE; // no frames traced |
1146 | } |
1147 | frames[ct++] = chudxnu_vm_unslide(currPC, supervisor); |
1148 | |
1149 | // build a backtrace of this 32 bit state. |
1150 | while (VALID_STACK_ADDRESS64(supervisor, currFP, kernStackMin, kernStackMax)) { |
1151 | // this is the address where caller lives in the user thread |
1152 | uint64_t caller = currFP + sizeof(uint64_t); |
1153 | |
1154 | if (!currFP) { |
1155 | currPC = 0; |
1156 | break; |
1157 | } |
1158 | |
1159 | if (ct >= max_idx) { |
1160 | *start_idx = ct; |
1161 | return KERN_RESOURCE_SHORTAGE; |
1162 | } |
1163 | |
1164 | /* read our caller */ |
1165 | if (supervisor) { |
1166 | kr = chudxnu_kern_read(&currPC, (vm_offset_t)caller, sizeof(uint64_t)); |
1167 | } else { |
1168 | kr = chudxnu_task_read(task, &currPC, caller, sizeof(uint64_t)); |
1169 | } |
1170 | |
1171 | if (kr != KERN_SUCCESS) { |
1172 | currPC = 0ULL; |
1173 | break; |
1174 | } |
1175 | |
1176 | /* |
1177 | * retrive contents of the frame pointer and advance to the next stack |
1178 | * frame if it's valid |
1179 | */ |
1180 | prevFP = 0; |
1181 | if (supervisor) { |
1182 | kr = chudxnu_kern_read(&prevFP, (vm_offset_t)currFP, sizeof(uint64_t)); |
1183 | } else { |
1184 | kr = chudxnu_task_read(task, &prevFP, currFP, sizeof(uint64_t)); |
1185 | } |
1186 | |
1187 | if (VALID_STACK_ADDRESS64(supervisor, prevFP, kernStackMin, kernStackMax)) { |
1188 | frames[ct++] = chudxnu_vm_unslide(currPC, supervisor); |
1189 | prevPC = currPC; |
1190 | } |
1191 | if (prevFP < currFP) { |
1192 | break; |
1193 | } else { |
1194 | currFP = prevFP; |
1195 | } |
1196 | } |
1197 | |
1198 | *start_idx = ct; |
1199 | return KERN_SUCCESS; |
1200 | } |
1201 | |
1202 | static kern_return_t |
1203 | chudxnu_thread_get_callstack64_internal( |
1204 | thread_t thread, |
1205 | uint64_t *callstack, |
1206 | mach_msg_type_number_t *count, |
1207 | boolean_t user_only, |
1208 | boolean_t kern_only) |
1209 | { |
1210 | kern_return_t kr = KERN_FAILURE; |
1211 | task_t task = get_threadtask(thread); |
1212 | uint64_t currPC = 0ULL; |
1213 | boolean_t supervisor = FALSE; |
1214 | mach_msg_type_number_t bufferIndex = 0; |
1215 | mach_msg_type_number_t bufferMaxIndex = *count; |
1216 | x86_saved_state_t *tagged_regs = NULL; // kernel register state |
1217 | x86_saved_state64_t *regs64 = NULL; |
1218 | x86_saved_state32_t *regs32 = NULL; |
1219 | x86_saved_state32_t *u_regs32 = NULL; |
1220 | x86_saved_state64_t *u_regs64 = NULL; |
1221 | struct x86_kernel_state *kregs = NULL; |
1222 | |
1223 | if (ml_at_interrupt_context()) { |
1224 | if (user_only) { |
1225 | /* can't backtrace user state on interrupt stack. */ |
1226 | return KERN_FAILURE; |
1227 | } |
1228 | |
1229 | /* backtracing at interrupt context? */ |
1230 | if (thread == current_thread() && current_cpu_datap()->cpu_int_state) { |
1231 | /* |
1232 | * Locate the registers for the interrupted thread, assuming it is |
1233 | * current_thread(). |
1234 | */ |
1235 | tagged_regs = current_cpu_datap()->cpu_int_state; |
1236 | |
1237 | if (is_saved_state64(tagged_regs)) { |
1238 | /* 64 bit registers */ |
1239 | regs64 = saved_state64(tagged_regs); |
1240 | supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U); |
1241 | } else { |
1242 | /* 32 bit registers */ |
1243 | regs32 = saved_state32(tagged_regs); |
1244 | supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U); |
1245 | } |
1246 | } |
1247 | } |
1248 | |
1249 | if (!ml_at_interrupt_context() && kernel_task == task) { |
1250 | if (!thread->kernel_stack) { |
1251 | return KERN_FAILURE; |
1252 | } |
1253 | |
1254 | // Kernel thread not at interrupt context |
1255 | kregs = (struct x86_kernel_state *)NULL; |
1256 | |
1257 | // nofault read of the thread->kernel_stack pointer |
1258 | if (KERN_SUCCESS != chudxnu_kern_read(&kregs, (vm_offset_t)&(thread->kernel_stack), sizeof(void *))) { |
1259 | return KERN_FAILURE; |
1260 | } |
1261 | |
1262 | // Adjust to find the saved kernel state |
1263 | kregs = STACK_IKS((vm_offset_t)(uintptr_t)kregs); |
1264 | |
1265 | supervisor = TRUE; |
1266 | } else if (!tagged_regs) { |
1267 | /* |
1268 | * not at interrupt context, or tracing a different thread than |
1269 | * current_thread() at interrupt context |
1270 | */ |
1271 | tagged_regs = USER_STATE(thread); |
1272 | if (is_saved_state64(tagged_regs)) { |
1273 | /* 64 bit registers */ |
1274 | regs64 = saved_state64(tagged_regs); |
1275 | supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U); |
1276 | } else { |
1277 | /* 32 bit registers */ |
1278 | regs32 = saved_state32(tagged_regs); |
1279 | supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U); |
1280 | } |
1281 | } |
1282 | |
1283 | *count = 0; |
1284 | |
1285 | if (supervisor) { |
1286 | // the caller only wants a user callstack. |
1287 | if (user_only) { |
1288 | // bail - we've only got kernel state |
1289 | return KERN_FAILURE; |
1290 | } |
1291 | } else { |
1292 | // regs32(64) is not in supervisor mode. |
1293 | u_regs32 = regs32; |
1294 | u_regs64 = regs64; |
1295 | regs32 = NULL; |
1296 | regs64 = NULL; |
1297 | } |
1298 | |
1299 | if (user_only) { |
1300 | /* we only want to backtrace the user mode */ |
1301 | if (!(u_regs32 || u_regs64)) { |
1302 | /* no user state to look at */ |
1303 | return KERN_FAILURE; |
1304 | } |
1305 | } |
1306 | |
1307 | /* |
1308 | * Order of preference for top of stack: |
1309 | * 64 bit kernel state (not likely) |
1310 | * 32 bit kernel state |
1311 | * 64 bit user land state |
1312 | * 32 bit user land state |
1313 | */ |
1314 | |
1315 | if (kregs) { |
1316 | /* |
1317 | * nofault read of the registers from the kernel stack (as they can |
1318 | * disappear on the fly). |
1319 | */ |
1320 | |
1321 | if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(kregs->k_rip), sizeof(uint64_t))) { |
1322 | return KERN_FAILURE; |
1323 | } |
1324 | } else if (regs64) { |
1325 | currPC = regs64->isf.rip; |
1326 | } else if (regs32) { |
1327 | currPC = (uint64_t) regs32->eip; |
1328 | } else if (u_regs64) { |
1329 | currPC = u_regs64->isf.rip; |
1330 | } else if (u_regs32) { |
1331 | currPC = (uint64_t) u_regs32->eip; |
1332 | } |
1333 | |
1334 | if (!currPC) { |
1335 | /* no top of the stack, bail out */ |
1336 | return KERN_FAILURE; |
1337 | } |
1338 | |
1339 | bufferIndex = 0; |
1340 | |
1341 | if (bufferMaxIndex < 1) { |
1342 | *count = 0; |
1343 | return KERN_RESOURCE_SHORTAGE; |
1344 | } |
1345 | |
1346 | /* backtrace kernel */ |
1347 | if (kregs) { |
1348 | addr64_t address = 0ULL; |
1349 | size_t size = 0UL; |
1350 | |
1351 | // do the backtrace |
1352 | kr = do_kernel_backtrace(thread, kregs, callstack, &bufferIndex, bufferMaxIndex); |
1353 | |
1354 | // and do a nofault read of (r|e)sp |
1355 | uint64_t rsp = 0ULL; |
1356 | size = sizeof(uint64_t); |
1357 | |
1358 | if (KERN_SUCCESS != chudxnu_kern_read(&address, (vm_offset_t)&(kregs->k_rsp), size)) { |
1359 | address = 0ULL; |
1360 | } |
1361 | |
1362 | if (address && KERN_SUCCESS == chudxnu_kern_read(&rsp, (vm_offset_t)address, size) && bufferIndex < bufferMaxIndex) { |
1363 | callstack[bufferIndex++] = (uint64_t)rsp; |
1364 | } |
1365 | } else if (regs64) { |
1366 | uint64_t rsp = 0ULL; |
1367 | |
1368 | // backtrace the 64bit side. |
1369 | kr = do_backtrace64(task, thread, regs64, callstack, &bufferIndex, |
1370 | bufferMaxIndex - 1, TRUE); |
1371 | |
1372 | if (KERN_SUCCESS == chudxnu_kern_read(&rsp, (vm_offset_t) regs64->isf.rsp, sizeof(uint64_t)) && |
1373 | bufferIndex < bufferMaxIndex) { |
1374 | callstack[bufferIndex++] = rsp; |
1375 | } |
1376 | } else if (regs32) { |
1377 | uint32_t esp = 0UL; |
1378 | |
1379 | // backtrace the 32bit side. |
1380 | kr = do_backtrace32(task, thread, regs32, callstack, &bufferIndex, |
1381 | bufferMaxIndex - 1, TRUE); |
1382 | |
1383 | if (KERN_SUCCESS == chudxnu_kern_read(&esp, (vm_offset_t) regs32->uesp, sizeof(uint32_t)) && |
1384 | bufferIndex < bufferMaxIndex) { |
1385 | callstack[bufferIndex++] = (uint64_t) esp; |
1386 | } |
1387 | } else if (u_regs64 && !kern_only) { |
1388 | /* backtrace user land */ |
1389 | uint64_t rsp = 0ULL; |
1390 | |
1391 | kr = do_backtrace64(task, thread, u_regs64, callstack, &bufferIndex, |
1392 | bufferMaxIndex - 1, FALSE); |
1393 | |
1394 | if (KERN_SUCCESS == chudxnu_task_read(task, &rsp, (addr64_t) u_regs64->isf.rsp, sizeof(uint64_t)) && |
1395 | bufferIndex < bufferMaxIndex) { |
1396 | callstack[bufferIndex++] = rsp; |
1397 | } |
1398 | } else if (u_regs32 && !kern_only) { |
1399 | uint32_t esp = 0UL; |
1400 | |
1401 | kr = do_backtrace32(task, thread, u_regs32, callstack, &bufferIndex, |
1402 | bufferMaxIndex - 1, FALSE); |
1403 | |
1404 | if (KERN_SUCCESS == chudxnu_task_read(task, &esp, (addr64_t) u_regs32->uesp, sizeof(uint32_t)) && |
1405 | bufferIndex < bufferMaxIndex) { |
1406 | callstack[bufferIndex++] = (uint64_t) esp; |
1407 | } |
1408 | } |
1409 | |
1410 | *count = bufferIndex; |
1411 | return kr; |
1412 | } |
1413 | |
1414 | __private_extern__ |
1415 | kern_return_t |
1416 | chudxnu_thread_get_callstack64_kperf( |
1417 | thread_t thread, |
1418 | uint64_t *callstack, |
1419 | mach_msg_type_number_t *count, |
1420 | boolean_t is_user) |
1421 | { |
1422 | return chudxnu_thread_get_callstack64_internal(thread, callstack, count, is_user, !is_user); |
1423 | } |
1424 | #else /* !__arm64__ && !__x86_64__ */ |
1425 | #error kperf: unsupported architecture |
1426 | #endif /* !__arm64__ && !__x86_64__ */ |
1427 | |