1 | /* |
2 | * Copyright (c) 2011 Apple Computer, Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | /* |
30 | * Called from a trigger. Actually takes the data from the different |
31 | * modules and puts them in a buffer |
32 | */ |
33 | |
34 | #include <mach/mach_types.h> |
35 | #include <machine/machine_routines.h> |
36 | #include <kern/kalloc.h> |
37 | #include <kern/debug.h> /* panic */ |
38 | #include <kern/thread.h> |
39 | #include <sys/errno.h> |
40 | #include <sys/vm.h> |
41 | #include <vm/vm_object.h> |
42 | #include <vm/vm_page.h> |
43 | #include <vm/vm_pageout.h> |
44 | |
45 | #ifdef CONFIG_EXCLAVES |
46 | #include <kern/exclaves.tightbeam.h> |
47 | #endif /* CONFIG_EXCLAVES */ |
48 | |
49 | #include <kperf/action.h> |
50 | #include <kperf/ast.h> |
51 | #include <kperf/buffer.h> |
52 | #include <kperf/callstack.h> |
53 | #include <kperf/context.h> |
54 | #include <kperf/kdebug_trigger.h> |
55 | #include <kperf/kperf.h> |
56 | #include <kperf/kperf_kpc.h> |
57 | #include <kperf/kptimer.h> |
58 | #include <kperf/pet.h> |
59 | #include <kperf/sample.h> |
60 | #include <kperf/thread_samplers.h> |
61 | |
62 | #define ACTION_MAX (32) |
63 | |
64 | /* the list of different actions to take */ |
65 | struct action { |
66 | uint32_t sample; |
67 | uint32_t ucallstack_depth; |
68 | uint32_t kcallstack_depth; |
69 | uint32_t userdata; |
70 | int pid_filter; |
71 | }; |
72 | |
73 | /* the list of actions */ |
74 | static unsigned int actionc = 0; |
75 | static struct action *actionv = NULL; |
76 | |
77 | /* should emit tracepoint on context switch */ |
78 | int kperf_kdebug_cswitch = 0; |
79 | |
80 | int kperf_max_actions = ACTION_MAX; |
81 | bool |
82 | kperf_action_has_non_system(unsigned int actionid) |
83 | { |
84 | if (actionid > actionc) { |
85 | return false; |
86 | } |
87 | |
88 | if (actionv[actionid - 1].sample & ~SAMPLER_SYS_MEM) { |
89 | return true; |
90 | } else { |
91 | return false; |
92 | } |
93 | } |
94 | |
95 | bool |
96 | kperf_action_has_task(unsigned int actionid) |
97 | { |
98 | if (actionid > actionc) { |
99 | return false; |
100 | } |
101 | |
102 | return actionv[actionid - 1].sample & SAMPLER_TASK_MASK; |
103 | } |
104 | |
105 | bool |
106 | kperf_action_has_thread(unsigned int actionid) |
107 | { |
108 | if (actionid > actionc) { |
109 | return false; |
110 | } |
111 | |
112 | return actionv[actionid - 1].sample & SAMPLER_THREAD_MASK; |
113 | } |
114 | |
115 | static void |
116 | kperf_system_memory_log(void) |
117 | { |
118 | extern unsigned int memorystatus_level; |
119 | |
120 | BUF_DATA(PERF_MI_SYS_DATA, (uintptr_t)vm_page_free_count, |
121 | (uintptr_t)vm_page_wire_count, (uintptr_t)vm_page_external_count, |
122 | (uintptr_t)(vm_page_active_count + vm_page_inactive_count + |
123 | vm_page_speculative_count)); |
124 | BUF_DATA(PERF_MI_SYS_DATA_2, (uintptr_t)vm_page_anonymous_count, |
125 | (uintptr_t)vm_page_internal_count, |
126 | (uintptr_t)vm_pageout_vminfo.vm_pageout_compressions, |
127 | (uintptr_t)VM_PAGE_COMPRESSOR_COUNT); |
128 | BUF_DATA(PERF_MI_SYS_DATA_3, |
129 | #if CONFIG_SECLUDED_MEMORY |
130 | (uintptr_t)vm_page_secluded_count, |
131 | #else // CONFIG_SECLUDED_MEMORY |
132 | 0, |
133 | #endif // !CONFIG_SECLUDED_MEMORY |
134 | (uintptr_t)vm_page_purgeable_count, |
135 | memorystatus_level); |
136 | } |
137 | |
138 | static void |
139 | kperf_sample_user_internal(struct kperf_usample *sbuf, |
140 | struct kperf_context *context, unsigned int actionid, |
141 | unsigned int sample_what) |
142 | { |
143 | if (sample_what & SAMPLER_USTACK) { |
144 | kperf_ucallstack_sample(cs: &sbuf->ucallstack, context); |
145 | } |
146 | if (sample_what & SAMPLER_TH_INFO) { |
147 | kperf_thread_info_sample(&sbuf->th_info, context); |
148 | } |
149 | |
150 | boolean_t intren = ml_set_interrupts_enabled(FALSE); |
151 | |
152 | /* |
153 | * No userdata or sample_flags for this one. |
154 | */ |
155 | BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_START, sample_what, actionid); |
156 | |
157 | if (sample_what & SAMPLER_USTACK) { |
158 | kperf_ucallstack_log(cs: &sbuf->ucallstack); |
159 | } |
160 | if (sample_what & SAMPLER_TH_DISPATCH) { |
161 | kperf_thread_dispatch_log(&sbuf->usample_min->th_dispatch); |
162 | } |
163 | if (sample_what & SAMPLER_TH_INFO) { |
164 | kperf_thread_info_log(&sbuf->th_info); |
165 | } |
166 | |
167 | BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_END, sample_what); |
168 | |
169 | ml_set_interrupts_enabled(enable: intren); |
170 | } |
171 | |
172 | static unsigned int |
173 | kperf_prepare_sample_what(unsigned int sample_what, unsigned int sample_flags) |
174 | { |
175 | /* callstacks should be explicitly ignored */ |
176 | if (sample_flags & SAMPLE_FLAG_EMPTY_CALLSTACK) { |
177 | sample_what &= ~(SAMPLER_KSTACK | SAMPLER_USTACK | SAMPLER_EXSTACK); |
178 | } |
179 | if (sample_flags & SAMPLE_FLAG_ONLY_SYSTEM) { |
180 | sample_what &= SAMPLER_SYS_MEM; |
181 | } |
182 | assert((sample_flags & (SAMPLE_FLAG_THREAD_ONLY | SAMPLE_FLAG_TASK_ONLY)) |
183 | != (SAMPLE_FLAG_THREAD_ONLY | SAMPLE_FLAG_TASK_ONLY)); |
184 | if (sample_flags & SAMPLE_FLAG_THREAD_ONLY) { |
185 | sample_what &= SAMPLER_THREAD_MASK; |
186 | } |
187 | if (sample_flags & SAMPLE_FLAG_TASK_ONLY) { |
188 | sample_what &= SAMPLER_TASK_MASK; |
189 | } |
190 | |
191 | return sample_what; |
192 | } |
193 | |
194 | void |
195 | kperf_sample_user(struct kperf_usample *sbuf, struct kperf_context *context, |
196 | unsigned int actionid, unsigned int sample_flags) |
197 | { |
198 | if (actionid == 0 || actionid > actionc) { |
199 | return; |
200 | } |
201 | |
202 | unsigned int sample_what = kperf_prepare_sample_what( |
203 | sample_what: actionv[actionid - 1].sample, sample_flags); |
204 | if (sample_what == 0) { |
205 | return; |
206 | } |
207 | |
208 | unsigned int ucallstack_depth = actionv[actionid - 1].ucallstack_depth; |
209 | sbuf->ucallstack.kpuc_nframes = ucallstack_depth ?: MAX_UCALLSTACK_FRAMES; |
210 | |
211 | kperf_sample_user_internal(sbuf, context, actionid, sample_what); |
212 | } |
213 | |
214 | static kern_return_t |
215 | kperf_sample_internal(struct kperf_sample *sbuf, |
216 | struct kperf_context *context, |
217 | unsigned sample_what, unsigned sample_flags, |
218 | unsigned actionid, unsigned ucallstack_depth) |
219 | { |
220 | int pended_ucallstack = 0; |
221 | int pended_th_dispatch = 0; |
222 | uint32_t userdata = actionid; |
223 | bool task_only = (sample_flags & SAMPLE_FLAG_TASK_ONLY) != 0; |
224 | bool pended_exclave_callstack = false; |
225 | uint64_t sample_meta_flags = 0; |
226 | |
227 | sample_what = kperf_prepare_sample_what(sample_what, sample_flags); |
228 | if (sample_what == 0) { |
229 | return SAMPLE_CONTINUE; |
230 | } |
231 | |
232 | if (!task_only) { |
233 | context->cur_thread->kperf_pet_gen = |
234 | os_atomic_load(&kppet_gencount, relaxed); |
235 | } |
236 | bool is_kernel = (context->cur_pid == 0); |
237 | |
238 | if (actionid && actionid <= actionc) { |
239 | sbuf->kcallstack.kpkc_nframes = |
240 | actionv[actionid - 1].kcallstack_depth; |
241 | } else { |
242 | sbuf->kcallstack.kpkc_nframes = MAX_KCALLSTACK_FRAMES; |
243 | } |
244 | |
245 | ucallstack_depth = ucallstack_depth ?: MAX_UCALLSTACK_FRAMES; |
246 | sbuf->kcallstack.kpkc_flags = 0; |
247 | sbuf->usample.ucallstack.kpuc_flags = 0; |
248 | |
249 | if (sample_what & SAMPLER_TH_INFO) { |
250 | kperf_thread_info_sample(&sbuf->th_info, context); |
251 | |
252 | if (!(sample_flags & SAMPLE_FLAG_IDLE_THREADS)) { |
253 | if (sbuf->th_info.kpthi_runmode & 0x40) { |
254 | sample_meta_flags |= SAMPLE_META_THREAD_WAS_IDLE; |
255 | goto log_sample; |
256 | } |
257 | } |
258 | } |
259 | |
260 | if (sample_what & SAMPLER_TH_SNAPSHOT) { |
261 | kperf_thread_snapshot_sample(&(sbuf->th_snapshot), context); |
262 | } |
263 | if (sample_what & SAMPLER_TH_SCHEDULING) { |
264 | kperf_thread_scheduling_sample(&(sbuf->th_scheduling), context); |
265 | } |
266 | if (sample_what & SAMPLER_KSTACK) { |
267 | if (sample_flags & SAMPLE_FLAG_CONTINUATION) { |
268 | kperf_continuation_sample(cs: &(sbuf->kcallstack), context); |
269 | } else if (sample_flags & SAMPLE_FLAG_NON_INTERRUPT) { |
270 | /* outside of interrupt context, backtrace the current thread */ |
271 | kperf_backtrace_sample(cs: &(sbuf->kcallstack), context); |
272 | } else { |
273 | kperf_kcallstack_sample(cs: &(sbuf->kcallstack), context); |
274 | } |
275 | } |
276 | if (sample_what & SAMPLER_TK_SNAPSHOT) { |
277 | kperf_task_snapshot_sample(task: context->cur_task, tksn: &(sbuf->tk_snapshot)); |
278 | } |
279 | |
280 | if (!is_kernel) { |
281 | if (sample_what & SAMPLER_MEMINFO) { |
282 | kperf_meminfo_sample(context->cur_task, &(sbuf->meminfo)); |
283 | } |
284 | |
285 | if (sample_flags & SAMPLE_FLAG_PEND_USER) { |
286 | if (sample_what & SAMPLER_USTACK) { |
287 | pended_ucallstack = kperf_ucallstack_pend(context, |
288 | depth: ucallstack_depth, actionid); |
289 | } |
290 | |
291 | if (sample_what & SAMPLER_TH_DISPATCH) { |
292 | pended_th_dispatch = |
293 | kperf_thread_dispatch_pend(context, actionid); |
294 | } |
295 | } |
296 | } |
297 | |
298 | #if CONFIG_EXCLAVES |
299 | if (sample_what & SAMPLER_EXSTACK) { |
300 | pended_exclave_callstack = kperf_exclave_callstack_pend(context, actionid); |
301 | } |
302 | #endif /* CONFIG_EXCLAVES */ |
303 | |
304 | #if CONFIG_CPU_COUNTERS |
305 | if (sample_what & SAMPLER_PMC_THREAD) { |
306 | kperf_kpc_thread_sample(&(sbuf->kpcdata), sample_what); |
307 | } else if (sample_what & SAMPLER_PMC_CPU) { |
308 | kperf_kpc_cpu_sample(&(sbuf->kpcdata), sample_what); |
309 | } |
310 | #endif /* CONFIG_CPU_COUNTERS */ |
311 | |
312 | log_sample: |
313 | /* lookup the user tag, if any */ |
314 | if (actionid && (actionid <= actionc)) { |
315 | userdata = actionv[actionid - 1].userdata; |
316 | } |
317 | |
318 | /* avoid logging if this sample only pended samples */ |
319 | if (sample_flags & SAMPLE_FLAG_PEND_USER && |
320 | !(sample_what & ~(SAMPLER_USTACK | SAMPLER_TH_DISPATCH))) { |
321 | return SAMPLE_CONTINUE; |
322 | } |
323 | |
324 | /* stash the data into the buffer |
325 | * interrupts off to ensure we don't get split |
326 | */ |
327 | boolean_t enabled = ml_set_interrupts_enabled(FALSE); |
328 | |
329 | BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_START, sample_what, |
330 | actionid, userdata, sample_flags); |
331 | |
332 | if (sample_flags & SAMPLE_FLAG_SYSTEM) { |
333 | if (sample_what & SAMPLER_SYS_MEM) { |
334 | kperf_system_memory_log(); |
335 | } |
336 | } |
337 | if (sample_meta_flags & SAMPLE_META_THREAD_WAS_IDLE) { |
338 | goto log_sample_end; |
339 | } |
340 | |
341 | if (sample_what & SAMPLER_TH_INFO) { |
342 | kperf_thread_info_log(&sbuf->th_info); |
343 | } |
344 | if (sample_what & SAMPLER_TH_SCHEDULING) { |
345 | kperf_thread_scheduling_log(&(sbuf->th_scheduling)); |
346 | } |
347 | if (sample_what & SAMPLER_TH_SNAPSHOT) { |
348 | kperf_thread_snapshot_log(&(sbuf->th_snapshot)); |
349 | } |
350 | if (sample_what & SAMPLER_KSTACK) { |
351 | kperf_kcallstack_log(cs: &sbuf->kcallstack); |
352 | } |
353 | if (sample_what & SAMPLER_TH_INSCYC) { |
354 | kperf_thread_inscyc_log(context); |
355 | } |
356 | if (sample_what & SAMPLER_TK_SNAPSHOT) { |
357 | kperf_task_snapshot_log(tksn: &(sbuf->tk_snapshot)); |
358 | } |
359 | if (sample_what & SAMPLER_TK_INFO) { |
360 | kperf_task_info_log(ctx: context); |
361 | } |
362 | |
363 | /* dump user stuff */ |
364 | if (!is_kernel) { |
365 | /* dump meminfo */ |
366 | if (sample_what & SAMPLER_MEMINFO) { |
367 | kperf_meminfo_log(mi: &(sbuf->meminfo)); |
368 | } |
369 | |
370 | if (sample_flags & SAMPLE_FLAG_PEND_USER) { |
371 | if (pended_ucallstack) { |
372 | BUF_INFO(PERF_CS_UPEND); |
373 | sample_meta_flags |= SAMPLE_META_UPEND; |
374 | } |
375 | |
376 | if (pended_th_dispatch) { |
377 | BUF_INFO(PERF_TI_DISPPEND); |
378 | } |
379 | } |
380 | } |
381 | |
382 | if (pended_exclave_callstack) { |
383 | sample_meta_flags |= SAMPLE_META_EXPEND; |
384 | } |
385 | |
386 | #if CONFIG_CPU_COUNTERS |
387 | if (sample_what & SAMPLER_PMC_CONFIG) { |
388 | kperf_kpc_config_log(&(sbuf->kpcdata)); |
389 | } |
390 | if (sample_what & SAMPLER_PMC_THREAD) { |
391 | kperf_kpc_thread_log(&(sbuf->kpcdata)); |
392 | } else if (sample_what & SAMPLER_PMC_CPU) { |
393 | kperf_kpc_cpu_log(&(sbuf->kpcdata)); |
394 | } |
395 | #endif /* CONFIG_CPU_COUNTERS */ |
396 | |
397 | log_sample_end: |
398 | BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_END, sample_what, sample_meta_flags); |
399 | |
400 | /* intrs back on */ |
401 | ml_set_interrupts_enabled(enable: enabled); |
402 | |
403 | return SAMPLE_CONTINUE; |
404 | } |
405 | |
406 | /* Translate actionid into sample bits and take a sample */ |
407 | kern_return_t |
408 | kperf_sample(struct kperf_sample *sbuf, |
409 | struct kperf_context *context, |
410 | unsigned actionid, unsigned sample_flags) |
411 | { |
412 | /* work out what to sample, if anything */ |
413 | if ((actionid > actionc) || (actionid == 0)) { |
414 | return SAMPLE_SHUTDOWN; |
415 | } |
416 | |
417 | /* check the pid filter against the context's current pid. |
418 | * filter pid == -1 means any pid |
419 | */ |
420 | int pid_filter = actionv[actionid - 1].pid_filter; |
421 | if ((pid_filter != -1) && (pid_filter != context->cur_pid)) { |
422 | return SAMPLE_CONTINUE; |
423 | } |
424 | |
425 | /* the samplers to run */ |
426 | unsigned int sample_what = actionv[actionid - 1].sample; |
427 | unsigned int ucallstack_depth = actionv[actionid - 1].ucallstack_depth; |
428 | |
429 | /* do the actual sample operation */ |
430 | return kperf_sample_internal(sbuf, context, sample_what, |
431 | sample_flags, actionid, ucallstack_depth); |
432 | } |
433 | |
434 | void |
435 | kperf_kdebug_handler(uint32_t debugid, uintptr_t *starting_fp) |
436 | { |
437 | uint32_t sample_flags = SAMPLE_FLAG_NON_INTERRUPT | SAMPLE_FLAG_PEND_USER; |
438 | struct kperf_sample *sample = NULL; |
439 | kern_return_t kr = KERN_SUCCESS; |
440 | int s; |
441 | |
442 | if (!kperf_kdebug_should_trigger(debugid)) { |
443 | return; |
444 | } |
445 | |
446 | BUF_VERB(PERF_KDBG_HNDLR | DBG_FUNC_START, debugid); |
447 | |
448 | thread_t thread = current_thread(); |
449 | task_t task = get_threadtask(thread); |
450 | struct kperf_context ctx = { |
451 | .cur_thread = thread, |
452 | .cur_task = task, |
453 | .cur_pid = task_pid(task), |
454 | .trigger_type = TRIGGER_TYPE_KDEBUG, |
455 | .trigger_id = 0, |
456 | .starting_fp = starting_fp, |
457 | }; |
458 | |
459 | s = ml_set_interrupts_enabled(enable: 0); |
460 | |
461 | sample = kperf_intr_sample_buffer(); |
462 | |
463 | kr = kperf_sample(sbuf: sample, context: &ctx, actionid: kperf_kdebug_get_action(), sample_flags); |
464 | |
465 | ml_set_interrupts_enabled(enable: s); |
466 | BUF_VERB(PERF_KDBG_HNDLR | DBG_FUNC_END, kr); |
467 | } |
468 | |
469 | /* |
470 | * Sample using a minimum of stack space during this phase. |
471 | */ |
472 | static void |
473 | kperf_ast_sample_min_stack_phase(struct kperf_usample_min *sbuf_min, |
474 | struct kperf_context *context, unsigned int sample_what) |
475 | { |
476 | if (sample_what & SAMPLER_TH_DISPATCH) { |
477 | kperf_thread_dispatch_sample(&sbuf_min->th_dispatch, context); |
478 | } |
479 | } |
480 | |
481 | /* |
482 | * This function should not be inlined with its caller, which would pollute |
483 | * the stack usage of the minimum stack phase, above. |
484 | */ |
485 | __attribute__((noinline)) |
486 | static void |
487 | kperf_ast_sample_max_stack_phase(struct kperf_usample_min *sbuf_min, |
488 | struct kperf_context *context, uint32_t actionid, unsigned int sample_what, |
489 | unsigned int nframes) |
490 | { |
491 | struct kperf_usample sbuf = { .usample_min = sbuf_min }; |
492 | sbuf.ucallstack.kpuc_nframes = nframes; |
493 | |
494 | kperf_sample_user_internal(sbuf: &sbuf, context, actionid, sample_what); |
495 | } |
496 | |
497 | /* |
498 | * This function allocates >2.3KB of the stack. Prevent the compiler from |
499 | * inlining this function into ast_taken and ensure the stack memory is only |
500 | * allocated for the kperf AST. |
501 | */ |
502 | __attribute__((noinline)) |
503 | void |
504 | kperf_thread_ast_handler(thread_t thread) |
505 | { |
506 | uint32_t ast = thread->kperf_ast; |
507 | |
508 | BUF_INFO(PERF_AST_HNDLR | DBG_FUNC_START, thread, ast); |
509 | |
510 | task_t task = get_threadtask(thread); |
511 | |
512 | if (task_did_exec(task) || task_is_exec_copy(task)) { |
513 | BUF_INFO(PERF_AST_HNDLR | DBG_FUNC_END, SAMPLE_CONTINUE); |
514 | return; |
515 | } |
516 | |
517 | struct kperf_context ctx = { |
518 | .cur_thread = thread, |
519 | .cur_task = task, |
520 | .cur_pid = task_pid(task), |
521 | }; |
522 | |
523 | unsigned int sample_what = 0; |
524 | if (ast & T_KPERF_AST_DISPATCH) { |
525 | sample_what |= SAMPLER_TH_DISPATCH; |
526 | } |
527 | if (ast & T_KPERF_AST_CALLSTACK) { |
528 | /* TH_INFO for backwards compatibility */ |
529 | sample_what |= SAMPLER_USTACK | SAMPLER_TH_INFO; |
530 | } |
531 | |
532 | unsigned int actionid = T_KPERF_GET_ACTIONID(ast); |
533 | |
534 | struct kperf_usample_min sbuf_min = { 0 }; |
535 | kperf_ast_sample_min_stack_phase(sbuf_min: &sbuf_min, context: &ctx, sample_what); |
536 | kperf_ast_sample_max_stack_phase(sbuf_min: &sbuf_min, context: &ctx, actionid, sample_what, |
537 | T_KPERF_GET_CALLSTACK_DEPTH(ast) ?: MAX_UCALLSTACK_FRAMES); |
538 | |
539 | BUF_INFO(PERF_AST_HNDLR | DBG_FUNC_END); |
540 | } |
541 | |
542 | |
543 | #if CONFIG_EXCLAVES |
544 | /* Called from Exclave inspection thread after collecting a sample */ |
545 | __attribute__((noinline)) |
546 | void kperf_thread_exclaves_ast_handler(thread_t thread, const stackshot_stackshotentry_s * _Nonnull entry); |
547 | |
548 | __attribute__((noinline)) |
549 | void |
550 | kperf_thread_exclaves_ast_handler(thread_t thread, const stackshot_stackshotentry_s * _Nonnull entry) |
551 | { |
552 | assert3u(entry->scid, ==, thread->th_exclaves_scheduling_context_id); |
553 | uint32_t ast = thread->kperf_exclaves_ast; |
554 | |
555 | BUF_INFO(PERF_AST_EXCLAVES | DBG_FUNC_START, thread, ast); |
556 | unsigned int actionid = T_KPERF_GET_ACTIONID(ast); |
557 | |
558 | boolean_t intren = ml_set_interrupts_enabled(false); |
559 | |
560 | __block size_t ipcstack_count = 0; |
561 | |
562 | BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_START, SAMPLER_EXSTACK, actionid); |
563 | if (entry->ipcstack.has_value) { |
564 | stackshot_ipcstackentry__v_visit(&entry->ipcstack.value, ^(size_t __unused i, const stackshot_ipcstackentry_s * _Nonnull __unused ipcstack) { |
565 | ipcstack_count += 1; |
566 | }); |
567 | |
568 | BUF_DATA(PERF_CS_EXSTACKHDR, ipcstack_count, thread->thread_id, entry->scid); |
569 | |
570 | stackshot_ipcstackentry__v_visit(&entry->ipcstack.value, ^(size_t __unused j, const stackshot_ipcstackentry_s * _Nonnull ipcstack) { |
571 | kperf_excallstack_log(ipcstack); |
572 | }); |
573 | } |
574 | BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_END, SAMPLER_EXSTACK); |
575 | |
576 | ml_set_interrupts_enabled(intren); |
577 | |
578 | BUF_INFO(PERF_AST_EXCLAVES | DBG_FUNC_END); |
579 | } |
580 | #endif /* CONFIG_EXCLAVES */ |
581 | |
582 | int |
583 | kperf_ast_pend(thread_t thread, uint32_t set_flags, unsigned int set_actionid) |
584 | { |
585 | if (thread != current_thread()) { |
586 | panic("kperf: pending AST to non-current thread" ); |
587 | } |
588 | |
589 | uint32_t ast = thread->kperf_ast; |
590 | unsigned int actionid = T_KPERF_GET_ACTIONID(ast); |
591 | uint32_t flags = ast & T_KPERF_AST_ALL; |
592 | |
593 | if ((flags | set_flags) != flags || actionid != set_actionid) { |
594 | ast &= ~T_KPERF_SET_ACTIONID(actionid); |
595 | ast |= T_KPERF_SET_ACTIONID(set_actionid); |
596 | ast |= set_flags; |
597 | |
598 | thread->kperf_ast = ast; |
599 | |
600 | /* set the actual AST */ |
601 | act_set_kperf(thread); |
602 | return 1; |
603 | } |
604 | |
605 | return 0; |
606 | } |
607 | |
608 | void |
609 | kperf_ast_set_callstack_depth(thread_t thread, uint32_t depth) |
610 | { |
611 | uint32_t ast = thread->kperf_ast; |
612 | uint32_t existing_depth = T_KPERF_GET_CALLSTACK_DEPTH(ast); |
613 | if (existing_depth < depth) { |
614 | ast &= ~T_KPERF_SET_CALLSTACK_DEPTH(existing_depth); |
615 | ast |= T_KPERF_SET_CALLSTACK_DEPTH(depth); |
616 | thread->kperf_ast = ast; |
617 | } |
618 | } |
619 | |
620 | int |
621 | kperf_kdbg_cswitch_get(void) |
622 | { |
623 | return kperf_kdebug_cswitch; |
624 | } |
625 | |
626 | int |
627 | kperf_kdbg_cswitch_set(int newval) |
628 | { |
629 | kperf_kdebug_cswitch = newval; |
630 | kperf_on_cpu_update(); |
631 | |
632 | return 0; |
633 | } |
634 | |
635 | /* |
636 | * Action configuration |
637 | */ |
638 | unsigned int |
639 | kperf_action_get_count(void) |
640 | { |
641 | return actionc; |
642 | } |
643 | |
644 | int |
645 | kperf_action_set_samplers(unsigned actionid, uint32_t samplers) |
646 | { |
647 | if ((actionid > actionc) || (actionid == 0)) { |
648 | return EINVAL; |
649 | } |
650 | |
651 | /* disallow both CPU and thread counters to be sampled in the same |
652 | * action */ |
653 | if ((samplers & SAMPLER_PMC_THREAD) && (samplers & SAMPLER_PMC_CPU)) { |
654 | return EINVAL; |
655 | } |
656 | |
657 | actionv[actionid - 1].sample = samplers; |
658 | |
659 | return 0; |
660 | } |
661 | |
662 | int |
663 | kperf_action_get_samplers(unsigned actionid, uint32_t *samplers_out) |
664 | { |
665 | if ((actionid > actionc)) { |
666 | return EINVAL; |
667 | } |
668 | |
669 | if (actionid == 0) { |
670 | *samplers_out = 0; /* "NULL" action */ |
671 | } else { |
672 | *samplers_out = actionv[actionid - 1].sample; |
673 | } |
674 | |
675 | return 0; |
676 | } |
677 | |
678 | int |
679 | kperf_action_set_userdata(unsigned actionid, uint32_t userdata) |
680 | { |
681 | if ((actionid > actionc) || (actionid == 0)) { |
682 | return EINVAL; |
683 | } |
684 | |
685 | actionv[actionid - 1].userdata = userdata; |
686 | |
687 | return 0; |
688 | } |
689 | |
690 | int |
691 | kperf_action_get_userdata(unsigned actionid, uint32_t *userdata_out) |
692 | { |
693 | if ((actionid > actionc)) { |
694 | return EINVAL; |
695 | } |
696 | |
697 | if (actionid == 0) { |
698 | *userdata_out = 0; /* "NULL" action */ |
699 | } else { |
700 | *userdata_out = actionv[actionid - 1].userdata; |
701 | } |
702 | |
703 | return 0; |
704 | } |
705 | |
706 | int |
707 | kperf_action_set_filter(unsigned actionid, int pid) |
708 | { |
709 | if ((actionid > actionc) || (actionid == 0)) { |
710 | return EINVAL; |
711 | } |
712 | |
713 | actionv[actionid - 1].pid_filter = pid; |
714 | |
715 | return 0; |
716 | } |
717 | |
718 | int |
719 | kperf_action_get_filter(unsigned actionid, int *pid_out) |
720 | { |
721 | if ((actionid > actionc)) { |
722 | return EINVAL; |
723 | } |
724 | |
725 | if (actionid == 0) { |
726 | *pid_out = -1; /* "NULL" action */ |
727 | } else { |
728 | *pid_out = actionv[actionid - 1].pid_filter; |
729 | } |
730 | |
731 | return 0; |
732 | } |
733 | |
734 | void |
735 | kperf_action_reset(void) |
736 | { |
737 | for (unsigned int i = 0; i < actionc; i++) { |
738 | kperf_action_set_samplers(actionid: i + 1, samplers: 0); |
739 | kperf_action_set_userdata(actionid: i + 1, userdata: 0); |
740 | kperf_action_set_filter(actionid: i + 1, pid: -1); |
741 | kperf_action_set_ucallstack_depth(actionid: i + 1, MAX_UCALLSTACK_FRAMES); |
742 | kperf_action_set_kcallstack_depth(actionid: i + 1, MAX_KCALLSTACK_FRAMES); |
743 | } |
744 | } |
745 | |
746 | int |
747 | kperf_action_set_count(unsigned count) |
748 | { |
749 | struct action *new_actionv = NULL, *old_actionv = NULL; |
750 | unsigned old_count; |
751 | |
752 | /* easy no-op */ |
753 | if (count == actionc) { |
754 | return 0; |
755 | } |
756 | |
757 | /* TODO: allow shrinking? */ |
758 | if (count < actionc) { |
759 | return EINVAL; |
760 | } |
761 | |
762 | /* cap it for good measure */ |
763 | if (count > ACTION_MAX) { |
764 | return EINVAL; |
765 | } |
766 | |
767 | /* creating the action arror for the first time. create a few |
768 | * more things, too. |
769 | */ |
770 | if (actionc == 0) { |
771 | kperf_setup(); |
772 | } |
773 | |
774 | /* create a new array */ |
775 | new_actionv = kalloc_data_tag(count * sizeof(*new_actionv), |
776 | Z_WAITOK, VM_KERN_MEMORY_DIAG); |
777 | if (new_actionv == NULL) { |
778 | return ENOMEM; |
779 | } |
780 | |
781 | old_actionv = actionv; |
782 | old_count = actionc; |
783 | |
784 | if (old_actionv != NULL) { |
785 | memcpy(dst: new_actionv, src: actionv, n: actionc * sizeof(*actionv)); |
786 | } |
787 | |
788 | memset(s: &(new_actionv[actionc]), c: 0, n: (count - old_count) * sizeof(*actionv)); |
789 | |
790 | for (unsigned int i = old_count; i < count; i++) { |
791 | new_actionv[i].pid_filter = -1; |
792 | new_actionv[i].ucallstack_depth = MAX_UCALLSTACK_FRAMES; |
793 | new_actionv[i].kcallstack_depth = MAX_KCALLSTACK_FRAMES; |
794 | } |
795 | |
796 | actionv = new_actionv; |
797 | actionc = count; |
798 | |
799 | kfree_data(old_actionv, old_count * sizeof(*actionv)); |
800 | |
801 | return 0; |
802 | } |
803 | |
804 | int |
805 | kperf_action_set_ucallstack_depth(unsigned action_id, uint32_t depth) |
806 | { |
807 | if ((action_id > actionc) || (action_id == 0)) { |
808 | return EINVAL; |
809 | } |
810 | |
811 | if (depth > MAX_UCALLSTACK_FRAMES) { |
812 | return EINVAL; |
813 | } |
814 | if (depth < 2) { |
815 | return EINVAL; |
816 | } |
817 | |
818 | actionv[action_id - 1].ucallstack_depth = depth; |
819 | |
820 | return 0; |
821 | } |
822 | |
823 | int |
824 | kperf_action_set_kcallstack_depth(unsigned action_id, uint32_t depth) |
825 | { |
826 | if ((action_id > actionc) || (action_id == 0)) { |
827 | return EINVAL; |
828 | } |
829 | |
830 | if (depth > MAX_KCALLSTACK_FRAMES) { |
831 | return EINVAL; |
832 | } |
833 | if (depth < 1) { |
834 | return EINVAL; |
835 | } |
836 | |
837 | actionv[action_id - 1].kcallstack_depth = depth; |
838 | |
839 | return 0; |
840 | } |
841 | |
842 | int |
843 | kperf_action_get_ucallstack_depth(unsigned action_id, uint32_t * depth_out) |
844 | { |
845 | if ((action_id > actionc)) { |
846 | return EINVAL; |
847 | } |
848 | |
849 | assert(depth_out); |
850 | |
851 | if (action_id == 0) { |
852 | *depth_out = MAX_UCALLSTACK_FRAMES; |
853 | } else { |
854 | *depth_out = actionv[action_id - 1].ucallstack_depth; |
855 | } |
856 | |
857 | return 0; |
858 | } |
859 | |
860 | int |
861 | kperf_action_get_kcallstack_depth(unsigned action_id, uint32_t * depth_out) |
862 | { |
863 | if ((action_id > actionc)) { |
864 | return EINVAL; |
865 | } |
866 | |
867 | assert(depth_out); |
868 | |
869 | if (action_id == 0) { |
870 | *depth_out = MAX_KCALLSTACK_FRAMES; |
871 | } else { |
872 | *depth_out = actionv[action_id - 1].kcallstack_depth; |
873 | } |
874 | |
875 | return 0; |
876 | } |
877 | |