1 | /* |
2 | * Copyright (c) 2011-2018 Apple Computer, Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <kern/ipc_tt.h> /* port_name_to_task */ |
30 | #include <kern/thread.h> |
31 | #include <kern/machine.h> |
32 | #include <kern/kalloc.h> |
33 | #include <mach/mach_types.h> |
34 | #include <sys/errno.h> |
35 | #include <sys/ktrace.h> |
36 | |
37 | #include <kperf/action.h> |
38 | #include <kperf/buffer.h> |
39 | #include <kperf/kdebug_trigger.h> |
40 | #include <kperf/kperf.h> |
41 | #include <kperf/kptimer.h> |
42 | #include <kperf/lazy.h> |
43 | #include <kperf/pet.h> |
44 | #include <kperf/sample.h> |
45 | |
46 | /* from libkern/libkern.h */ |
47 | extern uint64_t strtouq(const char *, char **, int); |
48 | |
49 | LCK_GRP_DECLARE(kperf_lck_grp, "kperf" ); |
50 | |
51 | /* one wired sample buffer per CPU */ |
52 | static struct kperf_sample *__zpercpu intr_samplev; |
53 | |
54 | /* current sampling status */ |
55 | enum kperf_sampling _Atomic kperf_status = KPERF_SAMPLING_OFF; |
56 | |
57 | /* |
58 | * Only set up kperf once. |
59 | */ |
60 | static bool kperf_is_setup = false; |
61 | |
62 | /* whether or not to callback to kperf on context switch */ |
63 | boolean_t kperf_on_cpu_active = FALSE; |
64 | |
65 | unsigned int kperf_thread_blocked_action; |
66 | unsigned int kperf_cpu_sample_action; |
67 | |
68 | struct kperf_sample * |
69 | kperf_intr_sample_buffer(void) |
70 | { |
71 | assert(ml_get_interrupts_enabled() == FALSE); |
72 | |
73 | return zpercpu_get(intr_samplev); |
74 | } |
75 | |
76 | void |
77 | kperf_init_early(void) |
78 | { |
79 | /* |
80 | * kperf allocates based on the number of CPUs and requires them to all be |
81 | * accounted for. |
82 | */ |
83 | ml_wait_max_cpus(); |
84 | |
85 | boolean_t found_kperf = FALSE; |
86 | char kperf_config_str[64]; |
87 | found_kperf = PE_parse_boot_arg_str(arg_string: "kperf" , arg_ptr: kperf_config_str, size: sizeof(kperf_config_str)); |
88 | if (found_kperf && kperf_config_str[0] != '\0') { |
89 | kperf_kernel_configure(config: kperf_config_str); |
90 | } |
91 | } |
92 | |
93 | void |
94 | kperf_init(void) |
95 | { |
96 | kptimer_init(); |
97 | } |
98 | |
99 | void |
100 | kperf_setup(void) |
101 | { |
102 | if (kperf_is_setup) { |
103 | return; |
104 | } |
105 | |
106 | intr_samplev = zalloc_percpu_permanent_type(struct kperf_sample); |
107 | |
108 | kperf_kdebug_setup(); |
109 | kperf_is_setup = true; |
110 | } |
111 | |
112 | void |
113 | kperf_reset(void) |
114 | { |
115 | /* |
116 | * Make sure samples aren't being taken before tearing everything down. |
117 | */ |
118 | (void)kperf_disable_sampling(); |
119 | |
120 | kperf_lazy_reset(); |
121 | (void)kperf_kdbg_cswitch_set(newval: 0); |
122 | kperf_kdebug_reset(); |
123 | kptimer_reset(); |
124 | kppet_reset(); |
125 | |
126 | /* |
127 | * Most of the other systems call into actions, so reset them last. |
128 | */ |
129 | kperf_action_reset(); |
130 | } |
131 | |
132 | void |
133 | kperf_kernel_configure(const char *config) |
134 | { |
135 | int pairs = 0; |
136 | char *end; |
137 | bool pet = false; |
138 | |
139 | assert(config != NULL); |
140 | |
141 | ktrace_start_single_threaded(); |
142 | |
143 | ktrace_kernel_configure(KTRACE_KPERF); |
144 | |
145 | if (config[0] == 'p') { |
146 | pet = true; |
147 | config++; |
148 | } |
149 | |
150 | do { |
151 | uint32_t action_samplers; |
152 | uint64_t timer_period_ns; |
153 | uint64_t timer_period; |
154 | |
155 | pairs += 1; |
156 | kperf_action_set_count(count: pairs); |
157 | kptimer_set_count(count: pairs); |
158 | |
159 | action_samplers = (uint32_t)strtouq(config, &end, 0); |
160 | if (config == end) { |
161 | kprintf(fmt: "kperf: unable to parse '%s' as action sampler\n" , config); |
162 | goto out; |
163 | } |
164 | config = end; |
165 | |
166 | kperf_action_set_samplers(actionid: pairs, samplers: action_samplers); |
167 | |
168 | if (config[0] == '\0') { |
169 | kprintf(fmt: "kperf: missing timer period in config\n" ); |
170 | goto out; |
171 | } |
172 | config++; |
173 | |
174 | timer_period_ns = strtouq(config, &end, 0); |
175 | if (config == end) { |
176 | kprintf(fmt: "kperf: unable to parse '%s' as timer period\n" , config); |
177 | goto out; |
178 | } |
179 | nanoseconds_to_absolutetime(nanoseconds: timer_period_ns, result: &timer_period); |
180 | config = end; |
181 | |
182 | kptimer_set_period(timerid: pairs - 1, period: timer_period); |
183 | kptimer_set_action(timer: pairs - 1, actionid: pairs); |
184 | |
185 | if (pet) { |
186 | kptimer_set_pet_timerid(timerid: pairs - 1); |
187 | kppet_set_lightweight_pet(on: 1); |
188 | pet = false; |
189 | } |
190 | } while (*(config++) == ','); |
191 | |
192 | int error = kperf_enable_sampling(); |
193 | if (error) { |
194 | printf(format: "kperf: cannot enable sampling at boot: %d\n" , error); |
195 | } |
196 | |
197 | out: |
198 | ktrace_end_single_threaded(); |
199 | } |
200 | |
201 | void kperf_on_cpu_internal(thread_t thread, thread_continue_t continuation, |
202 | uintptr_t *starting_fp); |
203 | void |
204 | kperf_on_cpu_internal(thread_t thread, thread_continue_t continuation, |
205 | uintptr_t *starting_fp) |
206 | { |
207 | if (kperf_kdebug_cswitch) { |
208 | /* trace the new thread's PID for Instruments */ |
209 | int pid = task_pid(task: get_threadtask(thread)); |
210 | BUF_DATA(PERF_TI_CSWITCH, thread_tid(thread), pid); |
211 | } |
212 | if (kppet_lightweight_active) { |
213 | kppet_on_cpu(thread, continuation, starting_frame: starting_fp); |
214 | } |
215 | if (kperf_lazy_wait_action != 0) { |
216 | kperf_lazy_wait_sample(thread, continuation, starting_fp); |
217 | } |
218 | } |
219 | |
220 | void |
221 | kperf_on_cpu_update(void) |
222 | { |
223 | kperf_on_cpu_active = kperf_kdebug_cswitch || |
224 | kppet_lightweight_active || |
225 | kperf_lazy_wait_action != 0; |
226 | } |
227 | |
228 | bool |
229 | kperf_is_sampling(void) |
230 | { |
231 | return os_atomic_load(&kperf_status, acquire) == KPERF_SAMPLING_ON; |
232 | } |
233 | |
234 | int |
235 | kperf_enable_sampling(void) |
236 | { |
237 | if (!kperf_is_setup || kperf_action_get_count() == 0) { |
238 | return ECANCELED; |
239 | } |
240 | |
241 | enum kperf_sampling prev_status = KPERF_SAMPLING_ON; |
242 | int ok = os_atomic_cmpxchgv(&kperf_status, KPERF_SAMPLING_OFF, |
243 | KPERF_SAMPLING_ON, &prev_status, seq_cst); |
244 | if (!ok) { |
245 | if (prev_status == KPERF_SAMPLING_ON) { |
246 | return 0; |
247 | } |
248 | panic("kperf: sampling was %d when asked to enable" , prev_status); |
249 | } |
250 | |
251 | kppet_lightweight_active_update(); |
252 | kptimer_start(); |
253 | |
254 | return 0; |
255 | } |
256 | |
257 | int |
258 | kperf_disable_sampling(void) |
259 | { |
260 | enum kperf_sampling prev_status = KPERF_SAMPLING_ON; |
261 | int ok = os_atomic_cmpxchgv(&kperf_status, KPERF_SAMPLING_ON, |
262 | KPERF_SAMPLING_SHUTDOWN, &prev_status, seq_cst); |
263 | if (!ok) { |
264 | if (prev_status == KPERF_SAMPLING_OFF) { |
265 | return 0; |
266 | } |
267 | panic("kperf: sampling was %d when asked to disable" , prev_status); |
268 | } |
269 | |
270 | kptimer_stop(); |
271 | |
272 | ok = os_atomic_cmpxchgv(&kperf_status, KPERF_SAMPLING_SHUTDOWN, |
273 | KPERF_SAMPLING_OFF, &prev_status, seq_cst); |
274 | if (!ok) { |
275 | panic("kperf: sampling was %d during disable" , prev_status); |
276 | } |
277 | kppet_lightweight_active_update(); |
278 | |
279 | return 0; |
280 | } |
281 | |
282 | void |
283 | kperf_timer_expire(void *param0, void * __unused param1) |
284 | { |
285 | processor_t processor = param0; |
286 | int cpuid = processor->cpu_id; |
287 | |
288 | kptimer_expire(processor, cpuid, now: mach_absolute_time()); |
289 | } |
290 | |
291 | boolean_t |
292 | kperf_thread_get_dirty(thread_t thread) |
293 | { |
294 | return thread->c_switch != thread->kperf_c_switch; |
295 | } |
296 | |
297 | void |
298 | kperf_thread_set_dirty(thread_t thread, boolean_t dirty) |
299 | { |
300 | if (dirty) { |
301 | thread->kperf_c_switch = thread->c_switch - 1; |
302 | } else { |
303 | thread->kperf_c_switch = thread->c_switch; |
304 | } |
305 | } |
306 | |
307 | int |
308 | kperf_port_to_pid(mach_port_name_t portname) |
309 | { |
310 | if (!MACH_PORT_VALID(portname)) { |
311 | return -1; |
312 | } |
313 | |
314 | task_t task = port_name_to_task(portname); |
315 | if (task == TASK_NULL) { |
316 | return -1; |
317 | } |
318 | |
319 | pid_t pid = task_pid(task); |
320 | |
321 | |
322 | assert(os_ref_get_count(&task->ref_count) > 1); |
323 | task_deallocate(task); |
324 | |
325 | return pid; |
326 | } |
327 | |