1 | /* |
2 | * Copyright (c) 2011 Apple Computer, Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | /* Sample thread data */ |
30 | |
31 | #include <kern/debug.h> /* panic */ |
32 | #include <kern/thread.h> /* thread_* */ |
33 | #include <kern/timer.h> /* timer_data_t */ |
34 | #include <kern/policy_internal.h> /* TASK_POLICY_* */ |
35 | #include <mach/mach_types.h> |
36 | |
37 | #include <kperf/kperf.h> |
38 | #include <kperf/buffer.h> |
39 | #include <kperf/context.h> |
40 | #include <kperf/thread_samplers.h> |
41 | #include <kperf/ast.h> |
42 | |
43 | #if MONOTONIC |
44 | #include <kern/monotonic.h> |
45 | #include <machine/monotonic.h> |
46 | #endif /* MONOTONIC */ |
47 | |
48 | extern boolean_t stackshot_thread_is_idle_worker_unsafe(thread_t thread); |
49 | |
50 | /* |
51 | * XXX Deprecated, use thread scheduling sampler instead. |
52 | * |
53 | * Taken from AppleProfileGetRunModeOfThread and CHUD. Still here for |
54 | * backwards compatibility. |
55 | */ |
56 | |
57 | #define KPERF_TI_RUNNING (1U << 0) |
58 | #define KPERF_TI_RUNNABLE (1U << 1) |
59 | #define KPERF_TI_WAIT (1U << 2) |
60 | #define KPERF_TI_UNINT (1U << 3) |
61 | #define KPERF_TI_SUSP (1U << 4) |
62 | #define KPERF_TI_TERMINATE (1U << 5) |
63 | #define KPERF_TI_IDLE (1U << 6) |
64 | |
65 | static uint32_t |
66 | kperf_thread_info_runmode_legacy(thread_t thread) |
67 | { |
68 | uint32_t kperf_state = 0; |
69 | int sched_state = thread->state; |
70 | processor_t last_processor = thread->last_processor; |
71 | |
72 | if ((last_processor != PROCESSOR_NULL) && (thread == last_processor->active_thread)) { |
73 | kperf_state |= KPERF_TI_RUNNING; |
74 | } |
75 | if (sched_state & TH_RUN) { |
76 | kperf_state |= KPERF_TI_RUNNABLE; |
77 | } |
78 | if (sched_state & TH_WAIT) { |
79 | kperf_state |= KPERF_TI_WAIT; |
80 | } |
81 | if (sched_state & TH_UNINT) { |
82 | kperf_state |= KPERF_TI_UNINT; |
83 | } |
84 | if (sched_state & TH_SUSP) { |
85 | kperf_state |= KPERF_TI_SUSP; |
86 | } |
87 | if (sched_state & TH_TERMINATE) { |
88 | kperf_state |= KPERF_TI_TERMINATE; |
89 | } |
90 | if (sched_state & TH_IDLE) { |
91 | kperf_state |= KPERF_TI_IDLE; |
92 | } |
93 | |
94 | #if !CONFIG_EMBEDDED |
95 | /* on desktop, if state is blank, leave not idle set */ |
96 | if (kperf_state == 0) { |
97 | return (TH_IDLE << 16); |
98 | } |
99 | #endif /* !CONFIG_EMBEDDED */ |
100 | |
101 | /* high two bytes are inverted mask, low two bytes are normal */ |
102 | return (((~kperf_state & 0xffff) << 16) | (kperf_state & 0xffff)); |
103 | } |
104 | |
105 | void |
106 | kperf_thread_info_sample(struct kperf_thread_info *ti, struct kperf_context *context) |
107 | { |
108 | thread_t cur_thread = context->cur_thread; |
109 | |
110 | BUF_INFO(PERF_TI_SAMPLE, (uintptr_t)thread_tid(cur_thread)); |
111 | |
112 | ti->kpthi_pid = context->cur_pid; |
113 | ti->kpthi_tid = thread_tid(cur_thread); |
114 | ti->kpthi_dq_addr = thread_dispatchqaddr(cur_thread); |
115 | ti->kpthi_runmode = kperf_thread_info_runmode_legacy(cur_thread); |
116 | |
117 | BUF_VERB(PERF_TI_SAMPLE | DBG_FUNC_END); |
118 | } |
119 | |
120 | void |
121 | kperf_thread_info_log(struct kperf_thread_info *ti) |
122 | { |
123 | BUF_DATA(PERF_TI_DATA, ti->kpthi_pid, ti->kpthi_tid /* K64-only */, |
124 | ti->kpthi_dq_addr, ti->kpthi_runmode); |
125 | } |
126 | |
127 | /* |
128 | * Scheduling information reports inputs and outputs of the scheduler state for |
129 | * a thread. |
130 | */ |
131 | |
132 | void |
133 | kperf_thread_scheduling_sample(struct kperf_thread_scheduling *thsc, |
134 | struct kperf_context *context) |
135 | { |
136 | assert(thsc != NULL); |
137 | assert(context != NULL); |
138 | |
139 | thread_t thread = context->cur_thread; |
140 | |
141 | BUF_INFO(PERF_TI_SCHEDSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread)); |
142 | |
143 | thsc->kpthsc_user_time = timer_grab(&thread->user_timer); |
144 | uint64_t system_time = timer_grab(&thread->system_timer); |
145 | |
146 | if (thread->precise_user_kernel_time) { |
147 | thsc->kpthsc_system_time = system_time; |
148 | } else { |
149 | thsc->kpthsc_user_time += system_time; |
150 | thsc->kpthsc_system_time = 0; |
151 | } |
152 | |
153 | thsc->kpthsc_runnable_time = timer_grab(&thread->runnable_timer); |
154 | thsc->kpthsc_state = thread->state; |
155 | thsc->kpthsc_base_priority = thread->base_pri; |
156 | thsc->kpthsc_sched_priority = thread->sched_pri; |
157 | thsc->kpthsc_effective_qos = thread->effective_policy.thep_qos; |
158 | thsc->kpthsc_requested_qos = thread->requested_policy.thrp_qos; |
159 | thsc->kpthsc_requested_qos_override = MAX(thread->requested_policy.thrp_qos_override, |
160 | thread->requested_policy.thrp_qos_workq_override); |
161 | thsc->kpthsc_requested_qos_promote = thread->requested_policy.thrp_qos_promote; |
162 | thsc->kpthsc_requested_qos_ipc_override = thread->requested_policy.thrp_qos_ipc_override; |
163 | thsc->kpthsc_requested_qos_sync_ipc_override = thread->requested_policy.thrp_qos_sync_ipc_override; |
164 | thsc->kpthsc_effective_latency_qos = thread->effective_policy.thep_latency_qos; |
165 | |
166 | BUF_INFO(PERF_TI_SCHEDSAMPLE | DBG_FUNC_END); |
167 | } |
168 | |
169 | |
170 | void |
171 | kperf_thread_scheduling_log(struct kperf_thread_scheduling *thsc) |
172 | { |
173 | assert(thsc != NULL); |
174 | #if defined(__LP64__) |
175 | BUF_DATA(PERF_TI_SCHEDDATA_2, thsc->kpthsc_user_time, |
176 | thsc->kpthsc_system_time, |
177 | (((uint64_t)thsc->kpthsc_base_priority) << 48) |
178 | | ((uint64_t)thsc->kpthsc_sched_priority << 32) |
179 | | ((uint64_t)(thsc->kpthsc_state & 0xff) << 24) |
180 | | (thsc->kpthsc_effective_qos << 6) |
181 | | (thsc->kpthsc_requested_qos << 3) |
182 | | thsc->kpthsc_requested_qos_override, |
183 | ((uint64_t)thsc->kpthsc_effective_latency_qos << 61) |
184 | | ((uint64_t)thsc->kpthsc_requested_qos_promote << 58) |
185 | | ((uint64_t)thsc->kpthsc_requested_qos_ipc_override << 55) |
186 | | ((uint64_t)thsc->kpthsc_requested_qos_sync_ipc_override << 52) |
187 | ); |
188 | BUF_DATA(PERF_TI_SCHEDDATA_3, thsc->kpthsc_runnable_time); |
189 | #else |
190 | BUF_DATA(PERF_TI_SCHEDDATA1_32, UPPER_32(thsc->kpthsc_user_time), |
191 | LOWER_32(thsc->kpthsc_user_time), |
192 | UPPER_32(thsc->kpthsc_system_time), |
193 | LOWER_32(thsc->kpthsc_system_time) |
194 | ); |
195 | BUF_DATA(PERF_TI_SCHEDDATA2_32_2, (((uint32_t)thsc->kpthsc_base_priority) << 16) |
196 | | thsc->kpthsc_sched_priority, |
197 | ((thsc->kpthsc_state & 0xff) << 24) |
198 | | (thsc->kpthsc_effective_qos << 6) |
199 | | (thsc->kpthsc_requested_qos << 3) |
200 | | thsc->kpthsc_requested_qos_override, |
201 | ((uint32_t)thsc->kpthsc_effective_latency_qos << 29) |
202 | | ((uint32_t)thsc->kpthsc_requested_qos_promote << 26) |
203 | | ((uint32_t)thsc->kpthsc_requested_qos_ipc_override << 23) |
204 | | ((uint32_t)thsc->kpthsc_requested_qos_sync_ipc_override << 20) |
205 | ); |
206 | BUF_DATA(PERF_TI_SCHEDDATA3_32, UPPER_32(thsc->kpthsc_runnable_time), |
207 | LOWER_32(thsc->kpthsc_runnable_time)); |
208 | #endif /* defined(__LP64__) */ |
209 | } |
210 | |
211 | /* |
212 | * Snapshot information maintains parity with stackshot information for other, |
213 | * miscellaneous information about threads. |
214 | */ |
215 | |
216 | #define KPERF_THREAD_SNAPSHOT_DARWIN_BG (1U << 0); |
217 | #define KPERF_THREAD_SNAPSHOT_PASSIVE_IO (1U << 1); |
218 | #define KPERF_THREAD_SNAPSHOT_GFI (1U << 2); |
219 | #define KPERF_THREAD_SNAPSHOT_IDLE_WQ (1U << 3); |
220 | /* max is 1U << 7 */ |
221 | |
222 | void |
223 | kperf_thread_snapshot_sample(struct kperf_thread_snapshot *thsn, |
224 | struct kperf_context *context) |
225 | { |
226 | assert(thsn != NULL); |
227 | assert(context != NULL); |
228 | |
229 | thread_t thread = context->cur_thread; |
230 | |
231 | BUF_INFO(PERF_TI_SNAPSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread)); |
232 | |
233 | thsn->kpthsn_last_made_runnable_time = thread->last_made_runnable_time; |
234 | |
235 | thsn->kpthsn_flags = 0; |
236 | if (thread->effective_policy.thep_darwinbg) { |
237 | thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_DARWIN_BG; |
238 | } |
239 | if (proc_get_effective_thread_policy(thread, TASK_POLICY_PASSIVE_IO)) { |
240 | thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_PASSIVE_IO; |
241 | } |
242 | if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE) { |
243 | thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_GFI |
244 | } |
245 | if (stackshot_thread_is_idle_worker_unsafe(thread)) { |
246 | thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_IDLE_WQ; |
247 | } |
248 | |
249 | thsn->kpthsn_suspend_count = thread->suspend_count; |
250 | thsn->kpthsn_io_tier = proc_get_effective_thread_policy(thread, TASK_POLICY_IO); |
251 | |
252 | BUF_VERB(PERF_TI_SNAPSAMPLE | DBG_FUNC_END); |
253 | } |
254 | |
255 | void |
256 | kperf_thread_snapshot_log(struct kperf_thread_snapshot *thsn) |
257 | { |
258 | assert(thsn != NULL); |
259 | #if defined(__LP64__) |
260 | BUF_DATA(PERF_TI_SNAPDATA, thsn->kpthsn_flags | ((uint32_t)(thsn->kpthsn_suspend_count) << 8) |
261 | | (thsn->kpthsn_io_tier << 24), |
262 | thsn->kpthsn_last_made_runnable_time); |
263 | #else |
264 | BUF_DATA(PERF_TI_SNAPDATA_32, thsn->kpthsn_flags | ((uint32_t)(thsn->kpthsn_suspend_count) << 8) |
265 | | (thsn->kpthsn_io_tier << 24), |
266 | UPPER_32(thsn->kpthsn_last_made_runnable_time), |
267 | LOWER_32(thsn->kpthsn_last_made_runnable_time)); |
268 | #endif /* defined(__LP64__) */ |
269 | } |
270 | |
271 | /* |
272 | * Dispatch information only contains the dispatch queue serial number from |
273 | * libdispatch. |
274 | * |
275 | * It's a separate sampler because queue data must be copied in from user space. |
276 | */ |
277 | |
278 | void |
279 | kperf_thread_dispatch_sample(struct kperf_thread_dispatch *thdi, |
280 | struct kperf_context *context) |
281 | { |
282 | assert(thdi != NULL); |
283 | assert(context != NULL); |
284 | |
285 | thread_t thread = context->cur_thread; |
286 | |
287 | BUF_INFO(PERF_TI_DISPSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread)); |
288 | |
289 | task_t task = thread->task; |
290 | boolean_t task_64 = task_has_64Bit_addr(task); |
291 | size_t user_addr_size = task_64 ? 8 : 4; |
292 | |
293 | assert(thread->task != kernel_task); |
294 | uint64_t user_dq_key_addr = thread_dispatchqaddr(thread); |
295 | if (user_dq_key_addr == 0) { |
296 | goto error; |
297 | } |
298 | |
299 | uint64_t user_dq_addr; |
300 | if ((copyin((user_addr_t)user_dq_key_addr, |
301 | (char *)&user_dq_addr, |
302 | user_addr_size) != 0) || |
303 | (user_dq_addr == 0)) |
304 | { |
305 | goto error; |
306 | } |
307 | |
308 | uint64_t user_dq_serialno_addr = |
309 | user_dq_addr + get_task_dispatchqueue_serialno_offset(task); |
310 | |
311 | if (copyin((user_addr_t)user_dq_serialno_addr, |
312 | (char *)&(thdi->kpthdi_dq_serialno), |
313 | user_addr_size) == 0) |
314 | { |
315 | goto out; |
316 | } |
317 | |
318 | error: |
319 | thdi->kpthdi_dq_serialno = 0; |
320 | |
321 | out: |
322 | BUF_VERB(PERF_TI_DISPSAMPLE | DBG_FUNC_END); |
323 | } |
324 | |
325 | int |
326 | kperf_thread_dispatch_pend(struct kperf_context *context) |
327 | { |
328 | return kperf_ast_pend(context->cur_thread, T_KPERF_AST_DISPATCH); |
329 | } |
330 | |
331 | void |
332 | kperf_thread_dispatch_log(struct kperf_thread_dispatch *thdi) |
333 | { |
334 | assert(thdi != NULL); |
335 | #if defined(__LP64__) |
336 | BUF_DATA(PERF_TI_DISPDATA, thdi->kpthdi_dq_serialno); |
337 | #else |
338 | BUF_DATA(PERF_TI_DISPDATA_32, UPPER_32(thdi->kpthdi_dq_serialno), |
339 | LOWER_32(thdi->kpthdi_dq_serialno)); |
340 | #endif /* defined(__LP64__) */ |
341 | } |
342 | |
343 | /* |
344 | * A bit different from other samplers -- since logging disables interrupts, |
345 | * it's a fine place to sample the thread counters. |
346 | */ |
347 | void |
348 | kperf_thread_inscyc_log(struct kperf_context *context) |
349 | { |
350 | #if MONOTONIC |
351 | thread_t cur_thread = current_thread(); |
352 | |
353 | if (context->cur_thread != cur_thread) { |
354 | /* can't safely access another thread's counters */ |
355 | return; |
356 | } |
357 | |
358 | uint64_t counts[MT_CORE_NFIXED]; |
359 | |
360 | int ret = mt_fixed_thread_counts(cur_thread, counts); |
361 | if (ret) { |
362 | return; |
363 | } |
364 | |
365 | #if defined(__LP64__) |
366 | BUF_DATA(PERF_TI_INSCYCDATA, counts[MT_CORE_INSTRS], counts[MT_CORE_CYCLES]); |
367 | #else /* defined(__LP64__) */ |
368 | /* 32-bit platforms don't count instructions */ |
369 | BUF_DATA(PERF_TI_INSCYCDATA_32, 0, 0, UPPER_32(counts[MT_CORE_CYCLES]), |
370 | LOWER_32(counts[MT_CORE_CYCLES])); |
371 | #endif /* !defined(__LP64__) */ |
372 | |
373 | #else |
374 | #pragma unused(context) |
375 | #endif /* MONOTONIC */ |
376 | |
377 | } |
378 | |