1 | /* |
2 | * Copyright (c) 2012 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #ifndef KERN_KPC_H |
30 | #define KERN_KPC_H |
31 | |
32 | /* Kernel interfaces to KPC PMC infrastructure. */ |
33 | |
34 | #include <kern/thread.h> /* thread_* */ |
35 | #include <machine/machine_kpc.h> |
36 | |
37 | __BEGIN_DECLS |
38 | |
39 | typedef uint64_t kpc_config_t; |
40 | |
41 | /* cross-platform class constants */ |
42 | #define KPC_CLASS_FIXED (0) |
43 | #define KPC_CLASS_CONFIGURABLE (1) |
44 | #define KPC_CLASS_POWER (2) |
45 | #define KPC_CLASS_RAWPMU (3) |
46 | |
47 | #define KPC_CLASS_FIXED_MASK (1u << KPC_CLASS_FIXED) |
48 | #define KPC_CLASS_CONFIGURABLE_MASK (1u << KPC_CLASS_CONFIGURABLE) |
49 | #define KPC_CLASS_POWER_MASK (1u << KPC_CLASS_POWER) |
50 | #define KPC_CLASS_RAWPMU_MASK (1u << KPC_CLASS_RAWPMU) |
51 | |
52 | #define KPC_PMU_ERROR (0) |
53 | #define KPC_PMU_INTEL_V3 (1) |
54 | #define KPC_PMU_ARM_APPLE (2) |
55 | #define KPC_PMU_INTEL_V2 (3) |
56 | #define KPC_PMU_ARM_V2 (4) |
57 | |
58 | #define KPC_ALL_CPUS (1u << 31) |
59 | |
60 | /* action id setters/getters */ |
61 | #define FIXED_ACTIONID(ctr) (kpc_actionid[(ctr)]) |
62 | #define CONFIGURABLE_ACTIONID(ctr) (kpc_actionid[(ctr) + kpc_fixed_count()]) |
63 | |
64 | /* reload counter setters/getters */ |
65 | #define FIXED_RELOAD(ctr) (current_cpu_datap()->cpu_kpc_reload[(ctr)]) |
66 | #define FIXED_RELOAD_CPU(cpu, ctr) (cpu_datap(cpu)->cpu_kpc_reload[(ctr)]) |
67 | #define CONFIGURABLE_RELOAD(ctr) (current_cpu_datap()->cpu_kpc_reload[(ctr) + kpc_fixed_count()]) |
68 | #define CONFIGURABLE_RELOAD_CPU(cpu, ctr) (cpu_datap(cpu)->cpu_kpc_reload[(ctr) + kpc_fixed_count()]) |
69 | |
70 | /* shadow counter setters/getters */ |
71 | #define FIXED_SHADOW(ctr) (current_cpu_datap()->cpu_kpc_shadow[(ctr)]) |
72 | #define FIXED_SHADOW_CPU(cpu, ctr) (cpu_datap(cpu)->cpu_kpc_shadow[(ctr)]) |
73 | #define CONFIGURABLE_SHADOW(ctr) (current_cpu_datap()->cpu_kpc_shadow[(ctr) + kpc_fixed_count()]) |
74 | #define CONFIGURABLE_SHADOW_CPU(cpu, ctr) (cpu_datap(cpu)->cpu_kpc_shadow[(ctr) + kpc_fixed_count()]) |
75 | |
76 | /** |
77 | * Callback for notification when PMCs are acquired/released by a task. The |
78 | * argument is equal to TRUE if the Power Manager (PM) can use its reserved PMCs. |
79 | * Otherwise, the argument is equal to FALSE. |
80 | */ |
81 | typedef void (*kpc_pm_handler_t)(boolean_t); |
82 | |
83 | struct cpu_data; |
84 | extern boolean_t kpc_register_cpu(struct cpu_data *cpu_data); |
85 | extern void kpc_unregister_cpu(struct cpu_data *cpu_data); |
86 | |
87 | extern bool kpc_supported; |
88 | |
89 | /* bootstrap */ |
90 | extern void kpc_init(void); |
91 | |
92 | /* Architecture specific initialisation */ |
93 | extern void kpc_arch_init(void); |
94 | |
95 | /* Get the bitmask of available classes */ |
96 | extern uint32_t kpc_get_classes(void); |
97 | |
98 | /* Get the bitmask of currently running counter classes */ |
99 | extern uint32_t kpc_get_running(void); |
100 | |
101 | /* Get the version of KPC that's being run */ |
102 | extern int kpc_get_pmu_version(void); |
103 | |
104 | /* Set the bitmask of currently running counter classes. Specify |
105 | * classes = 0 to stop counters |
106 | */ |
107 | extern int kpc_set_running(uint32_t classes); |
108 | |
109 | /* Read CPU counters */ |
110 | extern int kpc_get_cpu_counters(boolean_t all_cpus, uint32_t classes, |
111 | int *curcpu, uint64_t *buf); |
112 | |
113 | /* Read shadow counters */ |
114 | extern int kpc_get_shadow_counters( boolean_t all_cpus, uint32_t classes, |
115 | int *curcpu, uint64_t *buf ); |
116 | |
117 | /* Read current thread's counter accumulations */ |
118 | extern int kpc_get_curthread_counters(uint32_t *inoutcount, uint64_t *buf); |
119 | |
120 | /* Given a config, how many counters and config registers there are */ |
121 | extern uint32_t kpc_get_counter_count(uint32_t classes); |
122 | extern uint32_t kpc_get_config_count(uint32_t classes); |
123 | |
124 | /* enable/disable thread counting */ |
125 | extern uint32_t kpc_get_thread_counting(void); |
126 | extern int kpc_set_thread_counting(uint32_t classes); |
127 | |
128 | /* get and set config registers */ |
129 | extern int kpc_get_config(uint32_t classes, kpc_config_t *current_config); |
130 | extern int kpc_set_config(uint32_t classes, kpc_config_t *new_config); |
131 | |
132 | /* get and set PMI period */ |
133 | extern int kpc_get_period(uint32_t classes, uint64_t *period); |
134 | extern int kpc_set_period(uint32_t classes, uint64_t *period); |
135 | |
136 | /* get and set kperf actionid */ |
137 | extern int kpc_get_actionid(uint32_t classes, uint32_t *actionid); |
138 | extern int kpc_set_actionid(uint32_t classes, uint32_t *actionid); |
139 | |
140 | /* hooks on thread create and delete */ |
141 | extern void kpc_thread_create(thread_t thread); |
142 | extern void kpc_thread_destroy(thread_t thread); |
143 | |
144 | /* allocate a buffer big enough for all counters */ |
145 | extern uint64_t *kpc_counterbuf_alloc(void); |
146 | extern void kpc_counterbuf_free(uint64_t*); |
147 | extern uint32_t kpc_get_counterbuf_size(void); |
148 | |
149 | /* whether we're currently accounting into threads */ |
150 | extern int kpc_threads_counting; |
151 | |
152 | /* AST callback for KPC */ |
153 | extern void kpc_thread_ast_handler( thread_t thread ); |
154 | |
155 | #if XNU_KERNEL_PRIVATE |
156 | int kpc_set_config_kernel(uint32_t classes, kpc_config_t *new_config); |
157 | #endif /* XNU_KERNEL_PRIVATE */ |
158 | |
159 | #ifdef MACH_KERNEL_PRIVATE |
160 | |
161 | /* context switch callback for KPC */ |
162 | |
163 | extern boolean_t kpc_off_cpu_active; |
164 | |
165 | extern void kpc_off_cpu_internal(thread_t thread); |
166 | extern void kpc_off_cpu_update(void); |
167 | |
168 | static inline void |
169 | kpc_off_cpu(thread_t thread) |
170 | { |
171 | if (__improbable(kpc_off_cpu_active)) { |
172 | kpc_off_cpu_internal(thread); |
173 | } |
174 | } |
175 | |
176 | #endif /* defined(MACH_KERNEL_PRIVATE) */ |
177 | |
178 | /* acquire/release the counters used by the Power Manager */ |
179 | extern int kpc_force_all_ctrs( task_t task, int val ); |
180 | extern int kpc_get_force_all_ctrs( void ); |
181 | |
182 | /* arch-specific routine for acquire/release the counters used by the Power Manager */ |
183 | extern int kpc_force_all_ctrs_arch( task_t task, int val ); |
184 | |
185 | extern int kpc_set_sw_inc( uint32_t mask ); |
186 | |
187 | /* |
188 | * Register the Power Manager as a PMCs user. |
189 | * |
190 | * This is a deprecated function used by old Power Managers, new Power Managers |
191 | * should use the @em kpc_reserve_pm_counters() function. This function actually |
192 | * calls @em kpc_reserve_pm_counters() with the following arguments: |
193 | * - handler = handler |
194 | * - pmc_mask = 0x83 |
195 | * - custom_config = TRUE |
196 | * |
197 | * See @em kpc_reserve_pm_counters() for more details about the return value. |
198 | */ |
199 | extern boolean_t kpc_register_pm_handler(void (*handler)(boolean_t)); |
200 | |
201 | /* |
202 | * Register the Power Manager as a PMCs user. |
203 | * |
204 | * @param handler |
205 | * Notification callback to use when PMCs are acquired/released by a task. |
206 | * Power management must acknowledge the change using kpc_pm_acknowledge. |
207 | * |
208 | * @param pmc_mask |
209 | * Bitmask of the configurable PMCs used by the Power Manager. The number of bits |
210 | * set must less or equal than the number of configurable counters |
211 | * available on the SoC. |
212 | * |
213 | * @param custom_config |
214 | * If custom_config=TRUE, the legacy sharing mode is enabled, otherwise the |
215 | * Modern Sharing mode is enabled. These modes are explained in more details in |
216 | * the kperf documentation. |
217 | * |
218 | * @return |
219 | * FALSE if a task has acquired all the PMCs, otherwise TRUE and the Power |
220 | * Manager can start using the reserved PMCs. |
221 | */ |
222 | extern boolean_t kpc_reserve_pm_counters(uint64_t pmc_mask, kpc_pm_handler_t handler, |
223 | boolean_t custom_config); |
224 | |
225 | /* |
226 | * Unregister the Power Manager as a PMCs user, and release the previously |
227 | * reserved counters. |
228 | */ |
229 | extern void kpc_release_pm_counters(void); |
230 | |
231 | /* |
232 | * Acknowledge the callback that PMCs are available to power management. |
233 | * |
234 | * @param available_to_pm Whether the counters were made available to power |
235 | * management in the callback. Pass in whatever was passed into the handler |
236 | * function. After this point, power management is able to use POWER_CLASS |
237 | * counters. |
238 | */ |
239 | extern void kpc_pm_acknowledge(boolean_t available_to_pm); |
240 | |
241 | /* |
242 | * Is the PMU used by both the power manager and userspace? |
243 | * |
244 | * This is true when the power manager has been registered. It disables certain |
245 | * counter configurations (like RAWPMU) that are incompatible with sharing |
246 | * counters. |
247 | */ |
248 | extern boolean_t kpc_multiple_clients(void); |
249 | |
250 | /* |
251 | * Is kpc controlling the fixed counters? |
252 | * |
253 | * This returns false when the power manager has requested custom configuration |
254 | * control. |
255 | */ |
256 | extern boolean_t kpc_controls_fixed_counters(void); |
257 | |
258 | /* |
259 | * Is kpc controlling a specific PMC ? |
260 | */ |
261 | extern boolean_t kpc_controls_counter(uint32_t ctr); |
262 | |
263 | |
264 | extern void kpc_idle(void); |
265 | extern void kpc_idle_exit(void); |
266 | |
267 | |
268 | /* |
269 | * KPC PRIVATE |
270 | */ |
271 | |
272 | extern uint32_t kpc_actionid[KPC_MAX_COUNTERS]; |
273 | |
274 | /* handler for mp operations */ |
275 | struct kpc_config_remote { |
276 | uint32_t classes; |
277 | kpc_config_t *configv; |
278 | uint64_t pmc_mask; |
279 | bool allow_list; |
280 | }; |
281 | |
282 | /* handler for mp operations */ |
283 | struct kpc_running_remote { |
284 | uint32_t classes; /* classes to run */ |
285 | uint64_t cfg_target_mask; /* configurable counters selected */ |
286 | uint64_t cfg_state_mask; /* configurable counters new state */ |
287 | }; |
288 | |
289 | /* handler for mp operations */ |
290 | struct kpc_get_counters_remote { |
291 | uint32_t classes; |
292 | uint32_t nb_counters; |
293 | uint32_t buf_stride; |
294 | uint64_t *buf; |
295 | }; |
296 | |
297 | int kpc_get_all_cpus_counters(uint32_t classes, int *curcpu, uint64_t *buf); |
298 | int kpc_get_curcpu_counters(uint32_t classes, int *curcpu, uint64_t *buf); |
299 | int kpc_get_fixed_counters(uint64_t *counterv); |
300 | int kpc_get_configurable_counters(uint64_t *counterv, uint64_t pmc_mask); |
301 | boolean_t kpc_is_running_fixed(void); |
302 | boolean_t kpc_is_running_configurable(uint64_t pmc_mask); |
303 | uint32_t kpc_fixed_count(void); |
304 | uint32_t kpc_configurable_count(void); |
305 | uint32_t kpc_fixed_config_count(void); |
306 | uint32_t kpc_configurable_config_count(uint64_t pmc_mask); |
307 | uint32_t kpc_rawpmu_config_count(void); |
308 | int kpc_get_fixed_config(kpc_config_t *configv); |
309 | int kpc_get_configurable_config(kpc_config_t *configv, uint64_t pmc_mask); |
310 | int kpc_get_rawpmu_config(kpc_config_t *configv); |
311 | uint64_t kpc_fixed_max(void); |
312 | uint64_t kpc_configurable_max(void); |
313 | int kpc_set_config_arch(struct kpc_config_remote *mp_config); |
314 | int kpc_set_period_arch(struct kpc_config_remote *mp_config); |
315 | |
316 | __options_decl(kperf_kpc_flags_t, uint16_t, { |
317 | KPC_KERNEL_PC = 0x01, // the PC is a kernel address |
318 | KPC_KERNEL_COUNTING = 0x02, // the counter counts while running in the kernel |
319 | KPC_USER_COUNTING = 0x04, // the counter counts while running in user space |
320 | KPC_CAPTURED_PC = 0x08, // the PC was captured by hardware |
321 | }); |
322 | |
323 | void kpc_sample_kperf(uint32_t actionid, uint32_t counter, uint64_t config, |
324 | uint64_t count, uintptr_t pc, kperf_kpc_flags_t flags); |
325 | |
326 | int kpc_set_running_arch(struct kpc_running_remote *mp_config); |
327 | |
328 | |
329 | /* |
330 | * Helpers |
331 | */ |
332 | |
333 | /* count the number of bits set */ |
334 | extern uint8_t kpc_popcount(uint64_t value); |
335 | |
336 | /* for a set of classes, retrieve the configurable PMCs mask */ |
337 | extern uint64_t kpc_get_configurable_pmc_mask(uint32_t classes); |
338 | |
339 | __END_DECLS |
340 | |
341 | #endif /* !defined(KERN_KPC_H) */ |
342 | |