1/*
2 * Copyright (c) 2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#ifndef KERN_KPC_H
30#define KERN_KPC_H
31
32/* Kernel interfaces to KPC PMC infrastructure. */
33
34#include <kern/thread.h> /* thread_* */
35#include <machine/machine_kpc.h>
36
37__BEGIN_DECLS
38
39typedef uint64_t kpc_config_t;
40
41/* cross-platform class constants */
42#define KPC_CLASS_FIXED (0)
43#define KPC_CLASS_CONFIGURABLE (1)
44#define KPC_CLASS_POWER (2)
45#define KPC_CLASS_RAWPMU (3)
46
47#define KPC_CLASS_FIXED_MASK (1u << KPC_CLASS_FIXED)
48#define KPC_CLASS_CONFIGURABLE_MASK (1u << KPC_CLASS_CONFIGURABLE)
49#define KPC_CLASS_POWER_MASK (1u << KPC_CLASS_POWER)
50#define KPC_CLASS_RAWPMU_MASK (1u << KPC_CLASS_RAWPMU)
51
52#define KPC_PMU_ERROR (0)
53#define KPC_PMU_INTEL_V3 (1)
54#define KPC_PMU_ARM_APPLE (2)
55#define KPC_PMU_INTEL_V2 (3)
56#define KPC_PMU_ARM_V2 (4)
57
58#define KPC_ALL_CPUS (1u << 31)
59
60/* action id setters/getters */
61#define FIXED_ACTIONID(ctr) (kpc_actionid[(ctr)])
62#define CONFIGURABLE_ACTIONID(ctr) (kpc_actionid[(ctr) + kpc_fixed_count()])
63
64/* reload counter setters/getters */
65#define FIXED_RELOAD(ctr) (current_cpu_datap()->cpu_kpc_reload[(ctr)])
66#define FIXED_RELOAD_CPU(cpu, ctr) (cpu_datap(cpu)->cpu_kpc_reload[(ctr)])
67#define CONFIGURABLE_RELOAD(ctr) (current_cpu_datap()->cpu_kpc_reload[(ctr) + kpc_fixed_count()])
68#define CONFIGURABLE_RELOAD_CPU(cpu, ctr) (cpu_datap(cpu)->cpu_kpc_reload[(ctr) + kpc_fixed_count()])
69
70/* shadow counter setters/getters */
71#define FIXED_SHADOW(ctr) (current_cpu_datap()->cpu_kpc_shadow[(ctr)])
72#define FIXED_SHADOW_CPU(cpu, ctr) (cpu_datap(cpu)->cpu_kpc_shadow[(ctr)])
73#define CONFIGURABLE_SHADOW(ctr) (current_cpu_datap()->cpu_kpc_shadow[(ctr) + kpc_fixed_count()])
74#define CONFIGURABLE_SHADOW_CPU(cpu, ctr) (cpu_datap(cpu)->cpu_kpc_shadow[(ctr) + kpc_fixed_count()])
75
76/**
77 * Callback for notification when PMCs are acquired/released by a task. The
78 * argument is equal to TRUE if the Power Manager (PM) can use its reserved PMCs.
79 * Otherwise, the argument is equal to FALSE.
80 */
81typedef void (*kpc_pm_handler_t)(boolean_t);
82
83struct cpu_data;
84extern boolean_t kpc_register_cpu(struct cpu_data *cpu_data);
85extern void kpc_unregister_cpu(struct cpu_data *cpu_data);
86
87extern bool kpc_supported;
88
89/* bootstrap */
90extern void kpc_init(void);
91
92/* Architecture specific initialisation */
93extern void kpc_arch_init(void);
94
95/* Get the bitmask of available classes */
96extern uint32_t kpc_get_classes(void);
97
98/* Get the bitmask of currently running counter classes */
99extern uint32_t kpc_get_running(void);
100
101/* Get the version of KPC that's being run */
102extern int kpc_get_pmu_version(void);
103
104/* Set the bitmask of currently running counter classes. Specify
105 * classes = 0 to stop counters
106 */
107extern int kpc_set_running(uint32_t classes);
108
109/* Read CPU counters */
110extern int kpc_get_cpu_counters(boolean_t all_cpus, uint32_t classes,
111 int *curcpu, uint64_t *buf);
112
113/* Read shadow counters */
114extern int kpc_get_shadow_counters( boolean_t all_cpus, uint32_t classes,
115 int *curcpu, uint64_t *buf );
116
117/* Read current thread's counter accumulations */
118extern int kpc_get_curthread_counters(uint32_t *inoutcount, uint64_t *buf);
119
120/* Given a config, how many counters and config registers there are */
121extern uint32_t kpc_get_counter_count(uint32_t classes);
122extern uint32_t kpc_get_config_count(uint32_t classes);
123
124/* enable/disable thread counting */
125extern uint32_t kpc_get_thread_counting(void);
126extern int kpc_set_thread_counting(uint32_t classes);
127
128/* get and set config registers */
129extern int kpc_get_config(uint32_t classes, kpc_config_t *current_config);
130extern int kpc_set_config(uint32_t classes, kpc_config_t *new_config);
131
132/* get and set PMI period */
133extern int kpc_get_period(uint32_t classes, uint64_t *period);
134extern int kpc_set_period(uint32_t classes, uint64_t *period);
135
136/* get and set kperf actionid */
137extern int kpc_get_actionid(uint32_t classes, uint32_t *actionid);
138extern int kpc_set_actionid(uint32_t classes, uint32_t *actionid);
139
140/* hooks on thread create and delete */
141extern void kpc_thread_create(thread_t thread);
142extern void kpc_thread_destroy(thread_t thread);
143
144/* allocate a buffer big enough for all counters */
145extern uint64_t *kpc_counterbuf_alloc(void);
146extern void kpc_counterbuf_free(uint64_t*);
147extern uint32_t kpc_get_counterbuf_size(void);
148
149/* whether we're currently accounting into threads */
150extern int kpc_threads_counting;
151
152/* AST callback for KPC */
153extern void kpc_thread_ast_handler( thread_t thread );
154
155#if XNU_KERNEL_PRIVATE
156int kpc_set_config_kernel(uint32_t classes, kpc_config_t *new_config);
157#endif /* XNU_KERNEL_PRIVATE */
158
159#ifdef MACH_KERNEL_PRIVATE
160
161/* context switch callback for KPC */
162
163extern boolean_t kpc_off_cpu_active;
164
165extern void kpc_off_cpu_internal(thread_t thread);
166extern void kpc_off_cpu_update(void);
167
168static inline void
169kpc_off_cpu(thread_t thread)
170{
171 if (__improbable(kpc_off_cpu_active)) {
172 kpc_off_cpu_internal(thread);
173 }
174}
175
176#endif /* defined(MACH_KERNEL_PRIVATE) */
177
178/* acquire/release the counters used by the Power Manager */
179extern int kpc_force_all_ctrs( task_t task, int val );
180extern int kpc_get_force_all_ctrs( void );
181
182/* arch-specific routine for acquire/release the counters used by the Power Manager */
183extern int kpc_force_all_ctrs_arch( task_t task, int val );
184
185extern int kpc_set_sw_inc( uint32_t mask );
186
187/*
188 * Register the Power Manager as a PMCs user.
189 *
190 * This is a deprecated function used by old Power Managers, new Power Managers
191 * should use the @em kpc_reserve_pm_counters() function. This function actually
192 * calls @em kpc_reserve_pm_counters() with the following arguments:
193 * - handler = handler
194 * - pmc_mask = 0x83
195 * - custom_config = TRUE
196 *
197 * See @em kpc_reserve_pm_counters() for more details about the return value.
198 */
199extern boolean_t kpc_register_pm_handler(void (*handler)(boolean_t));
200
201/*
202 * Register the Power Manager as a PMCs user.
203 *
204 * @param handler
205 * Notification callback to use when PMCs are acquired/released by a task.
206 * Power management must acknowledge the change using kpc_pm_acknowledge.
207 *
208 * @param pmc_mask
209 * Bitmask of the configurable PMCs used by the Power Manager. The number of bits
210 * set must less or equal than the number of configurable counters
211 * available on the SoC.
212 *
213 * @param custom_config
214 * If custom_config=TRUE, the legacy sharing mode is enabled, otherwise the
215 * Modern Sharing mode is enabled. These modes are explained in more details in
216 * the kperf documentation.
217 *
218 * @return
219 * FALSE if a task has acquired all the PMCs, otherwise TRUE and the Power
220 * Manager can start using the reserved PMCs.
221 */
222extern boolean_t kpc_reserve_pm_counters(uint64_t pmc_mask, kpc_pm_handler_t handler,
223 boolean_t custom_config);
224
225/*
226 * Unregister the Power Manager as a PMCs user, and release the previously
227 * reserved counters.
228 */
229extern void kpc_release_pm_counters(void);
230
231/*
232 * Acknowledge the callback that PMCs are available to power management.
233 *
234 * @param available_to_pm Whether the counters were made available to power
235 * management in the callback. Pass in whatever was passed into the handler
236 * function. After this point, power management is able to use POWER_CLASS
237 * counters.
238 */
239extern void kpc_pm_acknowledge(boolean_t available_to_pm);
240
241/*
242 * Is the PMU used by both the power manager and userspace?
243 *
244 * This is true when the power manager has been registered. It disables certain
245 * counter configurations (like RAWPMU) that are incompatible with sharing
246 * counters.
247 */
248extern boolean_t kpc_multiple_clients(void);
249
250/*
251 * Is kpc controlling the fixed counters?
252 *
253 * This returns false when the power manager has requested custom configuration
254 * control.
255 */
256extern boolean_t kpc_controls_fixed_counters(void);
257
258/*
259 * Is kpc controlling a specific PMC ?
260 */
261extern boolean_t kpc_controls_counter(uint32_t ctr);
262
263
264extern void kpc_idle(void);
265extern void kpc_idle_exit(void);
266
267
268/*
269 * KPC PRIVATE
270 */
271
272extern uint32_t kpc_actionid[KPC_MAX_COUNTERS];
273
274/* handler for mp operations */
275struct kpc_config_remote {
276 uint32_t classes;
277 kpc_config_t *configv;
278 uint64_t pmc_mask;
279 bool allow_list;
280};
281
282/* handler for mp operations */
283struct kpc_running_remote {
284 uint32_t classes; /* classes to run */
285 uint64_t cfg_target_mask; /* configurable counters selected */
286 uint64_t cfg_state_mask; /* configurable counters new state */
287};
288
289/* handler for mp operations */
290struct kpc_get_counters_remote {
291 uint32_t classes;
292 uint32_t nb_counters;
293 uint32_t buf_stride;
294 uint64_t *buf;
295};
296
297int kpc_get_all_cpus_counters(uint32_t classes, int *curcpu, uint64_t *buf);
298int kpc_get_curcpu_counters(uint32_t classes, int *curcpu, uint64_t *buf);
299int kpc_get_fixed_counters(uint64_t *counterv);
300int kpc_get_configurable_counters(uint64_t *counterv, uint64_t pmc_mask);
301boolean_t kpc_is_running_fixed(void);
302boolean_t kpc_is_running_configurable(uint64_t pmc_mask);
303uint32_t kpc_fixed_count(void);
304uint32_t kpc_configurable_count(void);
305uint32_t kpc_fixed_config_count(void);
306uint32_t kpc_configurable_config_count(uint64_t pmc_mask);
307uint32_t kpc_rawpmu_config_count(void);
308int kpc_get_fixed_config(kpc_config_t *configv);
309int kpc_get_configurable_config(kpc_config_t *configv, uint64_t pmc_mask);
310int kpc_get_rawpmu_config(kpc_config_t *configv);
311uint64_t kpc_fixed_max(void);
312uint64_t kpc_configurable_max(void);
313int kpc_set_config_arch(struct kpc_config_remote *mp_config);
314int kpc_set_period_arch(struct kpc_config_remote *mp_config);
315
316__options_decl(kperf_kpc_flags_t, uint16_t, {
317 KPC_KERNEL_PC = 0x01, // the PC is a kernel address
318 KPC_KERNEL_COUNTING = 0x02, // the counter counts while running in the kernel
319 KPC_USER_COUNTING = 0x04, // the counter counts while running in user space
320 KPC_CAPTURED_PC = 0x08, // the PC was captured by hardware
321});
322
323void kpc_sample_kperf(uint32_t actionid, uint32_t counter, uint64_t config,
324 uint64_t count, uintptr_t pc, kperf_kpc_flags_t flags);
325
326int kpc_set_running_arch(struct kpc_running_remote *mp_config);
327
328
329/*
330 * Helpers
331 */
332
333/* count the number of bits set */
334extern uint8_t kpc_popcount(uint64_t value);
335
336/* for a set of classes, retrieve the configurable PMCs mask */
337extern uint64_t kpc_get_configurable_pmc_mask(uint32_t classes);
338
339__END_DECLS
340
341#endif /* !defined(KERN_KPC_H) */
342