1 | /* |
2 | * Copyright (c) 2012 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <kern/debug.h> |
30 | #include <kern/kalloc.h> |
31 | #include <sys/param.h> |
32 | #include <sys/mman.h> |
33 | #include <sys/stat.h> |
34 | #include <sys/sysctl.h> |
35 | #include <libkern/libkern.h> |
36 | #include <kern/assert.h> |
37 | |
38 | #include <kern/kpc.h> |
39 | #include <sys/ktrace.h> |
40 | |
41 | #include <pexpert/pexpert.h> |
42 | #include <kperf/kperf.h> |
43 | |
44 | /* Various sysctl requests */ |
45 | #define REQ_CLASSES (1) |
46 | #define REQ_COUNTING (2) |
47 | #define REQ_THREAD_COUNTING (3) |
48 | #define REQ_CONFIG_COUNT (4) |
49 | #define REQ_COUNTER_COUNT (5) |
50 | #define REQ_THREAD_COUNTERS (6) |
51 | #define REQ_COUNTERS (7) |
52 | #define REQ_SHADOW_COUNTERS (8) |
53 | #define REQ_CONFIG (9) |
54 | #define REQ_PERIOD (10) |
55 | #define REQ_ACTIONID (11) |
56 | #define REQ_SW_INC (14) |
57 | #define REQ_PMU_VERSION (15) |
58 | |
59 | /* Type-munging casts */ |
60 | typedef int (*getint_t)(void); |
61 | typedef int (*setint_t)(int); |
62 | |
63 | static int kpc_initted = 0; |
64 | |
65 | static lck_grp_attr_t *sysctl_lckgrp_attr = NULL; |
66 | static lck_grp_t *sysctl_lckgrp = NULL; |
67 | static lck_mtx_t sysctl_lock; |
68 | |
69 | /* |
70 | * Another element is needed to hold the CPU number when getting counter values. |
71 | */ |
72 | #define KPC_MAX_BUF_LEN (KPC_MAX_COUNTERS_COPIED + 1) |
73 | |
74 | typedef int (*setget_func_t)(int); |
75 | |
76 | void |
77 | kpc_init(void) |
78 | { |
79 | sysctl_lckgrp_attr = lck_grp_attr_alloc_init(); |
80 | sysctl_lckgrp = lck_grp_alloc_init("kpc" , sysctl_lckgrp_attr); |
81 | lck_mtx_init(&sysctl_lock, sysctl_lckgrp, LCK_ATTR_NULL); |
82 | |
83 | kpc_arch_init(); |
84 | kpc_common_init(); |
85 | kpc_thread_init(); |
86 | |
87 | kpc_initted = 1; |
88 | } |
89 | |
90 | static uint64_t * |
91 | kpc_get_bigarray(uint32_t *size_out) |
92 | { |
93 | static uint64_t *bigarray = NULL; |
94 | |
95 | LCK_MTX_ASSERT(&sysctl_lock, LCK_MTX_ASSERT_OWNED); |
96 | |
97 | uint32_t size = kpc_get_counterbuf_size() + sizeof(uint64_t); |
98 | *size_out = size; |
99 | |
100 | if (bigarray) { |
101 | return bigarray; |
102 | } |
103 | |
104 | /* |
105 | * Another element is needed to hold the CPU number when getting counter |
106 | * values. |
107 | */ |
108 | bigarray = kalloc_tag(size, VM_KERN_MEMORY_DIAG); |
109 | assert(bigarray != NULL); |
110 | return bigarray; |
111 | } |
112 | |
113 | /* abstract sysctl handlers */ |
114 | static int |
115 | sysctl_get_int( struct sysctl_oid *oidp, struct sysctl_req *req, |
116 | uint32_t value ) |
117 | { |
118 | int error = 0; |
119 | |
120 | /* copy out the old value */ |
121 | error = sysctl_handle_int(oidp, &value, 0, req); |
122 | |
123 | return error; |
124 | } |
125 | |
126 | static int |
127 | sysctl_set_int( struct sysctl_req *req, int (*set_func)(int)) |
128 | { |
129 | int error = 0; |
130 | int value = 0; |
131 | |
132 | error = SYSCTL_IN( req, &value, sizeof(value) ); |
133 | if( error ) |
134 | return error; |
135 | |
136 | error = set_func( value ); |
137 | |
138 | return error; |
139 | } |
140 | |
141 | static int |
142 | sysctl_getset_int( struct sysctl_oid *oidp, struct sysctl_req *req, |
143 | int (*get_func)(void), int (*set_func)(int) ) |
144 | { |
145 | int error = 0; |
146 | uint32_t value = 0; |
147 | |
148 | /* get the old value and process it */ |
149 | value = get_func(); |
150 | |
151 | /* copy out the old value, get the new value */ |
152 | error = sysctl_handle_int(oidp, &value, 0, req); |
153 | if (error || !req->newptr) |
154 | return (error); |
155 | |
156 | /* if that worked, and we're writing... */ |
157 | error = set_func( value ); |
158 | |
159 | return error; |
160 | } |
161 | |
162 | |
163 | static int |
164 | sysctl_setget_int( struct sysctl_req *req, |
165 | int (*setget_func)(int) ) |
166 | { |
167 | int error = 0; |
168 | int value = 0; |
169 | |
170 | error = SYSCTL_IN( req, &value, sizeof(value) ); |
171 | if( error ) |
172 | return error; |
173 | |
174 | value = setget_func(value); |
175 | |
176 | error = SYSCTL_OUT( req, &value, sizeof(value) ); |
177 | |
178 | return error; |
179 | } |
180 | |
181 | static int |
182 | sysctl_kpc_get_counters(uint32_t counters, |
183 | uint32_t *size, void *buf) |
184 | { |
185 | uint64_t *ctr_buf = (uint64_t*)buf; |
186 | int curcpu; |
187 | uint32_t count; |
188 | |
189 | count = kpc_get_cpu_counters(counters & KPC_ALL_CPUS, |
190 | counters, |
191 | &curcpu, &ctr_buf[1]); |
192 | if (!count) |
193 | return EINVAL; |
194 | |
195 | ctr_buf[0] = curcpu; |
196 | |
197 | *size = (count+1) * sizeof(uint64_t); |
198 | |
199 | return 0; |
200 | } |
201 | |
202 | static int |
203 | sysctl_kpc_get_shadow_counters(uint32_t counters, |
204 | uint32_t *size, void *buf) |
205 | { |
206 | uint64_t *ctr_buf = (uint64_t*)buf; |
207 | int curcpu; |
208 | uint32_t count; |
209 | |
210 | count = kpc_get_shadow_counters(counters & KPC_ALL_CPUS, |
211 | counters, |
212 | &curcpu, &ctr_buf[1]); |
213 | |
214 | if (!count) |
215 | return EINVAL; |
216 | |
217 | ctr_buf[0] = curcpu; |
218 | |
219 | *size = (count+1) * sizeof(uint64_t); |
220 | |
221 | return 0; |
222 | } |
223 | |
224 | static int |
225 | sysctl_kpc_get_thread_counters(uint32_t tid, |
226 | uint32_t *size, void *buf) |
227 | { |
228 | uint32_t count = *size / sizeof(uint64_t); |
229 | int r; |
230 | |
231 | if( tid != 0 ) |
232 | return EINVAL; |
233 | |
234 | r = kpc_get_curthread_counters(&count, buf); |
235 | if( !r ) |
236 | *size = count * sizeof(uint64_t); |
237 | |
238 | return r; |
239 | } |
240 | |
241 | static int |
242 | sysctl_kpc_get_config(uint32_t classes, void* buf) |
243 | { |
244 | return kpc_get_config( classes, buf ); |
245 | } |
246 | |
247 | static int |
248 | sysctl_kpc_set_config(uint32_t classes, void* buf) |
249 | { |
250 | /* userspace cannot reconfigure the power class */ |
251 | if (classes & KPC_CLASS_POWER_MASK) |
252 | return (EPERM); |
253 | return kpc_set_config( classes, buf); |
254 | } |
255 | |
256 | static int |
257 | sysctl_kpc_get_period(uint32_t classes, void* buf) |
258 | { |
259 | return kpc_get_period( classes, buf ); |
260 | } |
261 | |
262 | static int |
263 | sysctl_kpc_set_period(uint32_t classes, void* buf) |
264 | { |
265 | /* userspace cannot reconfigure the power class */ |
266 | if (classes & KPC_CLASS_POWER_MASK) |
267 | return (EPERM); |
268 | return kpc_set_period( classes, buf); |
269 | } |
270 | |
271 | static int |
272 | sysctl_kpc_get_actionid(uint32_t classes, void* buf) |
273 | { |
274 | return kpc_get_actionid( classes, buf ); |
275 | } |
276 | |
277 | static int |
278 | sysctl_kpc_set_actionid(uint32_t classes, void* buf) |
279 | { |
280 | return kpc_set_actionid( classes, buf); |
281 | } |
282 | |
283 | |
284 | static int |
285 | sysctl_get_bigarray(struct sysctl_req *req, |
286 | int (*get_fn)(uint32_t, uint32_t*, void*)) |
287 | { |
288 | uint32_t bufsize = 0; |
289 | uint64_t *buf = kpc_get_bigarray(&bufsize); |
290 | uint32_t arg = 0; |
291 | |
292 | /* get the argument */ |
293 | int error = SYSCTL_IN(req, &arg, sizeof(arg)); |
294 | if (error) { |
295 | return error; |
296 | } |
297 | |
298 | error = get_fn(arg, &bufsize, buf); |
299 | if (!error) { |
300 | error = SYSCTL_OUT(req, buf, bufsize); |
301 | } |
302 | |
303 | return error; |
304 | } |
305 | |
306 | /* given a config word, how many bytes does it take? */ |
307 | static int |
308 | sysctl_config_size( uint32_t config ) |
309 | { |
310 | return kpc_get_config_count(config) * sizeof(kpc_config_t); |
311 | } |
312 | |
313 | static int |
314 | sysctl_counter_size( uint32_t classes ) |
315 | { |
316 | return kpc_get_counter_count(classes) * sizeof(uint64_t); |
317 | } |
318 | |
319 | static int |
320 | sysctl_actionid_size( uint32_t classes ) |
321 | { |
322 | return kpc_get_counter_count(classes) * sizeof(int32_t); |
323 | } |
324 | |
325 | static int |
326 | sysctl_getset_bigarray(struct sysctl_req *req, int (*size_fn)(uint32_t arg), |
327 | int (*get_fn)(uint32_t, void*), int (*set_fn)(uint32_t, void*)) |
328 | { |
329 | int error = 0; |
330 | uint64_t arg; |
331 | |
332 | uint32_t bufsize = 0; |
333 | uint64_t *buf = kpc_get_bigarray(&bufsize); |
334 | |
335 | /* get the config word */ |
336 | error = SYSCTL_IN(req, &arg, sizeof(arg)); |
337 | if (error) { |
338 | return error; |
339 | } |
340 | |
341 | /* Determine the size of registers to modify. */ |
342 | uint32_t regsize = size_fn((uint32_t)arg); |
343 | if (regsize == 0 || regsize > bufsize) { |
344 | return EINVAL; |
345 | } |
346 | |
347 | /* if writing */ |
348 | if (req->newptr) { |
349 | /* copy the rest -- SYSCTL_IN knows the copyin should be shifted */ |
350 | error = SYSCTL_IN(req, buf, regsize); |
351 | |
352 | /* SYSCTL_IN failure means only need to read */ |
353 | if (!error) { |
354 | error = set_fn((uint32_t)arg, buf); |
355 | if (error) { |
356 | return error; |
357 | } |
358 | } |
359 | } |
360 | |
361 | /* if reading */ |
362 | if (req->oldptr) { |
363 | error = get_fn((uint32_t)arg, buf); |
364 | if (error) { |
365 | return error; |
366 | } |
367 | |
368 | error = SYSCTL_OUT(req, buf, regsize); |
369 | } |
370 | |
371 | return error; |
372 | } |
373 | |
374 | static int |
375 | kpc_sysctl SYSCTL_HANDLER_ARGS |
376 | { |
377 | int ret; |
378 | |
379 | // __unused struct sysctl_oid *unused_oidp = oidp; |
380 | (void)arg2; |
381 | |
382 | if (!kpc_initted) { |
383 | panic("kpc_init not called" ); |
384 | } |
385 | |
386 | if (!kpc_supported) { |
387 | return ENOTSUP; |
388 | } |
389 | |
390 | ktrace_lock(); |
391 | |
392 | // Most sysctls require an access check, but a few are public. |
393 | switch( (uintptr_t) arg1 ) { |
394 | case REQ_CLASSES: |
395 | case REQ_CONFIG_COUNT: |
396 | case REQ_COUNTER_COUNT: |
397 | // These read-only sysctls are public. |
398 | break; |
399 | |
400 | default: |
401 | // Require kperf access to read or write anything else. |
402 | // This is either root or the blessed pid. |
403 | if ((ret = ktrace_read_check())) { |
404 | ktrace_unlock(); |
405 | return ret; |
406 | } |
407 | break; |
408 | } |
409 | |
410 | ktrace_unlock(); |
411 | |
412 | lck_mtx_lock(&sysctl_lock); |
413 | |
414 | /* which request */ |
415 | switch( (uintptr_t) arg1 ) |
416 | { |
417 | case REQ_CLASSES: |
418 | ret = sysctl_get_int( oidp, req, |
419 | kpc_get_classes() ); |
420 | break; |
421 | case REQ_COUNTING: |
422 | ret = sysctl_getset_int( oidp, req, |
423 | (getint_t)kpc_get_running, |
424 | (setint_t)kpc_set_running ); |
425 | break; |
426 | case REQ_THREAD_COUNTING: |
427 | ret = sysctl_getset_int( oidp, req, |
428 | (getint_t)kpc_get_thread_counting, |
429 | (setint_t)kpc_set_thread_counting ); |
430 | break; |
431 | |
432 | case REQ_CONFIG_COUNT: |
433 | ret = sysctl_setget_int( req, |
434 | (setget_func_t)kpc_get_config_count ); |
435 | break; |
436 | |
437 | case REQ_COUNTER_COUNT: |
438 | ret = sysctl_setget_int( req, |
439 | (setget_func_t)kpc_get_counter_count ); |
440 | break; |
441 | |
442 | |
443 | case REQ_THREAD_COUNTERS: |
444 | ret = sysctl_get_bigarray( req, sysctl_kpc_get_thread_counters ); |
445 | break; |
446 | |
447 | case REQ_COUNTERS: |
448 | ret = sysctl_get_bigarray( req, sysctl_kpc_get_counters ); |
449 | break; |
450 | |
451 | case REQ_SHADOW_COUNTERS: |
452 | ret = sysctl_get_bigarray( req, sysctl_kpc_get_shadow_counters ); |
453 | break; |
454 | |
455 | case REQ_CONFIG: |
456 | ret = sysctl_getset_bigarray( req, |
457 | sysctl_config_size, |
458 | sysctl_kpc_get_config, |
459 | sysctl_kpc_set_config ); |
460 | break; |
461 | |
462 | case REQ_PERIOD: |
463 | ret = sysctl_getset_bigarray( req, |
464 | sysctl_counter_size, |
465 | sysctl_kpc_get_period, |
466 | sysctl_kpc_set_period ); |
467 | break; |
468 | |
469 | case REQ_ACTIONID: |
470 | ret = sysctl_getset_bigarray( req, |
471 | sysctl_actionid_size, |
472 | sysctl_kpc_get_actionid, |
473 | sysctl_kpc_set_actionid ); |
474 | break; |
475 | |
476 | |
477 | case REQ_SW_INC: |
478 | ret = sysctl_set_int( req, (setget_func_t)kpc_set_sw_inc ); |
479 | break; |
480 | |
481 | case REQ_PMU_VERSION: |
482 | ret = sysctl_get_int(oidp, req, kpc_get_pmu_version()); |
483 | break; |
484 | |
485 | default: |
486 | ret = ENOENT; |
487 | break; |
488 | } |
489 | |
490 | lck_mtx_unlock(&sysctl_lock); |
491 | |
492 | return ret; |
493 | } |
494 | |
495 | |
496 | /*** sysctl definitions ***/ |
497 | |
498 | /* root kperf node */ |
499 | SYSCTL_NODE(, OID_AUTO, kpc, CTLFLAG_RW|CTLFLAG_LOCKED, 0, |
500 | "kpc" ); |
501 | |
502 | /* values */ |
503 | SYSCTL_PROC(_kpc, OID_AUTO, classes, |
504 | CTLTYPE_INT|CTLFLAG_RD|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED, |
505 | (void*)REQ_CLASSES, |
506 | sizeof(int), kpc_sysctl, "I" , "Available classes" ); |
507 | |
508 | SYSCTL_PROC(_kpc, OID_AUTO, counting, |
509 | CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED, |
510 | (void*)REQ_COUNTING, |
511 | sizeof(int), kpc_sysctl, "I" , "PMCs counting" ); |
512 | |
513 | SYSCTL_PROC(_kpc, OID_AUTO, thread_counting, |
514 | CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED, |
515 | (void*)REQ_THREAD_COUNTING, |
516 | sizeof(int), kpc_sysctl, "I" , "Thread accumulation" ); |
517 | |
518 | SYSCTL_PROC(_kpc, OID_AUTO, pmu_version, |
519 | CTLTYPE_INT|CTLFLAG_RD|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED, |
520 | (void *)REQ_PMU_VERSION, |
521 | sizeof(int), kpc_sysctl, "I" , "PMU version for hardware" ); |
522 | |
523 | /* faux values */ |
524 | SYSCTL_PROC(_kpc, OID_AUTO, config_count, |
525 | CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED, |
526 | (void*)REQ_CONFIG_COUNT, |
527 | sizeof(int), kpc_sysctl, "S" , "Config count" ); |
528 | |
529 | SYSCTL_PROC(_kpc, OID_AUTO, counter_count, |
530 | CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED, |
531 | (void*)REQ_COUNTER_COUNT, |
532 | sizeof(int), kpc_sysctl, "S" , "Counter count" ); |
533 | |
534 | SYSCTL_PROC(_kpc, OID_AUTO, sw_inc, |
535 | CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED, |
536 | (void*)REQ_SW_INC, |
537 | sizeof(int), kpc_sysctl, "S" , "Software increment" ); |
538 | |
539 | /* arrays */ |
540 | SYSCTL_PROC(_kpc, OID_AUTO, thread_counters, |
541 | CTLFLAG_RD|CTLFLAG_WR|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED, |
542 | (void*)REQ_THREAD_COUNTERS, |
543 | sizeof(uint64_t), kpc_sysctl, |
544 | "QU" , "Current thread counters" ); |
545 | |
546 | SYSCTL_PROC(_kpc, OID_AUTO, counters, |
547 | CTLFLAG_RD|CTLFLAG_WR|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED, |
548 | (void*)REQ_COUNTERS, |
549 | sizeof(uint64_t), kpc_sysctl, |
550 | "QU" , "Current counters" ); |
551 | |
552 | SYSCTL_PROC(_kpc, OID_AUTO, shadow_counters, |
553 | CTLFLAG_RD|CTLFLAG_WR|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED, |
554 | (void*)REQ_SHADOW_COUNTERS, |
555 | sizeof(uint64_t), kpc_sysctl, |
556 | "QU" , "Current shadow counters" ); |
557 | |
558 | SYSCTL_PROC(_kpc, OID_AUTO, config, |
559 | CTLFLAG_RD|CTLFLAG_WR|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED, |
560 | (void*)REQ_CONFIG, |
561 | sizeof(uint64_t), kpc_sysctl, |
562 | "QU" , "Set counter configs" ); |
563 | |
564 | SYSCTL_PROC(_kpc, OID_AUTO, period, |
565 | CTLFLAG_RD|CTLFLAG_WR|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED, |
566 | (void*)REQ_PERIOD, |
567 | sizeof(uint64_t), kpc_sysctl, |
568 | "QU" , "Set counter periods" ); |
569 | |
570 | SYSCTL_PROC(_kpc, OID_AUTO, actionid, |
571 | CTLFLAG_RD|CTLFLAG_WR|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED, |
572 | (void*)REQ_ACTIONID, |
573 | sizeof(uint32_t), kpc_sysctl, |
574 | "QU" , "Set counter actionids" ); |
575 | |
576 | |
577 | |