1/*
2 * Copyright (c) 2014 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <machine/machine_routines.h>
30#include <sys/sysproto.h>
31#include <sys/malloc.h>
32#include <sys/systm.h>
33#include <sys/pgo.h>
34#include <sys/kauth.h>
35#include <security/mac_framework.h>
36#include <libkern/OSKextLib.h>
37
38
39#ifdef PROFILE
40
41/* These __llvm functions are defined in InstrProfiling.h in compiler_rt. That
42 * is a internal header, so we need to re-prototype them here. */
43
44uint64_t __llvm_profile_get_size_for_buffer(void);
45int __llvm_profile_write_buffer(char *Buffer);
46uint64_t __llvm_profile_get_size_for_buffer_internal(const char *DataBegin,
47 const char *DataEnd,
48 const char *CountersBegin,
49 const char *CountersEnd,
50 const char *NamesBegin,
51 const char *NamesEnd);
52int __llvm_profile_write_buffer_internal(char *Buffer,
53 const char *DataBegin,
54 const char *DataEnd,
55 const char *CountersBegin,
56 const char *CountersEnd,
57 const char *NamesBegin,
58 const char *NamesEnd);
59
60extern char __pgo_hib_DataStart __asm("section$start$__HIB$__llvm_prf_data");
61extern char __pgo_hib_DataEnd __asm("section$end$__HIB$__llvm_prf_data");
62extern char __pgo_hib_NamesStart __asm("section$start$__HIB$__llvm_prf_names");
63extern char __pgo_hib_NamesEnd __asm("section$end$__HIB$__llvm_prf_names");
64extern char __pgo_hib_CountersStart __asm("section$start$__HIB$__llvm_prf_cnts");
65extern char __pgo_hib_CountersEnd __asm("section$end$__HIB$__llvm_prf_cnts");
66
67
68static uint64_t
69get_size_for_buffer(int flags)
70{
71 if (flags & PGO_HIB) {
72 return __llvm_profile_get_size_for_buffer_internal(
73 &__pgo_hib_DataStart, &__pgo_hib_DataEnd,
74 &__pgo_hib_CountersStart, &__pgo_hib_CountersEnd,
75 &__pgo_hib_NamesStart, &__pgo_hib_NamesEnd);
76 } else {
77 return __llvm_profile_get_size_for_buffer();
78 }
79}
80
81
82static int
83write_buffer(int flags, char *buffer)
84{
85 if (flags & PGO_HIB) {
86 return __llvm_profile_write_buffer_internal(
87 buffer,
88 &__pgo_hib_DataStart, &__pgo_hib_DataEnd,
89 &__pgo_hib_CountersStart, &__pgo_hib_CountersEnd,
90 &__pgo_hib_NamesStart, &__pgo_hib_NamesEnd);
91 } else {
92 return __llvm_profile_write_buffer(buffer);
93 }
94}
95
96
97#endif
98
99/* this variable is used to signal to the debugger that we'd like it to reset
100 * the counters */
101int kdp_pgo_reset_counters = 0;
102
103/* called in debugger context */
104kern_return_t
105do_pgo_reset_counters()
106{
107#ifdef PROFILE
108 memset(&__pgo_hib_CountersStart, 0,
109 ((uintptr_t)(&__pgo_hib_CountersEnd)) - ((uintptr_t)(&__pgo_hib_CountersStart)));
110#endif
111 OSKextResetPgoCounters();
112 kdp_pgo_reset_counters = 0;
113 return KERN_SUCCESS;
114}
115
116static kern_return_t
117kextpgo_trap()
118{
119 return DebuggerTrapWithState(db_op: DBOP_RESET_PGO_COUNTERS, NULL, NULL, NULL, db_panic_options: 0, NULL, FALSE, db_panic_caller: 0);
120}
121
122static kern_return_t
123pgo_reset_counters()
124{
125 kern_return_t r;
126 boolean_t istate;
127
128 OSKextResetPgoCountersLock();
129
130 istate = ml_set_interrupts_enabled(FALSE);
131
132 kdp_pgo_reset_counters = 1;
133 r = kextpgo_trap();
134
135 ml_set_interrupts_enabled(enable: istate);
136
137 OSKextResetPgoCountersUnlock();
138 return r;
139}
140
141
142/*
143 * returns:
144 * EPERM unless you are root
145 * EINVAL for invalid args.
146 * ENOSYS for not implemented
147 * ERANGE for integer overflow
148 * ENOENT if kext not found
149 * ENOTSUP kext does not support PGO
150 * EIO llvm returned an error. shouldn't ever happen.
151 */
152
153int
154grab_pgo_data(struct proc *p,
155 struct grab_pgo_data_args *uap,
156 register_t *retval)
157{
158 char *buffer = NULL;
159 uint64_t size64 = 0;
160 int err = 0;
161
162 (void) p;
163
164 if (!kauth_cred_issuser(cred: kauth_cred_get())) {
165 err = EPERM;
166 goto out;
167 }
168
169#if CONFIG_MACF
170 err = mac_system_check_info(kauth_cred_get(), info_type: "kern.profiling_data");
171 if (err) {
172 goto out;
173 }
174#endif
175
176 if (uap->flags & ~PGO_ALL_FLAGS ||
177 uap->size < 0 ||
178 (uap->size > 0 && uap->buffer == 0)) {
179 err = EINVAL;
180 goto out;
181 }
182
183 if (uap->flags & PGO_RESET_ALL) {
184 if (uap->flags != PGO_RESET_ALL || uap->uuid || uap->buffer || uap->size) {
185 err = EINVAL;
186 } else {
187 kern_return_t r = pgo_reset_counters();
188 switch (r) {
189 case KERN_SUCCESS:
190 err = 0;
191 break;
192 case KERN_OPERATION_TIMED_OUT:
193 err = ETIMEDOUT;
194 break;
195 default:
196 err = EIO;
197 break;
198 }
199 }
200 goto out;
201 }
202
203 *retval = 0;
204
205 if (uap->uuid) {
206 uuid_t uuid;
207 err = copyin(uap->uuid, &uuid, sizeof(uuid));
208 if (err) {
209 goto out;
210 }
211
212 if (uap->buffer == 0 && uap->size == 0) {
213 if (uap->flags & PGO_WAIT_FOR_UNLOAD) {
214 err = EINVAL;
215 goto out;
216 }
217
218 err = OSKextGrabPgoData(uuid, pSize: &size64, NULL, bufferSize: 0, wait_for_unload: 0, metadata: !!(uap->flags & PGO_METADATA));
219 if (size64 == 0 && err == 0) {
220 err = EIO;
221 }
222 if (err) {
223 goto out;
224 }
225
226 ssize_t size = size64;
227 if (((uint64_t) size) != size64 ||
228 size < 0) {
229 err = ERANGE;
230 goto out;
231 }
232
233 *retval = size;
234 err = 0;
235 goto out;
236 } else if (!uap->buffer || uap->size <= 0) {
237 err = EINVAL;
238 goto out;
239 } else {
240 err = OSKextGrabPgoData(uuid, pSize: &size64, NULL, bufferSize: 0,
241 false,
242 metadata: !!(uap->flags & PGO_METADATA));
243
244 if (size64 == 0 && err == 0) {
245 err = EIO;
246 }
247 if (err) {
248 goto out;
249 }
250
251 if (uap->size < 0 || (uint64_t)uap->size < size64) {
252 err = EINVAL;
253 goto out;
254 }
255
256 buffer = kalloc_data(size64, Z_WAITOK | Z_ZERO);
257 if (!buffer) {
258 err = ENOMEM;
259 goto out;
260 }
261
262 err = OSKextGrabPgoData(uuid, pSize: &size64, pBuffer: buffer, bufferSize: size64,
263 wait_for_unload: !!(uap->flags & PGO_WAIT_FOR_UNLOAD),
264 metadata: !!(uap->flags & PGO_METADATA));
265 if (err) {
266 goto out;
267 }
268
269 ssize_t size = size64;
270 if (((uint64_t) size) != size64 ||
271 size < 0) {
272 err = ERANGE;
273 goto out;
274 }
275
276 err = copyout(buffer, uap->buffer, size);
277 if (err) {
278 goto out;
279 }
280
281 *retval = size;
282 goto out;
283 }
284 }
285
286
287#ifdef PROFILE
288
289 size64 = get_size_for_buffer(uap->flags);
290 ssize_t size = size64;
291
292 if (uap->flags & (PGO_WAIT_FOR_UNLOAD | PGO_METADATA)) {
293 err = EINVAL;
294 goto out;
295 }
296
297 if (((uint64_t) size) != size64 ||
298 size < 0) {
299 err = ERANGE;
300 goto out;
301 }
302
303
304 if (uap->buffer == 0 && uap->size == 0) {
305 *retval = size;
306 err = 0;
307 goto out;
308 } else if (uap->size < size) {
309 err = EINVAL;
310 goto out;
311 } else {
312 buffer = kalloc_data(size, Z_WAITOK | Z_ZERO);
313 if (!buffer) {
314 err = ENOMEM;
315 goto out;
316 }
317
318 err = write_buffer(uap->flags, buffer);
319 if (err) {
320 err = EIO;
321 goto out;
322 }
323
324 err = copyout(buffer, uap->buffer, size);
325 if (err) {
326 goto out;
327 }
328
329 *retval = size;
330 goto out;
331 }
332
333#else
334
335 *retval = -1;
336 err = ENOSYS;
337 goto out;
338
339#endif
340
341out:
342 if (buffer) {
343 kfree_data(buffer, size64);
344 }
345 if (err) {
346 *retval = -1;
347 }
348 return err;
349}
350