1 | /* |
2 | * Copyright (c) 2014 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <machine/machine_routines.h> |
30 | #include <sys/sysproto.h> |
31 | #include <sys/malloc.h> |
32 | #include <sys/systm.h> |
33 | #include <sys/pgo.h> |
34 | #include <sys/kauth.h> |
35 | #include <security/mac_framework.h> |
36 | #include <libkern/OSKextLib.h> |
37 | |
38 | |
39 | #ifdef PROFILE |
40 | |
41 | /* These __llvm functions are defined in InstrProfiling.h in compiler_rt. That |
42 | * is a internal header, so we need to re-prototype them here. */ |
43 | |
44 | uint64_t __llvm_profile_get_size_for_buffer(void); |
45 | int __llvm_profile_write_buffer(char *Buffer); |
46 | uint64_t __llvm_profile_get_size_for_buffer_internal(const char *DataBegin, |
47 | const char *DataEnd, |
48 | const char *CountersBegin, |
49 | const char *CountersEnd, |
50 | const char *NamesBegin, |
51 | const char *NamesEnd); |
52 | int __llvm_profile_write_buffer_internal(char *Buffer, |
53 | const char *DataBegin, |
54 | const char *DataEnd, |
55 | const char *CountersBegin, |
56 | const char *CountersEnd, |
57 | const char *NamesBegin, |
58 | const char *NamesEnd); |
59 | |
60 | extern char __pgo_hib_DataStart __asm("section$start$__HIB$__llvm_prf_data" ); |
61 | extern char __pgo_hib_DataEnd __asm("section$end$__HIB$__llvm_prf_data" ); |
62 | extern char __pgo_hib_NamesStart __asm("section$start$__HIB$__llvm_prf_names" ); |
63 | extern char __pgo_hib_NamesEnd __asm("section$end$__HIB$__llvm_prf_names" ); |
64 | extern char __pgo_hib_CountersStart __asm("section$start$__HIB$__llvm_prf_cnts" ); |
65 | extern char __pgo_hib_CountersEnd __asm("section$end$__HIB$__llvm_prf_cnts" ); |
66 | |
67 | |
68 | static uint64_t |
69 | get_size_for_buffer(int flags) |
70 | { |
71 | if (flags & PGO_HIB) { |
72 | return __llvm_profile_get_size_for_buffer_internal( |
73 | &__pgo_hib_DataStart, &__pgo_hib_DataEnd, |
74 | &__pgo_hib_CountersStart, &__pgo_hib_CountersEnd, |
75 | &__pgo_hib_NamesStart, &__pgo_hib_NamesEnd); |
76 | } else { |
77 | return __llvm_profile_get_size_for_buffer(); |
78 | } |
79 | } |
80 | |
81 | |
82 | static int |
83 | write_buffer(int flags, char *buffer) |
84 | { |
85 | if (flags & PGO_HIB) { |
86 | return __llvm_profile_write_buffer_internal( |
87 | buffer, |
88 | &__pgo_hib_DataStart, &__pgo_hib_DataEnd, |
89 | &__pgo_hib_CountersStart, &__pgo_hib_CountersEnd, |
90 | &__pgo_hib_NamesStart, &__pgo_hib_NamesEnd); |
91 | } else { |
92 | return __llvm_profile_write_buffer(buffer); |
93 | } |
94 | } |
95 | |
96 | |
97 | #endif |
98 | |
99 | /* this variable is used to signal to the debugger that we'd like it to reset |
100 | * the counters */ |
101 | int kdp_pgo_reset_counters = 0; |
102 | |
103 | /* called in debugger context */ |
104 | kern_return_t |
105 | do_pgo_reset_counters() |
106 | { |
107 | #ifdef PROFILE |
108 | memset(&__pgo_hib_CountersStart, 0, |
109 | ((uintptr_t)(&__pgo_hib_CountersEnd)) - ((uintptr_t)(&__pgo_hib_CountersStart))); |
110 | #endif |
111 | OSKextResetPgoCounters(); |
112 | kdp_pgo_reset_counters = 0; |
113 | return KERN_SUCCESS; |
114 | } |
115 | |
116 | static kern_return_t |
117 | kextpgo_trap() |
118 | { |
119 | return DebuggerTrapWithState(db_op: DBOP_RESET_PGO_COUNTERS, NULL, NULL, NULL, db_panic_options: 0, NULL, FALSE, db_panic_caller: 0); |
120 | } |
121 | |
122 | static kern_return_t |
123 | pgo_reset_counters() |
124 | { |
125 | kern_return_t r; |
126 | boolean_t istate; |
127 | |
128 | OSKextResetPgoCountersLock(); |
129 | |
130 | istate = ml_set_interrupts_enabled(FALSE); |
131 | |
132 | kdp_pgo_reset_counters = 1; |
133 | r = kextpgo_trap(); |
134 | |
135 | ml_set_interrupts_enabled(enable: istate); |
136 | |
137 | OSKextResetPgoCountersUnlock(); |
138 | return r; |
139 | } |
140 | |
141 | |
142 | /* |
143 | * returns: |
144 | * EPERM unless you are root |
145 | * EINVAL for invalid args. |
146 | * ENOSYS for not implemented |
147 | * ERANGE for integer overflow |
148 | * ENOENT if kext not found |
149 | * ENOTSUP kext does not support PGO |
150 | * EIO llvm returned an error. shouldn't ever happen. |
151 | */ |
152 | |
153 | int |
154 | grab_pgo_data(struct proc *p, |
155 | struct grab_pgo_data_args *uap, |
156 | register_t *retval) |
157 | { |
158 | char *buffer = NULL; |
159 | uint64_t size64 = 0; |
160 | int err = 0; |
161 | |
162 | (void) p; |
163 | |
164 | if (!kauth_cred_issuser(cred: kauth_cred_get())) { |
165 | err = EPERM; |
166 | goto out; |
167 | } |
168 | |
169 | #if CONFIG_MACF |
170 | err = mac_system_check_info(kauth_cred_get(), info_type: "kern.profiling_data" ); |
171 | if (err) { |
172 | goto out; |
173 | } |
174 | #endif |
175 | |
176 | if (uap->flags & ~PGO_ALL_FLAGS || |
177 | uap->size < 0 || |
178 | (uap->size > 0 && uap->buffer == 0)) { |
179 | err = EINVAL; |
180 | goto out; |
181 | } |
182 | |
183 | if (uap->flags & PGO_RESET_ALL) { |
184 | if (uap->flags != PGO_RESET_ALL || uap->uuid || uap->buffer || uap->size) { |
185 | err = EINVAL; |
186 | } else { |
187 | kern_return_t r = pgo_reset_counters(); |
188 | switch (r) { |
189 | case KERN_SUCCESS: |
190 | err = 0; |
191 | break; |
192 | case KERN_OPERATION_TIMED_OUT: |
193 | err = ETIMEDOUT; |
194 | break; |
195 | default: |
196 | err = EIO; |
197 | break; |
198 | } |
199 | } |
200 | goto out; |
201 | } |
202 | |
203 | *retval = 0; |
204 | |
205 | if (uap->uuid) { |
206 | uuid_t uuid; |
207 | err = copyin(uap->uuid, &uuid, sizeof(uuid)); |
208 | if (err) { |
209 | goto out; |
210 | } |
211 | |
212 | if (uap->buffer == 0 && uap->size == 0) { |
213 | if (uap->flags & PGO_WAIT_FOR_UNLOAD) { |
214 | err = EINVAL; |
215 | goto out; |
216 | } |
217 | |
218 | err = OSKextGrabPgoData(uuid, pSize: &size64, NULL, bufferSize: 0, wait_for_unload: 0, metadata: !!(uap->flags & PGO_METADATA)); |
219 | if (size64 == 0 && err == 0) { |
220 | err = EIO; |
221 | } |
222 | if (err) { |
223 | goto out; |
224 | } |
225 | |
226 | ssize_t size = size64; |
227 | if (((uint64_t) size) != size64 || |
228 | size < 0) { |
229 | err = ERANGE; |
230 | goto out; |
231 | } |
232 | |
233 | *retval = size; |
234 | err = 0; |
235 | goto out; |
236 | } else if (!uap->buffer || uap->size <= 0) { |
237 | err = EINVAL; |
238 | goto out; |
239 | } else { |
240 | err = OSKextGrabPgoData(uuid, pSize: &size64, NULL, bufferSize: 0, |
241 | false, |
242 | metadata: !!(uap->flags & PGO_METADATA)); |
243 | |
244 | if (size64 == 0 && err == 0) { |
245 | err = EIO; |
246 | } |
247 | if (err) { |
248 | goto out; |
249 | } |
250 | |
251 | if (uap->size < 0 || (uint64_t)uap->size < size64) { |
252 | err = EINVAL; |
253 | goto out; |
254 | } |
255 | |
256 | buffer = kalloc_data(size64, Z_WAITOK | Z_ZERO); |
257 | if (!buffer) { |
258 | err = ENOMEM; |
259 | goto out; |
260 | } |
261 | |
262 | err = OSKextGrabPgoData(uuid, pSize: &size64, pBuffer: buffer, bufferSize: size64, |
263 | wait_for_unload: !!(uap->flags & PGO_WAIT_FOR_UNLOAD), |
264 | metadata: !!(uap->flags & PGO_METADATA)); |
265 | if (err) { |
266 | goto out; |
267 | } |
268 | |
269 | ssize_t size = size64; |
270 | if (((uint64_t) size) != size64 || |
271 | size < 0) { |
272 | err = ERANGE; |
273 | goto out; |
274 | } |
275 | |
276 | err = copyout(buffer, uap->buffer, size); |
277 | if (err) { |
278 | goto out; |
279 | } |
280 | |
281 | *retval = size; |
282 | goto out; |
283 | } |
284 | } |
285 | |
286 | |
287 | #ifdef PROFILE |
288 | |
289 | size64 = get_size_for_buffer(uap->flags); |
290 | ssize_t size = size64; |
291 | |
292 | if (uap->flags & (PGO_WAIT_FOR_UNLOAD | PGO_METADATA)) { |
293 | err = EINVAL; |
294 | goto out; |
295 | } |
296 | |
297 | if (((uint64_t) size) != size64 || |
298 | size < 0) { |
299 | err = ERANGE; |
300 | goto out; |
301 | } |
302 | |
303 | |
304 | if (uap->buffer == 0 && uap->size == 0) { |
305 | *retval = size; |
306 | err = 0; |
307 | goto out; |
308 | } else if (uap->size < size) { |
309 | err = EINVAL; |
310 | goto out; |
311 | } else { |
312 | buffer = kalloc_data(size, Z_WAITOK | Z_ZERO); |
313 | if (!buffer) { |
314 | err = ENOMEM; |
315 | goto out; |
316 | } |
317 | |
318 | err = write_buffer(uap->flags, buffer); |
319 | if (err) { |
320 | err = EIO; |
321 | goto out; |
322 | } |
323 | |
324 | err = copyout(buffer, uap->buffer, size); |
325 | if (err) { |
326 | goto out; |
327 | } |
328 | |
329 | *retval = size; |
330 | goto out; |
331 | } |
332 | |
333 | #else |
334 | |
335 | *retval = -1; |
336 | err = ENOSYS; |
337 | goto out; |
338 | |
339 | #endif |
340 | |
341 | out: |
342 | if (buffer) { |
343 | kfree_data(buffer, size64); |
344 | } |
345 | if (err) { |
346 | *retval = -1; |
347 | } |
348 | return err; |
349 | } |
350 | |